index/mod_dom.c

   1 /* $Id: mod_dom.c,v 1.24 2007-02-28 16:46:19 marc Exp $
   2    Copyright (C) 1995-2007
   3    Index Data ApS
   4
   5    This file is part of the Zebra server.
   6
   7    Zebra is free software; you can redistribute it and/or modify it under
   8    the terms of the GNU General Public License as published by the Free
   9    Software Foundation; either version 2, or (at your option) any later
  10    version.
  11
  12    Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15    for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, write to the Free Software
  19    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #include <ctype.h>
  26 #include <stdarg.h>
  27
  28 #include <yaz/diagbib1.h>
  29 #include <yaz/tpath.h>
  30 #include <yaz/snprintf.h>
  31
  32 #include <libxml/xmlversion.h>
  33 #include <libxml/parser.h>
  34 #include <libxml/tree.h>
  35 #include <libxml/xmlIO.h>
  36 #include <libxml/xmlreader.h>
  37 #include <libxslt/transform.h>
  38 #include <libxslt/xsltutils.h>
  39
  40 #if YAZ_HAVE_EXSLT
  41 #include <libexslt/exslt.h>
  42 #endif
  43
  44 #include <idzebra/util.h>
  45 #include <idzebra/recctrl.h>
  46
  47 /* DOM filter style indexing */
  48 #define ZEBRA_DOM_NS "http://indexdata.com/zebra-2.0"
  49 static const char *zebra_dom_ns = ZEBRA_DOM_NS;
  50
  51 /* DOM filter style indexing */
  52 #define ZEBRA_PI_NAME "zebra-2.0"
  53 static const char *zebra_pi_name = ZEBRA_PI_NAME;
  54
  55
  56
  57 struct convert_s {
  58     const char *stylesheet;
  59     xsltStylesheetPtr stylesheet_xsp;
  60     struct convert_s *next;
  61 };
  62
  63 struct filter_extract {
  64     const char *name;
  65     struct convert_s *convert;
  66 };
  67
  68 struct filter_store {
  69     struct convert_s *convert;
  70 };
  71
  72 struct filter_retrieve {
  73     const char *name;
  74     const char *identifier;
  75     struct convert_s *convert;
  76     struct filter_retrieve *next;
  77 };
  78
  79 #define DOM_INPUT_XMLREADER 1
  80 #define DOM_INPUT_MARC 2
  81 struct filter_input {
  82     const char *syntax;
  83     const char *name;
  84     struct convert_s *convert;
  85     int type;
  86     union {
  87         struct {
  88             const char *input_charset;
  89             yaz_marc_t handle;
  90             yaz_iconv_t iconv;
  91         } marc;
  92         struct {
  93             xmlTextReaderPtr reader;
  94             int split_level;
  95         } xmlreader;
  96     } u;
  97     struct filter_input *next;
  98 };
  99
 100 struct filter_info {
 101     char *fname;
 102     char *full_name;
 103     const char *profile_path;
 104     ODR odr_record;
 105     ODR odr_config;
 106     xmlDocPtr doc_config;
 107     struct filter_extract *extract;
 108     struct filter_retrieve *retrieve_list;
 109     struct filter_input *input_list;
 110     struct filter_store *store;
 111 };
 112
 113
 114
 115 #define XML_STRCMP(a,b)   strcmp((char*)a, b)
 116 #define XML_STRLEN(a) strlen((char*)a)
 117
 118
 119 #define FOR_EACH_ELEMENT(ptr) for (; ptr; ptr = ptr->next) if (ptr->type == XML_ELEMENT_NODE)
 120
 121 static void dom_log(int level, struct filter_info *tinfo, xmlNodePtr ptr,
 122                     const char *fmt, ...)
 123 #ifdef __GNUC__
 124     __attribute__ ((format (printf, 4, 5)))
 125 #endif
 126     ;
 127
 128 static void dom_log(int level, struct filter_info *tinfo, xmlNodePtr ptr,
 129                     const char *fmt, ...)
 130 {
 131     va_list ap;
 132     char buf[4096];
 133
 134     va_start(ap, fmt);
 135     yaz_vsnprintf(buf, sizeof(buf)-1, fmt, ap);
 136     if (ptr)
 137     {
 138         yaz_log(level, "%s:%ld: %s", tinfo->fname ? tinfo->fname : "none",
 139                 xmlGetLineNo(ptr), buf);
 140     }
 141     else
 142     {
 143         yaz_log(level, "%s: %s", tinfo->fname ? tinfo->fname : "none", buf);
 144     }
 145     va_end(ap);
 146 }
 147
 148
 149 static void set_param_str(const char **params, const char *name,
 150                           const char *value, ODR odr)
 151 {
 152     char *quoted = odr_malloc(odr, 3 + strlen(value));
 153     sprintf(quoted, "'%s'", value);
 154     while (*params)
 155         params++;
 156     params[0] = name;
 157     params[1] = quoted;
 158     params[2] = 0;
 159 }
 160
 161 static void set_param_int(const char **params, const char *name,
 162                           zint value, ODR odr)
 163 {
 164     char *quoted = odr_malloc(odr, 30); /* 25 digits enough for 2^64 */
 165     while (*params)
 166         params++;
 167     sprintf(quoted, "'" ZINT_FORMAT "'", value);
 168     params[0] = name;
 169     params[1] = quoted;
 170     params[2] = 0;
 171 }
 172
 173 static void *filter_init(Res res, RecType recType)
 174 {
 175     struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo));
 176     tinfo->fname = 0;
 177     tinfo->full_name = 0;
 178     tinfo->profile_path = 0;
 179     tinfo->odr_record = odr_createmem(ODR_ENCODE);
 180     tinfo->odr_config = odr_createmem(ODR_ENCODE);
 181     tinfo->extract = 0;
 182     tinfo->retrieve_list = 0;
 183     tinfo->input_list = 0;
 184     tinfo->store = 0;
 185     tinfo->doc_config = 0;
 186
 187 #if YAZ_HAVE_EXSLT
 188     exsltRegisterAll();
 189 #endif
 190
 191     return tinfo;
 192 }
 193
 194 static int attr_content(struct _xmlAttr *attr, const char *name,
 195                         const char **dst_content)
 196 {
 197     if (!XML_STRCMP(attr->name, name) && attr->children
 198         && attr->children->type == XML_TEXT_NODE)
 199     {
 200         *dst_content = (const char *)(attr->children->content);
 201         return 1;
 202     }
 203     return 0;
 204 }
 205
 206 static void destroy_xsp(struct convert_s *c)
 207 {
 208     while(c)
 209     {
 210         if (c->stylesheet_xsp)
 211             xsltFreeStylesheet(c->stylesheet_xsp);
 212         c = c->next;
 213     }
 214 }
 215
 216 static void destroy_dom(struct filter_info *tinfo)
 217 {
 218     if (tinfo->extract)
 219     {
 220         destroy_xsp(tinfo->extract->convert);
 221         tinfo->extract = 0;
 222     }
 223     if (tinfo->store)
 224     {
 225         destroy_xsp(tinfo->store->convert);
 226         tinfo->store = 0;
 227     }
 228     if (tinfo->input_list)
 229     {
 230         struct filter_input *i_ptr;
 231         for (i_ptr = tinfo->input_list; i_ptr; i_ptr = i_ptr->next)
 232         {
 233             switch(i_ptr->type)
 234             {
 235             case DOM_INPUT_XMLREADER:
 236                 if (i_ptr->u.xmlreader.reader)
 237                     xmlFreeTextReader(i_ptr->u.xmlreader.reader);
 238                 break;
 239             case DOM_INPUT_MARC:
 240                 yaz_iconv_close(i_ptr->u.marc.iconv);
 241                 yaz_marc_destroy(i_ptr->u.marc.handle);
 242                 break;
 243             }
 244             destroy_xsp(i_ptr->convert);
 245         }
 246         tinfo->input_list = 0;
 247     }
 248     if (tinfo->retrieve_list)
 249     {
 250         struct filter_retrieve *r_ptr;
 251         for (r_ptr = tinfo->retrieve_list; r_ptr; r_ptr = r_ptr->next)
 252             destroy_xsp(r_ptr->convert);
 253         tinfo->retrieve_list = 0;
 254     }
 255
 256     if (tinfo->doc_config)
 257     {
 258         xmlFreeDoc(tinfo->doc_config);
 259         tinfo->doc_config = 0;
 260     }
 261     odr_reset(tinfo->odr_config);
 262 }
 263
 264 static ZEBRA_RES parse_convert(struct filter_info *tinfo, xmlNodePtr ptr,
 265                                struct convert_s **l)
 266 {
 267     *l = 0;
 268     FOR_EACH_ELEMENT(ptr) {
 269         if (!XML_STRCMP(ptr->name, "xslt"))
 270         {
 271             struct _xmlAttr *attr;
 272             struct convert_s *p
 273                 = odr_malloc(tinfo->odr_config, sizeof(*p));
 274
 275             p->next = 0;
 276             p->stylesheet = 0;
 277             p->stylesheet_xsp = 0;
 278
 279             for (attr = ptr->properties; attr; attr = attr->next)
 280                 if (attr_content(attr, "stylesheet", &p->stylesheet))
 281                     ;
 282                 else
 283                 {
 284                     dom_log(YLOG_WARN, tinfo, ptr,
 285                             "bad attribute @%s", attr->name);
 286                 }
 287             if (p->stylesheet)
 288             {
 289                 char tmp_xslt_full_name[1024];
 290                 if (!yaz_filepath_resolve(p->stylesheet,
 291                                           tinfo->profile_path,
 292                                           NULL,
 293                                           tmp_xslt_full_name))
 294                 {
 295                     dom_log(YLOG_WARN, tinfo, 0,
 296                             "stylesheet %s not found in "
 297                             "path %s",
 298                             p->stylesheet,
 299                             tinfo->profile_path);
 300                     return ZEBRA_FAIL;
 301                 }
 302
 303                 p->stylesheet_xsp
 304                     = xsltParseStylesheetFile((const xmlChar*)
 305                                               tmp_xslt_full_name);
 306                 if (!p->stylesheet_xsp)
 307                 {
 308                     dom_log(YLOG_WARN, tinfo, 0,
 309                             "could not parse xslt stylesheet %s",
 310                             tmp_xslt_full_name);
 311                     return ZEBRA_FAIL;
 312                 }
 313                 }
 314                 else
 315                 {
 316                     dom_log(YLOG_WARN, tinfo, ptr,
 317                             "missing attribute 'stylesheet' ");
 318                     return ZEBRA_FAIL;
 319                 }
 320                 *l = p;
 321                 l = &p->next;
 322         }
 323         else
 324         {
 325             dom_log(YLOG_WARN, tinfo, ptr,
 326                     "bad element '%s', expected <xslt>", ptr->name);
 327             return ZEBRA_FAIL;
 328         }
 329     }
 330     return ZEBRA_OK;
 331 }
 332
 333 static ZEBRA_RES perform_convert(struct filter_info *tinfo,
 334                                  struct recExtractCtrl *extctr,
 335                                  struct convert_s *convert,
 336                                  const char **params,
 337                                  xmlDocPtr *doc,
 338                                  xsltStylesheetPtr *last_xsp)
 339 {
 340     for (; convert; convert = convert->next)
 341     {
 342         xmlChar *buf_out = 0;
 343         int len_out = 0;
 344         xmlDocPtr res_doc = xsltApplyStylesheet(convert->stylesheet_xsp,
 345                                                 *doc, params);
 346         if (last_xsp)
 347             *last_xsp = convert->stylesheet_xsp;
 348
 349         xmlFreeDoc(*doc);
 350
 351         /* now saving into buffer and re-reading into DOM to avoid annoing
 352            XSLT problem with thrown-out indentation text nodes */
 353         if (res_doc){
 354             xsltSaveResultToString(&buf_out, &len_out, res_doc,
 355                                    convert->stylesheet_xsp);
 356             xmlFreeDoc(res_doc);
 357         }
 358
 359
 360         *doc =  xmlParseDoc(buf_out);
 361
 362         /* writing debug info out */
 363         if (extctr->flagShowRecords)
 364             yaz_log(YLOG_LOG, "%s: XSLT %s \n %s",
 365                     tinfo->fname ? tinfo->fname : "(none)",
 366                     convert->stylesheet,
 367                     buf_out);
 368
 369         xmlFree(buf_out);
 370     }
 371     return ZEBRA_OK;
 372 }
 373
 374 static struct filter_input *new_input(struct filter_info *tinfo, int type)
 375 {
 376     struct filter_input *p;
 377     struct filter_input **np = &tinfo->input_list;
 378     for (;*np; np = &(*np)->next)
 379         ;
 380     p = *np = odr_malloc(tinfo->odr_config, sizeof(*p));
 381     p->next = 0;
 382     p->syntax = 0;
 383     p->name = 0;
 384     p->convert = 0;
 385     p->type = type;
 386     return p;
 387 }
 388
 389 static ZEBRA_RES parse_input(struct filter_info *tinfo, xmlNodePtr ptr,
 390                              const char *syntax, const char *name)
 391 {
 392     FOR_EACH_ELEMENT(ptr) {
 393         if (!XML_STRCMP(ptr->name, "marc"))
 394         {
 395             yaz_iconv_t iconv = 0;
 396             const char *input_charset = "marc-8";
 397             struct _xmlAttr *attr;
 398
 399             for (attr = ptr->properties; attr; attr = attr->next)
 400             {
 401                 if (attr_content(attr, "inputcharset", &input_charset))
 402                     ;
 403                 else
 404                 {
 405                     dom_log(YLOG_WARN, tinfo, ptr,
 406                             "bad attribute @%s, expected @inputcharset",
 407                             attr->name);
 408                 }
 409             }
 410             iconv = yaz_iconv_open("utf-8", input_charset);
 411             if (!iconv)
 412             {
 413                 dom_log(YLOG_WARN, tinfo, ptr,
 414                         "unsupported @charset '%s'", input_charset);
 415                 return ZEBRA_FAIL;
 416             }
 417             else
 418             {
 419                 struct filter_input *p
 420                     = new_input(tinfo, DOM_INPUT_MARC);
 421                 p->u.marc.handle = yaz_marc_create();
 422                 p->u.marc.iconv = iconv;
 423
 424                 yaz_marc_iconv(p->u.marc.handle, p->u.marc.iconv);
 425
 426                 ptr = ptr->next;
 427
 428                 parse_convert(tinfo, ptr, &p->convert);
 429             }
 430             break;
 431
 432         }
 433         else if (!XML_STRCMP(ptr->name, "xmlreader"))
 434         {
 435             struct filter_input *p
 436                 = new_input(tinfo, DOM_INPUT_XMLREADER);
 437             struct _xmlAttr *attr;
 438             const char *level_str = 0;
 439
 440             p->u.xmlreader.split_level = 0;
 441             p->u.xmlreader.reader = 0;
 442
 443             for (attr = ptr->properties; attr; attr = attr->next)
 444             {
 445                 if (attr_content(attr, "level", &level_str))
 446                     ;
 447                 else
 448                 {
 449                     dom_log(YLOG_WARN, tinfo, ptr,
 450                             "bad attribute @%s, expected @level",
 451                             attr->name);
 452                 }
 453             }
 454             if (level_str)
 455                 p->u.xmlreader.split_level = atoi(level_str);
 456
 457             ptr = ptr->next;
 458
 459             parse_convert(tinfo, ptr, &p->convert);
 460             break;
 461         }
 462         else
 463         {
 464             dom_log(YLOG_WARN, tinfo, ptr,
 465                     "bad element <%s>, expected <marc>|<xmlreader>",
 466                     ptr->name);
 467             return ZEBRA_FAIL;
 468         }
 469     }
 470     return ZEBRA_OK;
 471 }
 472
 473 static ZEBRA_RES parse_dom(struct filter_info *tinfo, const char *fname)
 474 {
 475     char tmp_full_name[1024];
 476     xmlNodePtr ptr;
 477     xmlDocPtr doc;
 478
 479     tinfo->fname = odr_strdup(tinfo->odr_config, fname);
 480
 481     if (yaz_filepath_resolve(tinfo->fname, tinfo->profile_path,
 482                              NULL, tmp_full_name))
 483         tinfo->full_name = odr_strdup(tinfo->odr_config, tmp_full_name);
 484     else
 485         tinfo->full_name = odr_strdup(tinfo->odr_config, tinfo->fname);
 486
 487     yaz_log(YLOG_LOG, "%s dom filter: "
 488             "loading config file %s", tinfo->fname, tinfo->full_name);
 489
 490     doc = xmlParseFile(tinfo->full_name);
 491     if (!doc)
 492     {
 493         yaz_log(YLOG_WARN, "%s: dom filter: "
 494                 "failed to parse config file %s",
 495                 tinfo->fname, tinfo->full_name);
 496         return ZEBRA_FAIL;
 497     }
 498     /* save because we store ptrs to the content */
 499     tinfo->doc_config = doc;
 500
 501     ptr = xmlDocGetRootElement(doc);
 502     if (!ptr || ptr->type != XML_ELEMENT_NODE
 503         || XML_STRCMP(ptr->name, "dom"))
 504     {
 505         dom_log(YLOG_WARN, tinfo, ptr,
 506                 "bad root element <%s>, expected root element <dom>",
 507                 ptr->name);
 508         return ZEBRA_FAIL;
 509     }
 510
 511     ptr = ptr->children;
 512     FOR_EACH_ELEMENT(ptr) {
 513         if (!XML_STRCMP(ptr->name, "extract"))
 514         {
 515             /*
 516               <extract name="index">
 517               <xslt stylesheet="first.xsl"/>
 518               <xslt stylesheet="second.xsl"/>
 519               </extract>
 520             */
 521             struct _xmlAttr *attr;
 522             struct filter_extract *f =
 523                 odr_malloc(tinfo->odr_config, sizeof(*f));
 524
 525             tinfo->extract = f;
 526             f->name = 0;
 527             f->convert = 0;
 528             for (attr = ptr->properties; attr; attr = attr->next)
 529             {
 530                 if (attr_content(attr, "name", &f->name))
 531                     ;
 532                 else
 533                 {
 534                     dom_log(YLOG_WARN, tinfo, ptr,
 535                             "bad attribute @%s, expected @name",
 536                             attr->name);
 537                 }
 538             }
 539             parse_convert(tinfo, ptr->children, &f->convert);
 540         }
 541         else if (!XML_STRCMP(ptr->name, "retrieve"))
 542         {
 543             /*
 544                <retrieve name="F">
 545                <xslt stylesheet="some.xsl"/>
 546                <xslt stylesheet="some.xsl"/>
 547                </retrieve>
 548             */
 549             struct _xmlAttr *attr;
 550             struct filter_retrieve **fp = &tinfo->retrieve_list;
 551             struct filter_retrieve *f =
 552                 odr_malloc(tinfo->odr_config, sizeof(*f));
 553
 554             while (*fp)
 555                 fp = &(*fp)->next;
 556
 557             *fp = f;
 558             f->name = 0;
 559             f->identifier = 0;
 560             f->convert = 0;
 561             f->next = 0;
 562
 563             for (attr = ptr->properties; attr; attr = attr->next)
 564             {
 565                 if (attr_content(attr, "identifier",
 566                                  &f->identifier))
 567                     ;
 568                 else if (attr_content(attr, "name", &f->name))
 569                     ;
 570                 else
 571                 {
 572                     dom_log(YLOG_WARN, tinfo, ptr,
 573                             "bad attribute @%s,  expected @identifier|@name",
 574                             attr->name);
 575                 }
 576             }
 577             parse_convert(tinfo, ptr->children, &f->convert);
 578         }
 579         else if (!XML_STRCMP(ptr->name, "store"))
 580         {
 581             /*
 582               <store name="F">
 583               <xslt stylesheet="some.xsl"/>
 584               <xslt stylesheet="some.xsl"/>
 585               </retrieve>
 586             */
 587             struct filter_store *f =
 588                 odr_malloc(tinfo->odr_config, sizeof(*f));
 589
 590             tinfo->store = f;
 591             f->convert = 0;
 592             parse_convert(tinfo, ptr->children, &f->convert);
 593         }
 594         else if (!XML_STRCMP(ptr->name, "input"))
 595         {
 596             /*
 597               <input syntax="xml">
 598               <xmlreader level="1"/>
 599               </input>
 600               <input syntax="usmarc">
 601               <marc inputcharset="marc-8"/>
 602               </input>
 603             */
 604             struct _xmlAttr *attr;
 605             const char  *syntax = 0;
 606             const char *name = 0;
 607             for (attr = ptr->properties; attr; attr = attr->next)
 608             {
 609                 if (attr_content(attr, "syntax", &syntax))
 610                     ;
 611                 else if (attr_content(attr, "name", &name))
 612                     ;
 613                 else
 614                 {
 615                     dom_log(YLOG_WARN, tinfo, ptr,
 616                             "bad attribute @%s,  expected @syntax|@name",
 617                             attr->name);
 618                 }
 619             }
 620             parse_input(tinfo, ptr->children, syntax, name);
 621         }
 622         else
 623         {
 624             dom_log(YLOG_WARN, tinfo, ptr,
 625                     "bad element <%s>, "
 626                     "expected <extract>|<input>|<retrieve>|<store>",
 627                     ptr->name);
 628             return ZEBRA_FAIL;
 629         }
 630     }
 631     return ZEBRA_OK;
 632 }
 633
 634 static struct filter_retrieve *lookup_retrieve(struct filter_info *tinfo,
 635                                                const char *est)
 636 {
 637     struct filter_retrieve *f = tinfo->retrieve_list;
 638
 639     /* return first schema if no est is provided */
 640     if (!est)
 641         return f;
 642     for (; f; f = f->next)
 643     {
 644         /* find requested schema */
 645         if (est)
 646         {
 647             if (f->identifier && !strcmp(f->identifier, est))
 648                 return f;
 649             if (f->name && !strcmp(f->name, est))
 650                 return f;
 651         }
 652     }
 653     return 0;
 654 }
 655
 656 static ZEBRA_RES filter_config(void *clientData, Res res, const char *args)
 657 {
 658     struct filter_info *tinfo = clientData;
 659     if (!args || !*args)
 660     {
 661         yaz_log(YLOG_WARN, "dom filter: need config file");
 662         return ZEBRA_FAIL;
 663     }
 664
 665     if (tinfo->fname && !strcmp(args, tinfo->fname))
 666         return ZEBRA_OK;
 667
 668     tinfo->profile_path = res_get(res, "profilePath");
 669
 670     destroy_dom(tinfo);
 671     return parse_dom(tinfo, args);
 672 }
 673
 674 static void filter_destroy(void *clientData)
 675 {
 676     struct filter_info *tinfo = clientData;
 677     destroy_dom(tinfo);
 678     odr_destroy(tinfo->odr_config);
 679     odr_destroy(tinfo->odr_record);
 680     xfree(tinfo);
 681 }
 682
 683 static int ioread_ex(void *context, char *buffer, int len)
 684 {
 685     struct recExtractCtrl *p = context;
 686     return p->stream->readf(p->stream, buffer, len);
 687 }
 688
 689 static int ioclose_ex(void *context)
 690 {
 691     return 0;
 692 }
 693
 694
 695 /* DOM filter style indexing */
 696 static int attr_content_xml(struct _xmlAttr *attr, const char *name,
 697                             xmlChar **dst_content)
 698 {
 699     if (0 == XML_STRCMP(attr->name, name) && attr->children
 700         && attr->children->type == XML_TEXT_NODE)
 701     {
 702         *dst_content = (attr->children->content);
 703         return 1;
 704     }
 705     return 0;
 706 }
 707
 708
 709 /* DOM filter style indexing */
 710 static void index_value_of(struct filter_info *tinfo,
 711                            struct recExtractCtrl *extctr,
 712                            RecWord* recword,
 713                            xmlNodePtr node,
 714                            xmlChar * index_p)
 715 {
 716     xmlChar *text = xmlNodeGetContent(node);
 717     size_t text_len = strlen((const char *)text);
 718
 719     /* if there is no text, we do not need to proceed */
 720     if (text_len)
 721     {
 722         xmlChar *look = index_p;
 723         xmlChar *bval;
 724         xmlChar *eval;
 725
 726         xmlChar index[256];
 727         xmlChar type[256];
 728
 729         /* assingning text to be indexed */
 730         recword->term_buf = (const char *)text;
 731         recword->term_len = text_len;
 732
 733         /* parsing all index name/type pairs */
 734         /* may not start with ' ' or ':' */
 735         while (*look && ' ' != *look && ':' != *look)
 736         {
 737             /* setting name and type to zero */
 738             *index = '\0';
 739             *type = '\0';
 740
 741             /* parsing one index name */
 742             bval = look;
 743             while (*look && ':' != *look && ' ' != *look)
 744             {
 745                 look++;
 746             }
 747             eval = look;
 748             strncpy((char *)index, (const char *)bval, eval - bval);
 749             index[eval - bval] = '\0';
 750
 751
 752             /* parsing one index type, if existing */
 753             if (':' == *look)
 754             {
 755                 look++;
 756
 757                 bval = look;
 758                 while (*look && ' ' != *look)
 759                 {
 760                     look++;
 761                 }
 762                 eval = look;
 763                 strncpy((char *)type, (const char *)bval, eval - bval);
 764                 type[eval - bval] = '\0';
 765             }
 766
 767             /* writing debug out */
 768             if (extctr->flagShowRecords)
 769                 dom_log(YLOG_LOG, tinfo, 0,
 770                         "INDEX '%s:%s' '%s'",
 771                         index ? (const char *) index : "null",
 772                         type ? (const char *) type : "null",
 773                         text ? (const char *) text : "null");
 774
 775             /* actually indexing the text given */
 776             recword->index_name = (const char *)index;
 777             if (type && *type)
 778                 recword->index_type = *type;
 779             (extctr->tokenAdd)(recword);
 780
 781             /* eat whitespaces */
 782             if (*look && ' ' == *look && *(look+1))
 783             {
 784                 look++;
 785             }
 786         }
 787     }
 788
 789     xmlFree(text);
 790 }
 791
 792
 793 /* DOM filter style indexing */
 794 static void set_record_info(struct filter_info *tinfo,
 795                             struct recExtractCtrl *extctr,
 796                             xmlChar * id_p,
 797                             xmlChar * rank_p,
 798                             xmlChar * type_p)
 799 {
 800
 801     /* writing debug info out */
 802     if (extctr->flagShowRecords)
 803         dom_log(YLOG_LOG, tinfo, 0,
 804                 "RECORD id=%s rank=%s type=%s",
 805                 id_p ? (const char *) id_p : "(null)",
 806                 rank_p ? (const char *) rank_p : "(null)",
 807                 type_p ? (const char *) type_p : "(null)");
 808
 809
 810     if (id_p)
 811         sscanf((const char *)id_p, "%255s", extctr->match_criteria);
 812
 813     if (rank_p)
 814         extctr->staticrank = atozint((const char *)rank_p);
 815
 816     /*     if (!strcmp("update", type_str)) */
 817     /*         index_node(tinfo, ctrl, ptr, recword); */
 818     /*     else if (!strcmp("delete", type_str)) */
 819     /*         dom_log(YLOG_WARN, tinfo, ptr, "dom filter delete: to be implemented"); */
 820     /*     else */
 821     /*         dom_log(YLOG_WARN, tinfo, ptr, "dom filter: unknown record type '%s'",  */
 822     /*                 type_str); */
 823
 824 }
 825
 826
 827 /* DOM filter style indexing */
 828 static void process_xml_element_zebra_node(struct filter_info *tinfo,
 829                                            struct recExtractCtrl *extctr,
 830                                            RecWord* recword,
 831                                            xmlNodePtr node)
 832 {
 833     if (node->type == XML_ELEMENT_NODE && node->ns && node->ns->href
 834         && 0 == XML_STRCMP(node->ns->href, zebra_dom_ns))
 835     {
 836          if (0 == XML_STRCMP(node->name, "index"))
 837          {
 838             xmlChar *index_p = 0;
 839
 840             struct _xmlAttr *attr;
 841             for (attr = node->properties; attr; attr = attr->next)
 842             {
 843                 if (attr_content_xml(attr, "name", &index_p))
 844                 {
 845                     index_value_of(tinfo, extctr, recword,node, index_p);
 846                 }
 847                 else
 848                 {
 849                     dom_log(YLOG_WARN, tinfo, node,
 850                             "bad attribute @%s, expected @name",
 851                             attr->name);
 852                 }
 853             }
 854         }
 855         else if (0 == XML_STRCMP(node->name, "record"))
 856         {
 857             xmlChar *id_p = 0;
 858             xmlChar *rank_p = 0;
 859             xmlChar *type_p = 0;
 860
 861             struct _xmlAttr *attr;
 862             for (attr = node->properties; attr; attr = attr->next)
 863             {
 864                 if (attr_content_xml(attr, "id", &id_p))
 865                     ;
 866                 else if (attr_content_xml(attr, "rank", &rank_p))
 867                     ;
 868                 else if (attr_content_xml(attr, "type", &type_p))
 869                     ;
 870                 else
 871                 {
 872                     dom_log(YLOG_WARN, tinfo, node,
 873                             "bad attribute @%s, expected @id|@rank|@type",
 874                             attr->name);
 875                 }
 876
 877                 if (type_p && 0 != strcmp("update", (const char *)type_p))
 878                 {
 879                     dom_log(YLOG_WARN, tinfo, node,
 880                             "attribute @%s, only implemented '@type='update'",
 881                             attr->name);
 882                 }
 883             }
 884             set_record_info(tinfo, extctr, id_p, rank_p, type_p);
 885         }
 886         else
 887         {
 888             dom_log(YLOG_WARN, tinfo, node,
 889                     "bad element <%s>,"
 890                     " expected <record>|<index> in namespace '%s'",
 891                     node->name, zebra_dom_ns);
 892         }
 893     }
 894 }
 895
 896
 897 /* DOM filter style indexing */
 898 static void process_xml_pi_node(struct filter_info *tinfo,
 899                                 struct recExtractCtrl *extctr,
 900                                 xmlNodePtr node,
 901                                 xmlChar **index_pp)
 902 {
 903     /* if right PI name, continue parsing PI */
 904     if (0 == strcmp(zebra_pi_name, (const char *)node->name))
 905     {
 906         xmlChar *pi_p =  node->content;
 907         xmlChar *look = pi_p;
 908
 909         xmlChar *bval;
 910         xmlChar *eval;
 911
 912         /* parsing PI record instructions */
 913         if (0 == strncmp((const char *)look, "record", 6))
 914         {
 915             xmlChar id[256];
 916             xmlChar rank[256];
 917             xmlChar type[256];
 918
 919             *id = '\0';
 920             *rank = '\0';
 921             *type = '\0';
 922
 923             look += 6;
 924
 925             /* eat whitespace */
 926             while (*look && ' ' == *look && *(look+1))
 927                 look++;
 928
 929             /* parse possible id */
 930             if (*look && 0 == strncmp((const char *)look, "id=", 3))
 931             {
 932                 look += 3;
 933                 bval = look;
 934                 while (*look && ' ' != *look)
 935                     look++;
 936                 eval = look;
 937                 strncpy((char *)id, (const char *)bval, eval - bval);
 938                 id[eval - bval] = '\0';
 939             }
 940
 941             /* eat whitespace */
 942             while (*look && ' ' == *look && *(look+1))
 943                 look++;
 944
 945             /* parse possible rank */
 946             if (*look && 0 == strncmp((const char *)look, "rank=", 5))
 947             {
 948                 look += 6;
 949                 bval = look;
 950                 while (*look && ' ' != *look)
 951                     look++;
 952                 eval = look;
 953                 strncpy((char *)rank, (const char *)bval, eval - bval);
 954                 rank[eval - bval] = '\0';
 955             }
 956
 957             /* eat whitespace */
 958             while (*look && ' ' == *look && *(look+1))
 959                 look++;
 960
 961             if (look && '\0' != *look)
 962             {
 963                 dom_log(YLOG_WARN, tinfo, node,
 964                         "content '%s', can not parse '%s'",
 965                         pi_p, look);
 966             }
 967             else
 968                 set_record_info(tinfo, extctr, id, rank, 0);
 969
 970         }
 971         /* parsing index instruction */
 972         else if (0 == strncmp((const char *)look, "index", 5))
 973         {
 974             look += 5;
 975
 976             /* eat whitespace */
 977             while (*look && ' ' == *look && *(look+1))
 978                 look++;
 979
 980             /* export index instructions to outside */
 981             *index_pp = look;
 982         }
 983         else
 984         {
 985             dom_log(YLOG_WARN, tinfo, node,
 986                     "content '%s', can not parse '%s'",
 987                     pi_p, look);
 988         }
 989     }
 990 }
 991
 992 /* DOM filter style indexing */
 993 static void process_xml_element_node(struct filter_info *tinfo,
 994                                      struct recExtractCtrl *extctr,
 995                                      RecWord* recword,
 996                                      xmlNodePtr node)
 997 {
 998     /* remember indexing instruction from PI to next element node */
 999     xmlChar *index_p = 0;
1000
1001     /* check if we are an element node in the special zebra namespace
1002        and either set record data or index value-of node content*/
1003     process_xml_element_zebra_node(tinfo, extctr, recword, node);
1004
1005     /* loop through kid nodes */
1006     for (node = node->children; node; node = node->next)
1007     {
1008         /* check and set PI record and index index instructions */
1009         if (node->type == XML_PI_NODE)
1010         {
1011             process_xml_pi_node(tinfo, extctr, node, &index_p);
1012         }
1013         else if (node->type == XML_ELEMENT_NODE)
1014         {
1015             /* if there was a PI index instruction before this element */
1016             if (index_p)
1017             {
1018                 index_value_of(tinfo, extctr, recword, node, index_p);
1019                 index_p = 0;
1020             }
1021             process_xml_element_node(tinfo, extctr, recword,node);
1022         }
1023         else
1024             continue;
1025     }
1026 }
1027
1028
1029 /* DOM filter style indexing */
1030 static void extract_dom_doc_node(struct filter_info *tinfo,
1031                                  struct recExtractCtrl *extctr,
1032                                  xmlDocPtr doc)
1033 {
1034     xmlChar *buf_out;
1035     int len_out;
1036
1037     /* only need to do the initialization once, reuse recword for all terms */
1038     RecWord recword;
1039     (*extctr->init)(extctr, &recword);
1040
1041     /*
1042     if (extctr->flagShowRecords)
1043     {
1044         xmlDocDumpMemory(doc, &buf_out, &len_out);
1045         fwrite(buf_out, len_out, 1, stdout);
1046         xmlFree(buf_out);
1047     }
1048     */
1049
1050     process_xml_element_node(tinfo, extctr, &recword, (xmlNodePtr)doc);
1051 }
1052
1053
1054
1055
1056 static int convert_extract_doc(struct filter_info *tinfo,
1057                                struct filter_input *input,
1058                                struct recExtractCtrl *p,
1059                                xmlDocPtr doc)
1060
1061 {
1062     xmlChar *buf_out;
1063     int len_out;
1064     const char *params[10];
1065     xsltStylesheetPtr last_xsp = 0;
1066     xmlDocPtr store_doc = 0;
1067
1068     params[0] = 0;
1069     set_param_str(params, "schema", zebra_dom_ns, tinfo->odr_record);
1070
1071     /* input conversion */
1072     perform_convert(tinfo, p, input->convert, params, &doc, 0);
1073
1074     if (tinfo->store)
1075     {
1076         /* store conversion */
1077         store_doc = xmlCopyDoc(doc, 1);
1078         perform_convert(tinfo, p, tinfo->store->convert,
1079                         params, &store_doc, &last_xsp);
1080     }
1081
1082     if (last_xsp)
1083         xsltSaveResultToString(&buf_out, &len_out,
1084                                store_doc ? store_doc : doc, last_xsp);
1085     else
1086         xmlDocDumpMemory(store_doc ? store_doc : doc, &buf_out, &len_out);
1087
1088     /* if (p->flagShowRecords)
1089        fwrite(buf_out, len_out, 1, stdout); */
1090
1091     (*p->setStoreData)(p, buf_out, len_out);
1092     xmlFree(buf_out);
1093
1094     if (store_doc)
1095         xmlFreeDoc(store_doc);
1096
1097     /* extract conversion */
1098     perform_convert(tinfo, p, tinfo->extract->convert, params, &doc, 0);
1099
1100     /* finally, do the indexing */
1101     if (doc)
1102         extract_dom_doc_node(tinfo, p, doc);
1103
1104     if (doc)
1105         xmlFreeDoc(doc);
1106
1107     return RECCTRL_EXTRACT_OK;
1108 }
1109
1110 static int extract_xml_split(struct filter_info *tinfo,
1111                              struct filter_input *input,
1112                              struct recExtractCtrl *p)
1113 {
1114     int ret;
1115
1116     if (p->first_record)
1117     {
1118         if (input->u.xmlreader.reader)
1119             xmlFreeTextReader(input->u.xmlreader.reader);
1120         input->u.xmlreader.reader = xmlReaderForIO(ioread_ex, ioclose_ex,
1121                                                    p /* I/O handler */,
1122                                                    0 /* URL */,
1123                                                    0 /* encoding */,
1124                                                    XML_PARSE_XINCLUDE|
1125                                                    XML_PARSE_NOENT);
1126     }
1127     if (!input->u.xmlreader.reader)
1128         return RECCTRL_EXTRACT_ERROR_GENERIC;
1129
1130     ret = xmlTextReaderRead(input->u.xmlreader.reader);
1131     while (ret == 1)
1132     {
1133         int type = xmlTextReaderNodeType(input->u.xmlreader.reader);
1134         int depth = xmlTextReaderDepth(input->u.xmlreader.reader);
1135         if (type == XML_READER_TYPE_ELEMENT &&
1136             input->u.xmlreader.split_level == depth)
1137         {
1138             xmlNodePtr ptr
1139                 = xmlTextReaderExpand(input->u.xmlreader.reader);
1140             if (ptr)
1141             {
1142                 xmlNodePtr ptr2 = xmlCopyNode(ptr, 1);
1143                 xmlDocPtr doc = xmlNewDoc((const xmlChar*) "1.0");
1144
1145                 xmlDocSetRootElement(doc, ptr2);
1146
1147                 /* writing debug info out */
1148                 if (p->flagShowRecords){
1149                     xmlChar *buf_out = 0;
1150                     int len_out = 0;
1151                     xmlDocDumpMemory(doc, &buf_out, &len_out);
1152                     yaz_log(YLOG_LOG, "%s: XMLREADER depth: %i\n%s",
1153                             tinfo->fname ? tinfo->fname : "(none)",
1154                             depth,
1155                             buf_out);
1156                     xmlFree(buf_out);
1157                 }
1158
1159                 return convert_extract_doc(tinfo, input, p, doc);
1160             }
1161             else
1162             {
1163                 xmlFreeTextReader(input->u.xmlreader.reader);
1164                 input->u.xmlreader.reader = 0;
1165                 return RECCTRL_EXTRACT_ERROR_GENERIC;
1166             }
1167         }
1168         ret = xmlTextReaderRead(input->u.xmlreader.reader);
1169     }
1170     xmlFreeTextReader(input->u.xmlreader.reader);
1171     input->u.xmlreader.reader = 0;
1172     return RECCTRL_EXTRACT_EOF;
1173 }
1174
1175 static int extract_xml_full(struct filter_info *tinfo,
1176                             struct filter_input *input,
1177                             struct recExtractCtrl *p)
1178 {
1179     if (p->first_record) /* only one record per stream */
1180     {
1181         xmlDocPtr doc = xmlReadIO(ioread_ex, ioclose_ex,
1182                                   p /* I/O handler */,
1183                                   0 /* URL */,
1184                                   0 /* encoding */,
1185                                   XML_PARSE_XINCLUDE|XML_PARSE_NOENT);
1186         if (!doc)
1187         {
1188             return RECCTRL_EXTRACT_ERROR_GENERIC;
1189         }
1190         return convert_extract_doc(tinfo, input, p, doc);
1191     }
1192     else
1193         return RECCTRL_EXTRACT_EOF;
1194 }
1195
1196 static int extract_iso2709(struct filter_info *tinfo,
1197                            struct filter_input *input,
1198                            struct recExtractCtrl *p)
1199 {
1200     char buf[100000];
1201     int record_length;
1202     int read_bytes, r;
1203
1204     if (p->stream->readf(p->stream, buf, 5) != 5)
1205         return RECCTRL_EXTRACT_EOF;
1206     while (*buf < '0' || *buf > '9')
1207     {
1208         int i;
1209
1210         dom_log(YLOG_WARN, tinfo, 0,
1211                 "MARC: Skipping bad byte %d (0x%02X)",
1212                 *buf & 0xff, *buf & 0xff);
1213         for (i = 0; i<4; i++)
1214             buf[i] = buf[i+1];
1215
1216         if (p->stream->readf(p->stream, buf+4, 1) != 1)
1217             return RECCTRL_EXTRACT_EOF;
1218     }
1219     record_length = atoi_n (buf, 5);
1220     if (record_length < 25)
1221     {
1222         dom_log(YLOG_WARN, tinfo, 0,
1223                 "MARC record length < 25, is %d",  record_length);
1224         return RECCTRL_EXTRACT_ERROR_GENERIC;
1225     }
1226     read_bytes = p->stream->readf(p->stream, buf+5, record_length-5);
1227     if (read_bytes < record_length-5)
1228     {
1229         dom_log(YLOG_WARN, tinfo, 0,
1230                 "couldn't read whole MARC record");
1231         return RECCTRL_EXTRACT_ERROR_GENERIC;
1232     }
1233     r = yaz_marc_read_iso2709(input->u.marc.handle,  buf, record_length);
1234     if (r < record_length)
1235     {
1236         dom_log (YLOG_WARN, tinfo, 0,
1237                  "parsing of MARC record failed r=%d length=%d",
1238                  r, record_length);
1239         return RECCTRL_EXTRACT_ERROR_GENERIC;
1240     }
1241     else
1242     {
1243         xmlDocPtr rdoc;
1244         xmlNode *root_ptr;
1245         yaz_marc_write_xml(input->u.marc.handle, &root_ptr, 0, 0, 0);
1246         rdoc = xmlNewDoc((const xmlChar*) "1.0");
1247         xmlDocSetRootElement(rdoc, root_ptr);
1248         return convert_extract_doc(tinfo, input, p, rdoc);
1249     }
1250     return RECCTRL_EXTRACT_OK;
1251 }
1252
1253 static int filter_extract(void *clientData, struct recExtractCtrl *p)
1254 {
1255     struct filter_info *tinfo = clientData;
1256     struct filter_input *input = tinfo->input_list;
1257
1258     if (!input)
1259         return RECCTRL_EXTRACT_ERROR_GENERIC;
1260
1261     odr_reset(tinfo->odr_record);
1262     switch(input->type)
1263     {
1264     case DOM_INPUT_XMLREADER:
1265         if (input->u.xmlreader.split_level == 0)
1266             return extract_xml_full(tinfo, input, p);
1267         else
1268             return extract_xml_split(tinfo, input, p);
1269         break;
1270     case DOM_INPUT_MARC:
1271         return extract_iso2709(tinfo, input, p);
1272     }
1273     return RECCTRL_EXTRACT_ERROR_GENERIC;
1274 }
1275
1276 static int ioread_ret(void *context, char *buffer, int len)
1277 {
1278     struct recRetrieveCtrl *p = context;
1279     return p->stream->readf(p->stream, buffer, len);
1280 }
1281
1282 static int ioclose_ret(void *context)
1283 {
1284     return 0;
1285 }
1286
1287 static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p)
1288 {
1289     /* const char *esn = zebra_dom_ns; */
1290     const char *esn = 0;
1291     const char *params[32];
1292     struct filter_info *tinfo = clientData;
1293     xmlDocPtr doc;
1294     struct filter_retrieve *retrieve;
1295     xsltStylesheetPtr last_xsp = 0;
1296
1297     if (p->comp)
1298     {
1299         if (p->comp->which == Z_RecordComp_simple
1300             && p->comp->u.simple->which == Z_ElementSetNames_generic)
1301         {
1302             esn = p->comp->u.simple->u.generic;
1303         }
1304         else if (p->comp->which == Z_RecordComp_complex
1305                  && p->comp->u.complex->generic->elementSpec
1306                  && p->comp->u.complex->generic->elementSpec->which ==
1307                  Z_ElementSpec_elementSetName)
1308         {
1309             esn = p->comp->u.complex->generic->elementSpec->u.elementSetName;
1310         }
1311     }
1312     retrieve = lookup_retrieve(tinfo, esn);
1313     if (!retrieve)
1314     {
1315         p->diagnostic =
1316             YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
1317         return 0;
1318     }
1319
1320     params[0] = 0;
1321     set_param_int(params, "id", p->localno, p->odr);
1322     if (p->fname)
1323         set_param_str(params, "filename", p->fname, p->odr);
1324     if (p->staticrank >= 0)
1325         set_param_int(params, "rank", p->staticrank, p->odr);
1326
1327     if (esn)
1328         set_param_str(params, "schema", esn, p->odr);
1329     else
1330         if (retrieve->name)
1331             set_param_str(params, "schema", retrieve->name, p->odr);
1332         else if (retrieve->identifier)
1333             set_param_str(params, "schema", retrieve->identifier, p->odr);
1334         else
1335             set_param_str(params, "schema", "", p->odr);
1336
1337     if (p->score >= 0)
1338         set_param_int(params, "score", p->score, p->odr);
1339     set_param_int(params, "size", p->recordSize, p->odr);
1340
1341     doc = xmlReadIO(ioread_ret, ioclose_ret, p /* I/O handler */,
1342                     0 /* URL */,
1343                     0 /* encoding */,
1344                     XML_PARSE_XINCLUDE|XML_PARSE_NOENT);
1345     if (!doc)
1346     {
1347         p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1348         return 0;
1349     }
1350
1351     /* retrieve conversion */
1352     perform_convert(tinfo, 0, retrieve->convert, params, &doc, &last_xsp);
1353     if (!doc)
1354     {
1355         p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1356     }
1357     else if (p->input_format == VAL_NONE || p->input_format == VAL_TEXT_XML)
1358     {
1359         xmlChar *buf_out;
1360         int len_out;
1361
1362         if (last_xsp)
1363             xsltSaveResultToString(&buf_out, &len_out, doc, last_xsp);
1364         else
1365             xmlDocDumpMemory(doc, &buf_out, &len_out);
1366
1367         p->output_format = VAL_TEXT_XML;
1368         p->rec_len = len_out;
1369         p->rec_buf = odr_malloc(p->odr, p->rec_len);
1370         memcpy(p->rec_buf, buf_out, p->rec_len);
1371         xmlFree(buf_out);
1372     }
1373     else if (p->output_format == VAL_SUTRS)
1374     {
1375         xmlChar *buf_out;
1376         int len_out;
1377
1378         if (last_xsp)
1379             xsltSaveResultToString(&buf_out, &len_out, doc, last_xsp);
1380         else
1381             xmlDocDumpMemory(doc, &buf_out, &len_out);
1382
1383         p->output_format = VAL_SUTRS;
1384         p->rec_len = len_out;
1385         p->rec_buf = odr_malloc(p->odr, p->rec_len);
1386         memcpy(p->rec_buf, buf_out, p->rec_len);
1387
1388         xmlFree(buf_out);
1389     }
1390     else
1391     {
1392         p->diagnostic = YAZ_BIB1_RECORD_SYNTAX_UNSUPP;
1393     }
1394     xmlFreeDoc(doc);
1395     return 0;
1396 }
1397
1398 static struct recType filter_type = {
1399     0,
1400     "dom",
1401     filter_init,
1402     filter_config,
1403     filter_destroy,
1404     filter_extract,
1405     filter_retrieve
1406 };
1407
1408 RecType
1409 #ifdef IDZEBRA_STATIC_DOM
1410 idzebra_filter_dom
1411 #else
1412 idzebra_filter
1413 #endif
1414
1415 [] = {
1416     &filter_type,
1417     0,
1418 };
1419 /*
1420  * Local variables:
1421  * c-basic-offset: 4
1422  * indent-tabs-mode: nil
1423  * End:
1424  * vim: shiftwidth=4 tabstop=8 expandtab
1425  */
1426