index/mod_dom.c

   1 /* $Id: mod_dom.c,v 1.20 2007-02-23 14:59:12 adam Exp $
   2    Copyright (C) 1995-2007
   3    Index Data ApS
   4
   5    This file is part of the Zebra server.
   6
   7    Zebra is free software; you can redistribute it and/or modify it under
   8    the terms of the GNU General Public License as published by the Free
   9    Software Foundation; either version 2, or (at your option) any later
  10    version.
  11
  12    Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15    for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, write to the Free Software
  19    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #include <ctype.h>
  26 #include <stdarg.h>
  27
  28 #include <yaz/diagbib1.h>
  29 #include <yaz/tpath.h>
  30 #include <yaz/snprintf.h>
  31
  32 #include <libxml/xmlversion.h>
  33 #include <libxml/parser.h>
  34 #include <libxml/tree.h>
  35 #include <libxml/xmlIO.h>
  36 #include <libxml/xmlreader.h>
  37 #include <libxslt/transform.h>
  38 #include <libxslt/xsltutils.h>
  39
  40 #if YAZ_HAVE_EXSLT
  41 #include <libexslt/exslt.h>
  42 #endif
  43
  44 #include <idzebra/util.h>
  45 #include <idzebra/recctrl.h>
  46
  47 /* DOM filter style indexing */
  48 #define ZEBRA_DOM_NS "http://indexdata.com/zebra-2.0"
  49 static const char *zebra_dom_ns = ZEBRA_DOM_NS;
  50
  51 /* DOM filter style indexing */
  52 #define ZEBRA_PI_NAME "zebra-2.0"
  53 static const char *zebra_pi_name = ZEBRA_PI_NAME;
  54
  55
  56
  57 struct convert_s {
  58     const char *stylesheet;
  59     xsltStylesheetPtr stylesheet_xsp;
  60     struct convert_s *next;
  61 };
  62
  63 struct filter_extract {
  64     const char *name;
  65     struct convert_s *convert;
  66 };
  67
  68 struct filter_store {
  69     struct convert_s *convert;
  70 };
  71
  72 struct filter_retrieve {
  73     const char *name;
  74     const char *identifier;
  75     struct convert_s *convert;
  76     struct filter_retrieve *next;
  77 };
  78
  79 #define DOM_INPUT_XMLREADER 1
  80 #define DOM_INPUT_MARC 2
  81 struct filter_input {
  82     const char *syntax;
  83     const char *name;
  84     struct convert_s *convert;
  85     int type;
  86     union {
  87         struct {
  88             const char *input_charset;
  89             yaz_marc_t handle;
  90             yaz_iconv_t iconv;
  91         } marc;
  92         struct {
  93             xmlTextReaderPtr reader;
  94             int split_level;
  95         } xmlreader;
  96     } u;
  97     struct filter_input *next;
  98 };
  99
 100 struct filter_info {
 101     char *fname;
 102     char *full_name;
 103     const char *profile_path;
 104     ODR odr_record;
 105     ODR odr_config;
 106     xmlDocPtr doc_config;
 107     struct filter_extract *extract;
 108     struct filter_retrieve *retrieve_list;
 109     struct filter_input *input_list;
 110     struct filter_store *store;
 111 };
 112
 113
 114
 115 #define XML_STRCMP(a,b)   strcmp((char*)a, b)
 116 #define XML_STRLEN(a) strlen((char*)a)
 117
 118
 119 #define FOR_EACH_ELEMENT(ptr) for (; ptr; ptr = ptr->next) if (ptr->type == XML_ELEMENT_NODE)
 120
 121 static void dom_log(int level, struct filter_info *tinfo, xmlNodePtr ptr,
 122                     const char *fmt, ...)
 123 #ifdef __GNUC__
 124     __attribute__ ((format (printf, 4, 5)))
 125 #endif
 126     ;
 127
 128 static void dom_log(int level, struct filter_info *tinfo, xmlNodePtr ptr,
 129                     const char *fmt, ...)
 130 {
 131     va_list ap;
 132     char buf[4096];
 133
 134     va_start(ap, fmt);
 135     yaz_vsnprintf(buf, sizeof(buf)-1, fmt, ap);
 136     if (ptr)
 137     {
 138         yaz_log(level, "%s:%ld: %s", tinfo->fname ? tinfo->fname : "none",
 139                 xmlGetLineNo(ptr), buf);
 140     }
 141     else
 142     {
 143         yaz_log(level, "%s: %s", tinfo->fname ? tinfo->fname : "none", buf);
 144     }
 145     va_end(ap);
 146 }
 147
 148
 149 static void set_param_str(const char **params, const char *name,
 150                           const char *value, ODR odr)
 151 {
 152     char *quoted = odr_malloc(odr, 3 + strlen(value));
 153     sprintf(quoted, "'%s'", value);
 154     while (*params)
 155         params++;
 156     params[0] = name;
 157     params[1] = quoted;
 158     params[2] = 0;
 159 }
 160
 161 static void set_param_int(const char **params, const char *name,
 162                           zint value, ODR odr)
 163 {
 164     char *quoted = odr_malloc(odr, 30); /* 25 digits enough for 2^64 */
 165     while (*params)
 166         params++;
 167     sprintf(quoted, "'" ZINT_FORMAT "'", value);
 168     params[0] = name;
 169     params[1] = quoted;
 170     params[2] = 0;
 171 }
 172
 173 static void *filter_init(Res res, RecType recType)
 174 {
 175     struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo));
 176     tinfo->fname = 0;
 177     tinfo->full_name = 0;
 178     tinfo->profile_path = 0;
 179     tinfo->odr_record = odr_createmem(ODR_ENCODE);
 180     tinfo->odr_config = odr_createmem(ODR_ENCODE);
 181     tinfo->extract = 0;
 182     tinfo->retrieve_list = 0;
 183     tinfo->input_list = 0;
 184     tinfo->store = 0;
 185     tinfo->doc_config = 0;
 186
 187 #if YAZ_HAVE_EXSLT
 188     exsltRegisterAll();
 189 #endif
 190
 191     return tinfo;
 192 }
 193
 194 static int attr_content(struct _xmlAttr *attr, const char *name,
 195                         const char **dst_content)
 196 {
 197     if (!XML_STRCMP(attr->name, name) && attr->children
 198         && attr->children->type == XML_TEXT_NODE)
 199     {
 200         *dst_content = (const char *)(attr->children->content);
 201         return 1;
 202     }
 203     return 0;
 204 }
 205
 206 static void destroy_xsp(struct convert_s *c)
 207 {
 208     while(c)
 209     {
 210         if (c->stylesheet_xsp)
 211             xsltFreeStylesheet(c->stylesheet_xsp);
 212         c = c->next;
 213     }
 214 }
 215
 216 static void destroy_dom(struct filter_info *tinfo)
 217 {
 218     if (tinfo->extract)
 219     {
 220         destroy_xsp(tinfo->extract->convert);
 221         tinfo->extract = 0;
 222     }
 223     if (tinfo->store)
 224     {
 225         destroy_xsp(tinfo->store->convert);
 226         tinfo->store = 0;
 227     }
 228     if (tinfo->input_list)
 229     {
 230         struct filter_input *i_ptr;
 231         for (i_ptr = tinfo->input_list; i_ptr; i_ptr = i_ptr->next)
 232         {
 233             switch(i_ptr->type)
 234             {
 235             case DOM_INPUT_XMLREADER:
 236                 if (i_ptr->u.xmlreader.reader)
 237                     xmlFreeTextReader(i_ptr->u.xmlreader.reader);
 238                 break;
 239             case DOM_INPUT_MARC:
 240                 yaz_iconv_close(i_ptr->u.marc.iconv);
 241                 yaz_marc_destroy(i_ptr->u.marc.handle);
 242                 break;
 243             }
 244             destroy_xsp(i_ptr->convert);
 245         }
 246         tinfo->input_list = 0;
 247     }
 248     if (tinfo->retrieve_list)
 249     {
 250         struct filter_retrieve *r_ptr;
 251         for (r_ptr = tinfo->retrieve_list; r_ptr; r_ptr = r_ptr->next)
 252             destroy_xsp(r_ptr->convert);
 253         tinfo->retrieve_list = 0;
 254     }
 255
 256     if (tinfo->doc_config)
 257     {
 258         xmlFreeDoc(tinfo->doc_config);
 259         tinfo->doc_config = 0;
 260     }
 261     odr_reset(tinfo->odr_config);
 262 }
 263
 264 static ZEBRA_RES parse_convert(struct filter_info *tinfo, xmlNodePtr ptr,
 265                                struct convert_s **l)
 266 {
 267     *l = 0;
 268     FOR_EACH_ELEMENT(ptr) {
 269         if (!XML_STRCMP(ptr->name, "xslt"))
 270         {
 271             struct _xmlAttr *attr;
 272             struct convert_s *p
 273                 = odr_malloc(tinfo->odr_config, sizeof(*p));
 274
 275             p->next = 0;
 276             p->stylesheet = 0;
 277             p->stylesheet_xsp = 0;
 278
 279             for (attr = ptr->properties; attr; attr = attr->next)
 280                 if (attr_content(attr, "stylesheet", &p->stylesheet))
 281                     ;
 282                 else
 283                 {
 284                     dom_log(YLOG_WARN, tinfo, ptr,
 285                             "bad attribute @%s", attr->name);
 286                 }
 287             if (p->stylesheet)
 288             {
 289                 char tmp_xslt_full_name[1024];
 290                 if (!yaz_filepath_resolve(p->stylesheet,
 291                                           tinfo->profile_path,
 292                                           NULL,
 293                                           tmp_xslt_full_name))
 294                 {
 295                     dom_log(YLOG_WARN, tinfo, 0,
 296                             "stylesheet %s not found in "
 297                             "path %s",
 298                             p->stylesheet,
 299                             tinfo->profile_path);
 300                     return ZEBRA_FAIL;
 301                 }
 302
 303                 p->stylesheet_xsp
 304                     = xsltParseStylesheetFile((const xmlChar*)
 305                                               tmp_xslt_full_name);
 306                 if (!p->stylesheet_xsp)
 307                 {
 308                     dom_log(YLOG_WARN, tinfo, 0,
 309                             "could not parse xslt stylesheet %s",
 310                             tmp_xslt_full_name);
 311                     return ZEBRA_FAIL;
 312                 }
 313                 }
 314                 else
 315                 {
 316                     dom_log(YLOG_WARN, tinfo, ptr,
 317                             "missing attribute 'stylesheet' ");
 318                     return ZEBRA_FAIL;
 319                 }
 320                 *l = p;
 321                 l = &p->next;
 322         }
 323         else
 324         {
 325             dom_log(YLOG_WARN, tinfo, ptr,
 326                     "bad element '%s', expected <xslt>", ptr->name);
 327             return ZEBRA_FAIL;
 328         }
 329     }
 330     return ZEBRA_OK;
 331 }
 332
 333 static ZEBRA_RES perform_convert(struct filter_info *tinfo,
 334                                  struct convert_s *convert,
 335                                  const char **params,
 336                                  xmlDocPtr *doc,
 337                                  xsltStylesheetPtr *last_xsp)
 338 {
 339     for (; convert; convert = convert->next)
 340     {
 341         xmlDocPtr res_doc = xsltApplyStylesheet(convert->stylesheet_xsp,
 342                                                 *doc, params);
 343         if (last_xsp)
 344             *last_xsp = convert->stylesheet_xsp;
 345         xmlFreeDoc(*doc);
 346         *doc = res_doc;
 347     }
 348     return ZEBRA_OK;
 349 }
 350
 351 static struct filter_input *new_input(struct filter_info *tinfo, int type)
 352 {
 353     struct filter_input *p;
 354     struct filter_input **np = &tinfo->input_list;
 355     for (;*np; np = &(*np)->next)
 356         ;
 357     p = *np = odr_malloc(tinfo->odr_config, sizeof(*p));
 358     p->next = 0;
 359     p->syntax = 0;
 360     p->name = 0;
 361     p->convert = 0;
 362     p->type = type;
 363     return p;
 364 }
 365
 366 static ZEBRA_RES parse_input(struct filter_info *tinfo, xmlNodePtr ptr,
 367                              const char *syntax, const char *name)
 368 {
 369     FOR_EACH_ELEMENT(ptr) {
 370         if (!XML_STRCMP(ptr->name, "marc"))
 371         {
 372             yaz_iconv_t iconv = 0;
 373             const char *input_charset = "marc-8";
 374             struct _xmlAttr *attr;
 375
 376             for (attr = ptr->properties; attr; attr = attr->next)
 377             {
 378                 if (attr_content(attr, "inputcharset", &input_charset))
 379                     ;
 380                 else
 381                 {
 382                     dom_log(YLOG_WARN, tinfo, ptr,
 383                             "bad attribute @%s, expected @inputcharset",
 384                             attr->name);
 385                 }
 386             }
 387             iconv = yaz_iconv_open("utf-8", input_charset);
 388             if (!iconv)
 389             {
 390                 dom_log(YLOG_WARN, tinfo, ptr,
 391                         "unsupported @charset '%s'", input_charset);
 392                 return ZEBRA_FAIL;
 393             }
 394             else
 395             {
 396                 struct filter_input *p
 397                     = new_input(tinfo, DOM_INPUT_MARC);
 398                 p->u.marc.handle = yaz_marc_create();
 399                 p->u.marc.iconv = iconv;
 400
 401                 yaz_marc_iconv(p->u.marc.handle, p->u.marc.iconv);
 402
 403                 ptr = ptr->next;
 404
 405                 parse_convert(tinfo, ptr, &p->convert);
 406             }
 407             break;
 408
 409         }
 410         else if (!XML_STRCMP(ptr->name, "xmlreader"))
 411         {
 412             struct filter_input *p
 413                 = new_input(tinfo, DOM_INPUT_XMLREADER);
 414             struct _xmlAttr *attr;
 415             const char *level_str = 0;
 416
 417             p->u.xmlreader.split_level = 0;
 418             p->u.xmlreader.reader = 0;
 419
 420             for (attr = ptr->properties; attr; attr = attr->next)
 421             {
 422                 if (attr_content(attr, "level", &level_str))
 423                     ;
 424                 else
 425                 {
 426                     dom_log(YLOG_WARN, tinfo, ptr,
 427                             "bad attribute @%s, expected @level",
 428                             attr->name);
 429                 }
 430             }
 431             if (level_str)
 432                 p->u.xmlreader.split_level = atoi(level_str);
 433
 434             ptr = ptr->next;
 435
 436             parse_convert(tinfo, ptr, &p->convert);
 437             break;
 438         }
 439         else
 440         {
 441             dom_log(YLOG_WARN, tinfo, ptr,
 442                     "bad element <%s>, expected <marc>|<xmlreader>",
 443                     ptr->name);
 444             return ZEBRA_FAIL;
 445         }
 446     }
 447     return ZEBRA_OK;
 448 }
 449
 450 static ZEBRA_RES parse_dom(struct filter_info *tinfo, const char *fname)
 451 {
 452     char tmp_full_name[1024];
 453     xmlNodePtr ptr;
 454     xmlDocPtr doc;
 455
 456     tinfo->fname = odr_strdup(tinfo->odr_config, fname);
 457
 458     if (yaz_filepath_resolve(tinfo->fname, tinfo->profile_path,
 459                              NULL, tmp_full_name))
 460         tinfo->full_name = odr_strdup(tinfo->odr_config, tmp_full_name);
 461     else
 462         tinfo->full_name = odr_strdup(tinfo->odr_config, tinfo->fname);
 463
 464     yaz_log(YLOG_LOG, "%s dom filter: "
 465             "loading config file %s", tinfo->fname, tinfo->full_name);
 466
 467     doc = xmlParseFile(tinfo->full_name);
 468     if (!doc)
 469     {
 470         yaz_log(YLOG_WARN, "%s: dom filter: "
 471                 "failed to parse config file %s",
 472                 tinfo->fname, tinfo->full_name);
 473         return ZEBRA_FAIL;
 474     }
 475     /* save because we store ptrs to the content */
 476     tinfo->doc_config = doc;
 477
 478     ptr = xmlDocGetRootElement(doc);
 479     if (!ptr || ptr->type != XML_ELEMENT_NODE
 480         || XML_STRCMP(ptr->name, "dom"))
 481     {
 482         dom_log(YLOG_WARN, tinfo, ptr,
 483                 "bad root element <%s>, expected root element <dom>",
 484                 ptr->name);
 485         return ZEBRA_FAIL;
 486     }
 487
 488     ptr = ptr->children;
 489     FOR_EACH_ELEMENT(ptr) {
 490         if (!XML_STRCMP(ptr->name, "extract"))
 491         {
 492             /*
 493               <extract name="index">
 494               <xslt stylesheet="first.xsl"/>
 495               <xslt stylesheet="second.xsl"/>
 496               </extract>
 497             */
 498             struct _xmlAttr *attr;
 499             struct filter_extract *f =
 500                 odr_malloc(tinfo->odr_config, sizeof(*f));
 501
 502             tinfo->extract = f;
 503             f->name = 0;
 504             f->convert = 0;
 505             for (attr = ptr->properties; attr; attr = attr->next)
 506             {
 507                 if (attr_content(attr, "name", &f->name))
 508                     ;
 509                 else
 510                 {
 511                     dom_log(YLOG_WARN, tinfo, ptr,
 512                             "bad attribute @%s, expected @name",
 513                             attr->name);
 514                 }
 515             }
 516             parse_convert(tinfo, ptr->children, &f->convert);
 517         }
 518         else if (!XML_STRCMP(ptr->name, "retrieve"))
 519         {
 520             /*
 521                <retrieve name="F">
 522                <xslt stylesheet="some.xsl"/>
 523                <xslt stylesheet="some.xsl"/>
 524                </retrieve>
 525             */
 526             struct _xmlAttr *attr;
 527             struct filter_retrieve **fp = &tinfo->retrieve_list;
 528             struct filter_retrieve *f =
 529                 odr_malloc(tinfo->odr_config, sizeof(*f));
 530
 531             while (*fp)
 532                 fp = &(*fp)->next;
 533
 534             *fp = f;
 535             f->name = 0;
 536             f->identifier = 0;
 537             f->convert = 0;
 538             f->next = 0;
 539
 540             for (attr = ptr->properties; attr; attr = attr->next)
 541             {
 542                 if (attr_content(attr, "identifier",
 543                                  &f->identifier))
 544                     ;
 545                 else if (attr_content(attr, "name", &f->name))
 546                     ;
 547                 else
 548                 {
 549                     dom_log(YLOG_WARN, tinfo, ptr,
 550                             "bad attribute @%s,  expected @identifier|@name",
 551                             attr->name);
 552                 }
 553             }
 554             parse_convert(tinfo, ptr->children, &f->convert);
 555         }
 556         else if (!XML_STRCMP(ptr->name, "store"))
 557         {
 558             /*
 559               <store name="F">
 560               <xslt stylesheet="some.xsl"/>
 561               <xslt stylesheet="some.xsl"/>
 562               </retrieve>
 563             */
 564             struct filter_store *f =
 565                 odr_malloc(tinfo->odr_config, sizeof(*f));
 566
 567             tinfo->store = f;
 568             f->convert = 0;
 569             parse_convert(tinfo, ptr->children, &f->convert);
 570         }
 571         else if (!XML_STRCMP(ptr->name, "input"))
 572         {
 573             /*
 574               <input syntax="xml">
 575               <xmlreader level="1"/>
 576               </input>
 577               <input syntax="usmarc">
 578               <marc inputcharset="marc-8"/>
 579               </input>
 580             */
 581             struct _xmlAttr *attr;
 582             const char  *syntax = 0;
 583             const char *name = 0;
 584             for (attr = ptr->properties; attr; attr = attr->next)
 585             {
 586                 if (attr_content(attr, "syntax", &syntax))
 587                     ;
 588                 else if (attr_content(attr, "name", &name))
 589                     ;
 590                 else
 591                 {
 592                     dom_log(YLOG_WARN, tinfo, ptr,
 593                             "bad attribute @%s,  expected @syntax|@name",
 594                             attr->name);
 595                 }
 596             }
 597             parse_input(tinfo, ptr->children, syntax, name);
 598         }
 599         else
 600         {
 601             dom_log(YLOG_WARN, tinfo, ptr,
 602                     "bad element <%s>, "
 603                     "expected <extract>|<input>|<retrieve>|<store>",
 604                     ptr->name);
 605             return ZEBRA_FAIL;
 606         }
 607     }
 608     return ZEBRA_OK;
 609 }
 610
 611 static struct filter_retrieve *lookup_retrieve(struct filter_info *tinfo,
 612                                                const char *est)
 613 {
 614     struct filter_retrieve *f = tinfo->retrieve_list;
 615
 616     /* return first schema if no est is provided */
 617     if (!est)
 618         return f;
 619     for (; f; f = f->next)
 620     {
 621         /* find requested schema */
 622         if (est)
 623         {
 624             if (f->identifier && !strcmp(f->identifier, est))
 625                 return f;
 626             if (f->name && !strcmp(f->name, est))
 627                 return f;
 628         }
 629     }
 630     return 0;
 631 }
 632
 633 static ZEBRA_RES filter_config(void *clientData, Res res, const char *args)
 634 {
 635     struct filter_info *tinfo = clientData;
 636     if (!args || !*args)
 637     {
 638         yaz_log(YLOG_WARN, "dom filter: need config file");
 639         return ZEBRA_FAIL;
 640     }
 641
 642     if (tinfo->fname && !strcmp(args, tinfo->fname))
 643         return ZEBRA_OK;
 644
 645     tinfo->profile_path = res_get(res, "profilePath");
 646
 647     destroy_dom(tinfo);
 648     return parse_dom(tinfo, args);
 649 }
 650
 651 static void filter_destroy(void *clientData)
 652 {
 653     struct filter_info *tinfo = clientData;
 654     destroy_dom(tinfo);
 655     odr_destroy(tinfo->odr_config);
 656     odr_destroy(tinfo->odr_record);
 657     xfree(tinfo);
 658 }
 659
 660 static int ioread_ex(void *context, char *buffer, int len)
 661 {
 662     struct recExtractCtrl *p = context;
 663     return p->stream->readf(p->stream, buffer, len);
 664 }
 665
 666 static int ioclose_ex(void *context)
 667 {
 668     return 0;
 669 }
 670
 671
 672 /* DOM filter style indexing */
 673 static int attr_content_xml(struct _xmlAttr *attr, const char *name,
 674                             xmlChar **dst_content)
 675 {
 676     if (0 == XML_STRCMP(attr->name, name) && attr->children
 677         && attr->children->type == XML_TEXT_NODE)
 678     {
 679         *dst_content = (attr->children->content);
 680         return 1;
 681     }
 682     return 0;
 683 }
 684
 685
 686 /* DOM filter style indexing */
 687 static void index_value_of(struct filter_info *tinfo,
 688                            struct recExtractCtrl *extctr,
 689                            RecWord* recword,
 690                            xmlNodePtr node,
 691                            xmlChar * index_p)
 692 {
 693     xmlChar *text = xmlNodeGetContent(node);
 694     size_t text_len = strlen((const char *)text);
 695
 696
 697     /* if there is no text, we do not need to proceed */
 698     if (text_len)
 699     {
 700         xmlChar *look = index_p;
 701         xmlChar *bval;
 702         xmlChar *eval;
 703
 704         xmlChar index[256];
 705         xmlChar type[256];
 706
 707         /* assingning text to be indexed */
 708         recword->term_buf = (const char *)text;
 709         recword->term_len = text_len;
 710
 711         /* parsing all index name/type pairs */
 712         /* may not start with ' ' or ':' */
 713         while (*look && ' ' != *look && ':' != *look)
 714         {
 715             /* setting name and type to zero */
 716             *index = '\0';
 717             *type = '\0';
 718
 719             /* parsing one index name */
 720             bval = look;
 721             while (*look && ':' != *look && ' ' != *look)
 722             {
 723                 look++;
 724             }
 725             eval = look;
 726             strncpy((char *)index, (const char *)bval, eval - bval);
 727             index[eval - bval] = '\0';
 728
 729
 730             /* parsing one index type, if existing */
 731             if (':' == *look)
 732             {
 733                 look++;
 734
 735                 bval = look;
 736                 while (*look && ' ' != *look)
 737                 {
 738                     look++;
 739                 }
 740                 eval = look;
 741                 strncpy((char *)type, (const char *)bval, eval - bval);
 742                 type[eval - bval] = '\0';
 743             }
 744
 745             /* actually indexing the text given */
 746             dom_log(YLOG_DEBUG, tinfo, 0,
 747                     "INDEX '%s:%s' '%s'",
 748                     index, type, text);
 749
 750             recword->index_name = (const char *)index;
 751             if (type && *type)
 752                 recword->index_type = *type;
 753             (extctr->tokenAdd)(recword);
 754
 755             /* eat whitespaces */
 756             if (*look && ' ' == *look && *(look+1))
 757             {
 758                 look++;
 759             }
 760         }
 761     }
 762
 763     xmlFree(text);
 764 }
 765
 766
 767 /* DOM filter style indexing */
 768 static void set_record_info(struct filter_info *tinfo,
 769                             struct recExtractCtrl *extctr,
 770                             xmlChar * id_p,
 771                             xmlChar * rank_p,
 772                             xmlChar * type_p)
 773 {
 774     dom_log(YLOG_DEBUG, tinfo, 0,
 775             "RECORD id=%s rank=%s type=%s",
 776             id_p, rank_p, type_p);
 777
 778     if (id_p)
 779         sscanf((const char *)id_p, "%255s", extctr->match_criteria);
 780
 781     if (rank_p)
 782         extctr->staticrank = atozint((const char *)rank_p);
 783
 784     /*     if (!strcmp("update", type_str)) */
 785     /*         index_node(tinfo, ctrl, ptr, recword); */
 786     /*     else if (!strcmp("delete", type_str)) */
 787     /*         dom_log(YLOG_WARN, tinfo, ptr, "dom filter delete: to be implemented"); */
 788     /*     else */
 789     /*         dom_log(YLOG_WARN, tinfo, ptr, "dom filter: unknown record type '%s'",  */
 790     /*                 type_str); */
 791
 792 }
 793
 794
 795 /* DOM filter style indexing */
 796 static void process_xml_element_zebra_node(struct filter_info *tinfo,
 797                                            struct recExtractCtrl *extctr,
 798                                            RecWord* recword,
 799                                            xmlNodePtr node)
 800 {
 801     if (node->type == XML_ELEMENT_NODE && node->ns && node->ns->href
 802         && 0 == XML_STRCMP(node->ns->href, zebra_dom_ns))
 803     {
 804          if (0 == XML_STRCMP(node->name, "index"))
 805          {
 806             xmlChar *index_p = 0;
 807
 808             struct _xmlAttr *attr;
 809             for (attr = node->properties; attr; attr = attr->next)
 810             {
 811                 if (attr_content_xml(attr, "name", &index_p))
 812                 {
 813                     index_value_of(tinfo, extctr, recword,node, index_p);
 814                 }
 815                 else
 816                 {
 817                     dom_log(YLOG_WARN, tinfo, node,
 818                             "bad attribute @%s, expected @name",
 819                             attr->name);
 820                 }
 821             }
 822         }
 823         else if (0 == XML_STRCMP(node->name, "record"))
 824         {
 825             xmlChar *id_p = 0;
 826             xmlChar *rank_p = 0;
 827             xmlChar *type_p = 0;
 828
 829             struct _xmlAttr *attr;
 830             for (attr = node->properties; attr; attr = attr->next)
 831             {
 832                 if (attr_content_xml(attr, "id", &id_p))
 833                     ;
 834                 else if (attr_content_xml(attr, "rank", &rank_p))
 835                     ;
 836                 else if (attr_content_xml(attr, "type", &type_p))
 837                     ;
 838                 else
 839                 {
 840                     dom_log(YLOG_WARN, tinfo, node,
 841                             "bad attribute @%s, expected @id|@rank|@type",
 842                             attr->name);
 843                 }
 844
 845                 if (type_p && 0 != strcmp("update", (const char *)type_p))
 846                 {
 847                     dom_log(YLOG_WARN, tinfo, node,
 848                             "attribute @%s, only implemented '@type='update'",
 849                             attr->name);
 850                 }
 851             }
 852             set_record_info(tinfo, extctr, id_p, rank_p, type_p);
 853         }
 854         else
 855         {
 856             dom_log(YLOG_WARN, tinfo, node,
 857                     "bad element <%s>,"
 858                     " expected <record>|<index> in namespace '%s'",
 859                     node->name, zebra_dom_ns);
 860         }
 861     }
 862 }
 863
 864
 865 /* DOM filter style indexing */
 866 static void process_xml_pi_node(struct filter_info *tinfo,
 867                                 struct recExtractCtrl *extctr,
 868                                 xmlNodePtr node,
 869                                 xmlChar **index_pp)
 870 {
 871     /* if right PI name, continue parsing PI */
 872     if (0 == strcmp(zebra_pi_name, (const char *)node->name))
 873     {
 874         xmlChar *pi_p =  node->content;
 875         xmlChar *look = pi_p;
 876
 877         xmlChar *bval;
 878         xmlChar *eval;
 879
 880         /* parsing PI record instructions */
 881         if (0 == strncmp((const char *)look, "record", 6))
 882         {
 883             xmlChar id[256];
 884             xmlChar rank[256];
 885             xmlChar type[256];
 886
 887             *id = '\0';
 888             *rank = '\0';
 889             *type = '\0';
 890
 891             look += 6;
 892
 893             /* eat whitespace */
 894             while (*look && ' ' == *look && *(look+1))
 895                 look++;
 896
 897             /* parse possible id */
 898             if (*look && 0 == strncmp((const char *)look, "id=", 3))
 899             {
 900                 look += 3;
 901                 bval = look;
 902                 while (*look && ' ' != *look)
 903                     look++;
 904                 eval = look;
 905                 strncpy((char *)id, (const char *)bval, eval - bval);
 906                 id[eval - bval] = '\0';
 907             }
 908
 909             /* eat whitespace */
 910             while (*look && ' ' == *look && *(look+1))
 911                 look++;
 912
 913             /* parse possible rank */
 914             if (*look && 0 == strncmp((const char *)look, "rank=", 5))
 915             {
 916                 look += 6;
 917                 bval = look;
 918                 while (*look && ' ' != *look)
 919                     look++;
 920                 eval = look;
 921                 strncpy((char *)rank, (const char *)bval, eval - bval);
 922                 rank[eval - bval] = '\0';
 923             }
 924
 925             /* eat whitespace */
 926             while (*look && ' ' == *look && *(look+1))
 927                 look++;
 928
 929             if (look && '\0' != *look)
 930             {
 931                 dom_log(YLOG_WARN, tinfo, node,
 932                         "content '%s', can not parse '%s'",
 933                         pi_p, look);
 934             }
 935             else
 936                 set_record_info(tinfo, extctr, id, rank, 0);
 937
 938         }
 939         /* parsing index instruction */
 940         else if (0 == strncmp((const char *)look, "index", 5))
 941         {
 942             look += 5;
 943
 944             /* eat whitespace */
 945             while (*look && ' ' == *look && *(look+1))
 946                 look++;
 947
 948             /* export index instructions to outside */
 949             *index_pp = look;
 950         }
 951         else
 952         {
 953             dom_log(YLOG_WARN, tinfo, node,
 954                     "content '%s', can not parse '%s'",
 955                     pi_p, look);
 956         }
 957     }
 958 }
 959
 960 /* DOM filter style indexing */
 961 static void process_xml_element_node(struct filter_info *tinfo,
 962                                      struct recExtractCtrl *extctr,
 963                                      RecWord* recword,
 964                                      xmlNodePtr node)
 965 {
 966     /* remember indexing instruction from PI to next element node */
 967     xmlChar *index_p = 0;
 968
 969     /* check if we are an element node in the special zebra namespace
 970        and either set record data or index value-of node content*/
 971     process_xml_element_zebra_node(tinfo, extctr, recword, node);
 972
 973     /* loop through kid nodes */
 974     for (node = node->children; node; node = node->next)
 975     {
 976         /* check and set PI record and index index instructions */
 977         if (node->type == XML_PI_NODE)
 978         {
 979             process_xml_pi_node(tinfo, extctr, node, &index_p);
 980         }
 981         else if (node->type == XML_ELEMENT_NODE)
 982         {
 983             /* if there was a PI index instruction before this element */
 984             if (index_p)
 985             {
 986                 index_value_of(tinfo, extctr, recword, node, index_p);
 987                 index_p = 0;
 988             }
 989             process_xml_element_node(tinfo, extctr, recword,node);
 990         }
 991         else
 992             continue;
 993     }
 994 }
 995
 996
 997 /* DOM filter style indexing */
 998 static void extract_dom_doc_node(struct filter_info *tinfo,
 999                                  struct recExtractCtrl *extctr,
1000                                  xmlDocPtr doc)
1001 {
1002     xmlChar *buf_out;
1003     int len_out;
1004
1005     /* only need to do the initialization once, reuse recword for all terms */
1006     RecWord recword;
1007     (*extctr->init)(extctr, &recword);
1008
1009     if (extctr->flagShowRecords)
1010     {
1011         xmlDocDumpMemory(doc, &buf_out, &len_out);
1012         fwrite(buf_out, len_out, 1, stdout);
1013         xmlFree(buf_out);
1014     }
1015
1016     process_xml_element_node(tinfo, extctr, &recword, (xmlNodePtr)doc);
1017 }
1018
1019
1020
1021
1022 static int convert_extract_doc(struct filter_info *tinfo,
1023                                struct filter_input *input,
1024                                struct recExtractCtrl *p,
1025                                xmlDocPtr doc)
1026
1027 {
1028     xmlChar *buf_out;
1029     int len_out;
1030     const char *params[10];
1031     xsltStylesheetPtr last_xsp = 0;
1032     xmlDocPtr store_doc = 0;
1033
1034     params[0] = 0;
1035     set_param_str(params, "schema", zebra_dom_ns, tinfo->odr_record);
1036
1037     /* input conversion */
1038     perform_convert(tinfo, input->convert, params, &doc, 0);
1039
1040     if (tinfo->store)
1041     {
1042         /* store conversion */
1043         store_doc = xmlCopyDoc(doc, 1);
1044         perform_convert(tinfo, tinfo->store->convert,
1045                         params, &store_doc, &last_xsp);
1046     }
1047
1048     if (last_xsp)
1049         xsltSaveResultToString(&buf_out, &len_out,
1050                                store_doc ? store_doc : doc, last_xsp);
1051     else
1052         xmlDocDumpMemory(store_doc ? store_doc : doc, &buf_out, &len_out);
1053     if (p->flagShowRecords)
1054         fwrite(buf_out, len_out, 1, stdout);
1055     (*p->setStoreData)(p, buf_out, len_out);
1056     xmlFree(buf_out);
1057
1058     if (store_doc)
1059         xmlFreeDoc(store_doc);
1060
1061     /* extract conversion */
1062     perform_convert(tinfo, tinfo->extract->convert, params, &doc, 0);
1063
1064     /* finally, do the indexing */
1065     if (doc)
1066     {
1067         extract_dom_doc_node(tinfo, p, doc);
1068         /* extract_doc_alvis(tinfo, p, doc); */
1069         xmlFreeDoc(doc);
1070     }
1071
1072     return RECCTRL_EXTRACT_OK;
1073 }
1074
1075 static int extract_xml_split(struct filter_info *tinfo,
1076                              struct filter_input *input,
1077                              struct recExtractCtrl *p)
1078 {
1079     int ret;
1080
1081     if (p->first_record)
1082     {
1083         if (input->u.xmlreader.reader)
1084             xmlFreeTextReader(input->u.xmlreader.reader);
1085         input->u.xmlreader.reader = xmlReaderForIO(ioread_ex, ioclose_ex,
1086                                                    p /* I/O handler */,
1087                                                    0 /* URL */,
1088                                                    0 /* encoding */,
1089                                                    XML_PARSE_XINCLUDE|
1090                                                    XML_PARSE_NOENT);
1091     }
1092     if (!input->u.xmlreader.reader)
1093         return RECCTRL_EXTRACT_ERROR_GENERIC;
1094
1095     ret = xmlTextReaderRead(input->u.xmlreader.reader);
1096     while (ret == 1)
1097     {
1098         int type = xmlTextReaderNodeType(input->u.xmlreader.reader);
1099         int depth = xmlTextReaderDepth(input->u.xmlreader.reader);
1100         if (type == XML_READER_TYPE_ELEMENT &&
1101             input->u.xmlreader.split_level == depth)
1102         {
1103             xmlNodePtr ptr
1104                 = xmlTextReaderExpand(input->u.xmlreader.reader);
1105             if (ptr)
1106             {
1107                 xmlNodePtr ptr2 = xmlCopyNode(ptr, 1);
1108                 xmlDocPtr doc = xmlNewDoc((const xmlChar*) "1.0");
1109
1110                 xmlDocSetRootElement(doc, ptr2);
1111
1112                 return convert_extract_doc(tinfo, input, p, doc);
1113             }
1114             else
1115             {
1116                 xmlFreeTextReader(input->u.xmlreader.reader);
1117                 input->u.xmlreader.reader = 0;
1118                 return RECCTRL_EXTRACT_ERROR_GENERIC;
1119             }
1120         }
1121         ret = xmlTextReaderRead(input->u.xmlreader.reader);
1122     }
1123     xmlFreeTextReader(input->u.xmlreader.reader);
1124     input->u.xmlreader.reader = 0;
1125     return RECCTRL_EXTRACT_EOF;
1126 }
1127
1128 static int extract_xml_full(struct filter_info *tinfo,
1129                             struct filter_input *input,
1130                             struct recExtractCtrl *p)
1131 {
1132     if (p->first_record) /* only one record per stream */
1133     {
1134         xmlDocPtr doc = xmlReadIO(ioread_ex, ioclose_ex,
1135                                   p /* I/O handler */,
1136                                   0 /* URL */,
1137                                   0 /* encoding */,
1138                                   XML_PARSE_XINCLUDE|XML_PARSE_NOENT);
1139         if (!doc)
1140         {
1141             return RECCTRL_EXTRACT_ERROR_GENERIC;
1142         }
1143         return convert_extract_doc(tinfo, input, p, doc);
1144     }
1145     else
1146         return RECCTRL_EXTRACT_EOF;
1147 }
1148
1149 static int extract_iso2709(struct filter_info *tinfo,
1150                            struct filter_input *input,
1151                            struct recExtractCtrl *p)
1152 {
1153     char buf[100000];
1154     int record_length;
1155     int read_bytes, r;
1156
1157     if (p->stream->readf(p->stream, buf, 5) != 5)
1158         return RECCTRL_EXTRACT_EOF;
1159     while (*buf < '0' || *buf > '9')
1160     {
1161         int i;
1162
1163         dom_log(YLOG_WARN, tinfo, 0,
1164                 "MARC: Skipping bad byte %d (0x%02X)",
1165                 *buf & 0xff, *buf & 0xff);
1166         for (i = 0; i<4; i++)
1167             buf[i] = buf[i+1];
1168
1169         if (p->stream->readf(p->stream, buf+4, 1) != 1)
1170             return RECCTRL_EXTRACT_EOF;
1171     }
1172     record_length = atoi_n (buf, 5);
1173     if (record_length < 25)
1174     {
1175         dom_log(YLOG_WARN, tinfo, 0,
1176                 "MARC record length < 25, is %d",  record_length);
1177         return RECCTRL_EXTRACT_ERROR_GENERIC;
1178     }
1179     read_bytes = p->stream->readf(p->stream, buf+5, record_length-5);
1180     if (read_bytes < record_length-5)
1181     {
1182         dom_log(YLOG_WARN, tinfo, 0,
1183                 "couldn't read whole MARC record");
1184         return RECCTRL_EXTRACT_ERROR_GENERIC;
1185     }
1186     r = yaz_marc_read_iso2709(input->u.marc.handle,  buf, record_length);
1187     if (r < record_length)
1188     {
1189         dom_log (YLOG_WARN, tinfo, 0,
1190                  "parsing of MARC record failed r=%d length=%d",
1191                  r, record_length);
1192         return RECCTRL_EXTRACT_ERROR_GENERIC;
1193     }
1194     else
1195     {
1196         xmlDocPtr rdoc;
1197         xmlNode *root_ptr;
1198         yaz_marc_write_xml(input->u.marc.handle, &root_ptr, 0, 0, 0);
1199         rdoc = xmlNewDoc((const xmlChar*) "1.0");
1200         xmlDocSetRootElement(rdoc, root_ptr);
1201         return convert_extract_doc(tinfo, input, p, rdoc);
1202     }
1203     return RECCTRL_EXTRACT_OK;
1204 }
1205
1206 static int filter_extract(void *clientData, struct recExtractCtrl *p)
1207 {
1208     struct filter_info *tinfo = clientData;
1209     struct filter_input *input = tinfo->input_list;
1210
1211     if (!input)
1212         return RECCTRL_EXTRACT_ERROR_GENERIC;
1213
1214     odr_reset(tinfo->odr_record);
1215     switch(input->type)
1216     {
1217     case DOM_INPUT_XMLREADER:
1218         if (input->u.xmlreader.split_level == 0)
1219             return extract_xml_full(tinfo, input, p);
1220         else
1221             return extract_xml_split(tinfo, input, p);
1222         break;
1223     case DOM_INPUT_MARC:
1224         return extract_iso2709(tinfo, input, p);
1225     }
1226     return RECCTRL_EXTRACT_ERROR_GENERIC;
1227 }
1228
1229 static int ioread_ret(void *context, char *buffer, int len)
1230 {
1231     struct recRetrieveCtrl *p = context;
1232     return p->stream->readf(p->stream, buffer, len);
1233 }
1234
1235 static int ioclose_ret(void *context)
1236 {
1237     return 0;
1238 }
1239
1240 static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p)
1241 {
1242     /* const char *esn = zebra_dom_ns; */
1243     const char *esn = 0;
1244     const char *params[32];
1245     struct filter_info *tinfo = clientData;
1246     xmlDocPtr doc;
1247     struct filter_retrieve *retrieve;
1248     xsltStylesheetPtr last_xsp = 0;
1249
1250     if (p->comp)
1251     {
1252         if (p->comp->which == Z_RecordComp_simple
1253             && p->comp->u.simple->which == Z_ElementSetNames_generic)
1254         {
1255             esn = p->comp->u.simple->u.generic;
1256         }
1257         else if (p->comp->which == Z_RecordComp_complex
1258                  && p->comp->u.complex->generic->elementSpec
1259                  && p->comp->u.complex->generic->elementSpec->which ==
1260                  Z_ElementSpec_elementSetName)
1261         {
1262             esn = p->comp->u.complex->generic->elementSpec->u.elementSetName;
1263         }
1264     }
1265     retrieve = lookup_retrieve(tinfo, esn);
1266     if (!retrieve)
1267     {
1268         p->diagnostic =
1269             YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
1270         return 0;
1271     }
1272
1273     params[0] = 0;
1274     set_param_int(params, "id", p->localno, p->odr);
1275     if (p->fname)
1276         set_param_str(params, "filename", p->fname, p->odr);
1277     if (p->staticrank >= 0)
1278         set_param_int(params, "rank", p->staticrank, p->odr);
1279
1280     if (esn)
1281         set_param_str(params, "schema", esn, p->odr);
1282     else
1283         if (retrieve->name)
1284             set_param_str(params, "schema", retrieve->name, p->odr);
1285         else if (retrieve->identifier)
1286             set_param_str(params, "schema", retrieve->identifier, p->odr);
1287         else
1288             set_param_str(params, "schema", "", p->odr);
1289
1290     if (p->score >= 0)
1291         set_param_int(params, "score", p->score, p->odr);
1292     set_param_int(params, "size", p->recordSize, p->odr);
1293
1294     doc = xmlReadIO(ioread_ret, ioclose_ret, p /* I/O handler */,
1295                     0 /* URL */,
1296                     0 /* encoding */,
1297                     XML_PARSE_XINCLUDE|XML_PARSE_NOENT);
1298     if (!doc)
1299     {
1300         p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1301         return 0;
1302     }
1303
1304     /* retrieve conversion */
1305     perform_convert(tinfo, retrieve->convert, params, &doc, &last_xsp);
1306     if (!doc)
1307     {
1308         p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1309     }
1310     else if (p->input_format == VAL_NONE || p->input_format == VAL_TEXT_XML)
1311     {
1312         xmlChar *buf_out;
1313         int len_out;
1314
1315         if (last_xsp)
1316             xsltSaveResultToString(&buf_out, &len_out, doc, last_xsp);
1317         else
1318             xmlDocDumpMemory(doc, &buf_out, &len_out);
1319
1320         p->output_format = VAL_TEXT_XML;
1321         p->rec_len = len_out;
1322         p->rec_buf = odr_malloc(p->odr, p->rec_len);
1323         memcpy(p->rec_buf, buf_out, p->rec_len);
1324         xmlFree(buf_out);
1325     }
1326     else if (p->output_format == VAL_SUTRS)
1327     {
1328         xmlChar *buf_out;
1329         int len_out;
1330
1331         if (last_xsp)
1332             xsltSaveResultToString(&buf_out, &len_out, doc, last_xsp);
1333         else
1334             xmlDocDumpMemory(doc, &buf_out, &len_out);
1335
1336         p->output_format = VAL_SUTRS;
1337         p->rec_len = len_out;
1338         p->rec_buf = odr_malloc(p->odr, p->rec_len);
1339         memcpy(p->rec_buf, buf_out, p->rec_len);
1340
1341         xmlFree(buf_out);
1342     }
1343     else
1344     {
1345         p->diagnostic = YAZ_BIB1_RECORD_SYNTAX_UNSUPP;
1346     }
1347     xmlFreeDoc(doc);
1348     return 0;
1349 }
1350
1351 static struct recType filter_type = {
1352     0,
1353     "dom",
1354     filter_init,
1355     filter_config,
1356     filter_destroy,
1357     filter_extract,
1358     filter_retrieve
1359 };
1360
1361 RecType
1362 #ifdef IDZEBRA_STATIC_DOM
1363 idzebra_filter_dom
1364 #else
1365 idzebra_filter
1366 #endif
1367
1368 [] = {
1369     &filter_type,
1370     0,
1371 };
1372 /*
1373  * Local variables:
1374  * c-basic-offset: 4
1375  * indent-tabs-mode: nil
1376  * End:
1377  * vim: shiftwidth=4 tabstop=8 expandtab
1378  */
1379