index/mod_dom.c

   1 /* $Id: mod_dom.c,v 1.19 2007-02-23 11:35:08 adam Exp $
   2    Copyright (C) 1995-2007
   3    Index Data ApS
   4
   5    This file is part of the Zebra server.
   6
   7    Zebra is free software; you can redistribute it and/or modify it under
   8    the terms of the GNU General Public License as published by the Free
   9    Software Foundation; either version 2, or (at your option) any later
  10    version.
  11
  12    Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15    for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, write to the Free Software
  19    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #include <ctype.h>
  26 #include <stdarg.h>
  27
  28 #include <yaz/diagbib1.h>
  29 #include <yaz/tpath.h>
  30 #include <yaz/snprintf.h>
  31
  32 #include <libxml/xmlversion.h>
  33 #include <libxml/parser.h>
  34 #include <libxml/tree.h>
  35 #include <libxml/xmlIO.h>
  36 #include <libxml/xmlreader.h>
  37 #include <libxslt/transform.h>
  38 #include <libxslt/xsltutils.h>
  39
  40 #if YAZ_HAVE_EXSLT
  41 #include <libexslt/exslt.h>
  42 #endif
  43
  44 #include <idzebra/util.h>
  45 #include <idzebra/recctrl.h>
  46
  47 /* DOM filter style indexing */
  48 #define ZEBRA_DOM_NS "http://indexdata.com/zebra-2.0"
  49 static const char *zebra_dom_ns = ZEBRA_DOM_NS;
  50
  51 /* DOM filter style indexing */
  52 #define ZEBRA_PI_NAME "zebra-2.0"
  53 static const char *zebra_pi_name = ZEBRA_PI_NAME;
  54
  55
  56
  57 struct convert_s {
  58     const char *stylesheet;
  59     xsltStylesheetPtr stylesheet_xsp;
  60     struct convert_s *next;
  61 };
  62
  63 struct filter_extract {
  64     const char *name;
  65     struct convert_s *convert;
  66 };
  67
  68 struct filter_store {
  69     struct convert_s *convert;
  70 };
  71
  72 struct filter_retrieve {
  73     const char *name;
  74     const char *identifier;
  75     struct convert_s *convert;
  76     struct filter_retrieve *next;
  77 };
  78
  79 #define DOM_INPUT_XMLREADER 1
  80 #define DOM_INPUT_MARC 2
  81 struct filter_input {
  82     const char *syntax;
  83     const char *name;
  84     struct convert_s *convert;
  85     int type;
  86     union {
  87         struct {
  88             const char *input_charset;
  89             yaz_marc_t handle;
  90             yaz_iconv_t iconv;
  91         } marc;
  92         struct {
  93             xmlTextReaderPtr reader;
  94             int split_level;
  95         } xmlreader;
  96     } u;
  97     struct filter_input *next;
  98 };
  99
 100 struct filter_info {
 101     char *fname;
 102     char *full_name;
 103     const char *profile_path;
 104     ODR odr_record;
 105     ODR odr_config;
 106     xmlDocPtr doc_config;
 107     struct filter_extract *extract;
 108     struct filter_retrieve *retrieve_list;
 109     struct filter_input *input_list;
 110     struct filter_store *store;
 111 };
 112
 113
 114
 115 #define XML_STRCMP(a,b)   strcmp((char*)a, b)
 116 #define XML_STRLEN(a) strlen((char*)a)
 117
 118
 119 #define FOR_EACH_ELEMENT(ptr) for (; ptr; ptr = ptr->next) if (ptr->type == XML_ELEMENT_NODE)
 120
 121 static void dom_log(int level, struct filter_info *tinfo, xmlNodePtr ptr,
 122                     const char *fmt, ...)
 123 #ifdef __GNUC__
 124     __attribute__ ((format (printf, 4, 5)))
 125 #endif
 126     ;
 127
 128 static void dom_log(int level, struct filter_info *tinfo, xmlNodePtr ptr,
 129                     const char *fmt, ...)
 130 {
 131     va_list ap;
 132     char buf[4096];
 133     xmlChar *node_path = 0;
 134
 135     if (ptr)
 136         node_path = xmlGetNodePath(ptr);
 137
 138     va_start(ap, fmt);
 139     yaz_vsnprintf(buf, sizeof(buf)-1, fmt, ap);
 140     yaz_log(level, "%s: dom filter %s%s: %s",
 141             tinfo->fname ? tinfo->fname : "none",
 142             node_path ? "in " : "",
 143             node_path ? (const char *) node_path : "", buf);
 144
 145     if (node_path)
 146         xmlFree(node_path);
 147     va_end(ap);
 148 }
 149
 150
 151 static void set_param_str(const char **params, const char *name,
 152                           const char *value, ODR odr)
 153 {
 154     char *quoted = odr_malloc(odr, 3 + strlen(value));
 155     sprintf(quoted, "'%s'", value);
 156     while (*params)
 157         params++;
 158     params[0] = name;
 159     params[1] = quoted;
 160     params[2] = 0;
 161 }
 162
 163 static void set_param_int(const char **params, const char *name,
 164                           zint value, ODR odr)
 165 {
 166     char *quoted = odr_malloc(odr, 30); /* 25 digits enough for 2^64 */
 167     while (*params)
 168         params++;
 169     sprintf(quoted, "'" ZINT_FORMAT "'", value);
 170     params[0] = name;
 171     params[1] = quoted;
 172     params[2] = 0;
 173 }
 174
 175 static void *filter_init(Res res, RecType recType)
 176 {
 177     struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo));
 178     tinfo->fname = 0;
 179     tinfo->full_name = 0;
 180     tinfo->profile_path = 0;
 181     tinfo->odr_record = odr_createmem(ODR_ENCODE);
 182     tinfo->odr_config = odr_createmem(ODR_ENCODE);
 183     tinfo->extract = 0;
 184     tinfo->retrieve_list = 0;
 185     tinfo->input_list = 0;
 186     tinfo->store = 0;
 187     tinfo->doc_config = 0;
 188
 189 #if YAZ_HAVE_EXSLT
 190     exsltRegisterAll();
 191 #endif
 192
 193     return tinfo;
 194 }
 195
 196 static int attr_content(struct _xmlAttr *attr, const char *name,
 197                         const char **dst_content)
 198 {
 199     if (!XML_STRCMP(attr->name, name) && attr->children
 200         && attr->children->type == XML_TEXT_NODE)
 201     {
 202         *dst_content = (const char *)(attr->children->content);
 203         return 1;
 204     }
 205     return 0;
 206 }
 207
 208 static void destroy_xsp(struct convert_s *c)
 209 {
 210     while(c)
 211     {
 212         if (c->stylesheet_xsp)
 213             xsltFreeStylesheet(c->stylesheet_xsp);
 214         c = c->next;
 215     }
 216 }
 217
 218 static void destroy_dom(struct filter_info *tinfo)
 219 {
 220     if (tinfo->extract)
 221     {
 222         destroy_xsp(tinfo->extract->convert);
 223         tinfo->extract = 0;
 224     }
 225     if (tinfo->store)
 226     {
 227         destroy_xsp(tinfo->store->convert);
 228         tinfo->store = 0;
 229     }
 230     if (tinfo->input_list)
 231     {
 232         struct filter_input *i_ptr;
 233         for (i_ptr = tinfo->input_list; i_ptr; i_ptr = i_ptr->next)
 234         {
 235             switch(i_ptr->type)
 236             {
 237             case DOM_INPUT_XMLREADER:
 238                 if (i_ptr->u.xmlreader.reader)
 239                     xmlFreeTextReader(i_ptr->u.xmlreader.reader);
 240                 break;
 241             case DOM_INPUT_MARC:
 242                 yaz_iconv_close(i_ptr->u.marc.iconv);
 243                 yaz_marc_destroy(i_ptr->u.marc.handle);
 244                 break;
 245             }
 246             destroy_xsp(i_ptr->convert);
 247         }
 248         tinfo->input_list = 0;
 249     }
 250     if (tinfo->retrieve_list)
 251     {
 252         struct filter_retrieve *r_ptr;
 253         for (r_ptr = tinfo->retrieve_list; r_ptr; r_ptr = r_ptr->next)
 254             destroy_xsp(r_ptr->convert);
 255         tinfo->retrieve_list = 0;
 256     }
 257
 258     if (tinfo->doc_config)
 259     {
 260         xmlFreeDoc(tinfo->doc_config);
 261         tinfo->doc_config = 0;
 262     }
 263     odr_reset(tinfo->odr_config);
 264 }
 265
 266 static ZEBRA_RES parse_convert(struct filter_info *tinfo, xmlNodePtr ptr,
 267                                struct convert_s **l)
 268 {
 269     *l = 0;
 270     FOR_EACH_ELEMENT(ptr) {
 271         if (!XML_STRCMP(ptr->name, "xslt"))
 272         {
 273             struct _xmlAttr *attr;
 274             struct convert_s *p
 275                 = odr_malloc(tinfo->odr_config, sizeof(*p));
 276
 277             p->next = 0;
 278             p->stylesheet = 0;
 279             p->stylesheet_xsp = 0;
 280
 281             for (attr = ptr->properties; attr; attr = attr->next)
 282                 if (attr_content(attr, "stylesheet", &p->stylesheet))
 283                     ;
 284                 else
 285                 {
 286                     dom_log(YLOG_WARN, tinfo, ptr,
 287                             "bad attribute @%s", attr->name);
 288                 }
 289             if (p->stylesheet)
 290             {
 291                 char tmp_xslt_full_name[1024];
 292                 if (!yaz_filepath_resolve(p->stylesheet,
 293                                           tinfo->profile_path,
 294                                           NULL,
 295                                           tmp_xslt_full_name))
 296                 {
 297                     dom_log(YLOG_WARN, tinfo, 0,
 298                             "stylesheet %s not found in "
 299                             "path %s",
 300                             p->stylesheet,
 301                             tinfo->profile_path);
 302                     return ZEBRA_FAIL;
 303                 }
 304
 305                 p->stylesheet_xsp
 306                     = xsltParseStylesheetFile((const xmlChar*)
 307                                               tmp_xslt_full_name);
 308                 if (!p->stylesheet_xsp)
 309                 {
 310                     dom_log(YLOG_WARN, tinfo, 0,
 311                             "could not parse xslt stylesheet %s",
 312                             tmp_xslt_full_name);
 313                     return ZEBRA_FAIL;
 314                 }
 315                 }
 316                 else
 317                 {
 318                     dom_log(YLOG_WARN, tinfo, ptr,
 319                             "missing attribute 'stylesheet' ");
 320                     return ZEBRA_FAIL;
 321                 }
 322                 *l = p;
 323                 l = &p->next;
 324         }
 325         else
 326         {
 327             dom_log(YLOG_WARN, tinfo, ptr,
 328                     "bad element '%s', expected <xslt>", ptr->name);
 329             return ZEBRA_FAIL;
 330         }
 331     }
 332     return ZEBRA_OK;
 333 }
 334
 335 static ZEBRA_RES perform_convert(struct filter_info *tinfo,
 336                                  struct convert_s *convert,
 337                                  const char **params,
 338                                  xmlDocPtr *doc,
 339                                  xsltStylesheetPtr *last_xsp)
 340 {
 341     for (; convert; convert = convert->next)
 342     {
 343         xmlDocPtr res_doc = xsltApplyStylesheet(convert->stylesheet_xsp,
 344                                                 *doc, params);
 345         if (last_xsp)
 346             *last_xsp = convert->stylesheet_xsp;
 347         xmlFreeDoc(*doc);
 348         *doc = res_doc;
 349     }
 350     return ZEBRA_OK;
 351 }
 352
 353 static struct filter_input *new_input(struct filter_info *tinfo, int type)
 354 {
 355     struct filter_input *p;
 356     struct filter_input **np = &tinfo->input_list;
 357     for (;*np; np = &(*np)->next)
 358         ;
 359     p = *np = odr_malloc(tinfo->odr_config, sizeof(*p));
 360     p->next = 0;
 361     p->syntax = 0;
 362     p->name = 0;
 363     p->convert = 0;
 364     p->type = type;
 365     return p;
 366 }
 367
 368 static ZEBRA_RES parse_input(struct filter_info *tinfo, xmlNodePtr ptr,
 369                              const char *syntax, const char *name)
 370 {
 371     FOR_EACH_ELEMENT(ptr) {
 372         if (!XML_STRCMP(ptr->name, "marc"))
 373         {
 374             yaz_iconv_t iconv = 0;
 375             const char *input_charset = "marc-8";
 376             struct _xmlAttr *attr;
 377
 378             for (attr = ptr->properties; attr; attr = attr->next)
 379             {
 380                 if (attr_content(attr, "inputcharset", &input_charset))
 381                     ;
 382                 else
 383                 {
 384                     dom_log(YLOG_WARN, tinfo, ptr,
 385                             "bad attribute @%s, expected @inputcharset",
 386                             attr->name);
 387                 }
 388             }
 389             iconv = yaz_iconv_open("utf-8", input_charset);
 390             if (!iconv)
 391             {
 392                 dom_log(YLOG_WARN, tinfo, ptr,
 393                         "unsupported @charset '%s'", input_charset);
 394                 return ZEBRA_FAIL;
 395             }
 396             else
 397             {
 398                 struct filter_input *p
 399                     = new_input(tinfo, DOM_INPUT_MARC);
 400                 p->u.marc.handle = yaz_marc_create();
 401                 p->u.marc.iconv = iconv;
 402
 403                 yaz_marc_iconv(p->u.marc.handle, p->u.marc.iconv);
 404
 405                 ptr = ptr->next;
 406
 407                 parse_convert(tinfo, ptr, &p->convert);
 408             }
 409             break;
 410
 411         }
 412         else if (!XML_STRCMP(ptr->name, "xmlreader"))
 413         {
 414             struct filter_input *p
 415                 = new_input(tinfo, DOM_INPUT_XMLREADER);
 416             struct _xmlAttr *attr;
 417             const char *level_str = 0;
 418
 419             p->u.xmlreader.split_level = 0;
 420             p->u.xmlreader.reader = 0;
 421
 422             for (attr = ptr->properties; attr; attr = attr->next)
 423             {
 424                 if (attr_content(attr, "level", &level_str))
 425                     ;
 426                 else
 427                 {
 428                     dom_log(YLOG_WARN, tinfo, ptr,
 429                             "bad attribute @%s, expected @level",
 430                             attr->name);
 431                 }
 432             }
 433             if (level_str)
 434                 p->u.xmlreader.split_level = atoi(level_str);
 435
 436             ptr = ptr->next;
 437
 438             parse_convert(tinfo, ptr, &p->convert);
 439             break;
 440         }
 441         else
 442         {
 443             dom_log(YLOG_WARN, tinfo, ptr,
 444                     "bad element <%s>, expected <marc>|<xmlreader>",
 445                     ptr->name);
 446             return ZEBRA_FAIL;
 447         }
 448     }
 449     return ZEBRA_OK;
 450 }
 451
 452 static ZEBRA_RES parse_dom(struct filter_info *tinfo, const char *fname)
 453 {
 454     char tmp_full_name[1024];
 455     xmlNodePtr ptr;
 456     xmlDocPtr doc;
 457
 458     tinfo->fname = odr_strdup(tinfo->odr_config, fname);
 459
 460     if (yaz_filepath_resolve(tinfo->fname, tinfo->profile_path,
 461                              NULL, tmp_full_name))
 462         tinfo->full_name = odr_strdup(tinfo->odr_config, tmp_full_name);
 463     else
 464         tinfo->full_name = odr_strdup(tinfo->odr_config, tinfo->fname);
 465
 466     yaz_log(YLOG_LOG, "%s dom filter: "
 467             "loading config file %s", tinfo->fname, tinfo->full_name);
 468
 469     doc = xmlParseFile(tinfo->full_name);
 470     if (!doc)
 471     {
 472         yaz_log(YLOG_WARN, "%s: dom filter: "
 473                 "failed to parse config file %s",
 474                 tinfo->fname, tinfo->full_name);
 475         return ZEBRA_FAIL;
 476     }
 477     /* save because we store ptrs to the content */
 478     tinfo->doc_config = doc;
 479
 480     ptr = xmlDocGetRootElement(doc);
 481     if (!ptr || ptr->type != XML_ELEMENT_NODE
 482         || XML_STRCMP(ptr->name, "dom"))
 483     {
 484         dom_log(YLOG_WARN, tinfo, ptr,
 485                 "bad root element <%s>, expected root element <dom>",
 486                 ptr->name);
 487         return ZEBRA_FAIL;
 488     }
 489
 490     ptr = ptr->children;
 491     FOR_EACH_ELEMENT(ptr) {
 492         if (!XML_STRCMP(ptr->name, "extract"))
 493         {
 494             /*
 495               <extract name="index">
 496               <xslt stylesheet="first.xsl"/>
 497               <xslt stylesheet="second.xsl"/>
 498               </extract>
 499             */
 500             struct _xmlAttr *attr;
 501             struct filter_extract *f =
 502                 odr_malloc(tinfo->odr_config, sizeof(*f));
 503
 504             tinfo->extract = f;
 505             f->name = 0;
 506             f->convert = 0;
 507             for (attr = ptr->properties; attr; attr = attr->next)
 508             {
 509                 if (attr_content(attr, "name", &f->name))
 510                     ;
 511                 else
 512                 {
 513                     dom_log(YLOG_WARN, tinfo, ptr,
 514                             "bad attribute @%s, expected @name",
 515                             attr->name);
 516                 }
 517             }
 518             parse_convert(tinfo, ptr->children, &f->convert);
 519         }
 520         else if (!XML_STRCMP(ptr->name, "retrieve"))
 521         {
 522             /*
 523                <retrieve name="F">
 524                <xslt stylesheet="some.xsl"/>
 525                <xslt stylesheet="some.xsl"/>
 526                </retrieve>
 527             */
 528             struct _xmlAttr *attr;
 529             struct filter_retrieve **fp = &tinfo->retrieve_list;
 530             struct filter_retrieve *f =
 531                 odr_malloc(tinfo->odr_config, sizeof(*f));
 532
 533             while (*fp)
 534                 fp = &(*fp)->next;
 535
 536             *fp = f;
 537             f->name = 0;
 538             f->identifier = 0;
 539             f->convert = 0;
 540             f->next = 0;
 541
 542             for (attr = ptr->properties; attr; attr = attr->next)
 543             {
 544                 if (attr_content(attr, "identifier",
 545                                  &f->identifier))
 546                     ;
 547                 else if (attr_content(attr, "name", &f->name))
 548                     ;
 549                 else
 550                 {
 551                     dom_log(YLOG_WARN, tinfo, ptr,
 552                             "bad attribute @%s,  expected @identifier|@name",
 553                             attr->name);
 554                 }
 555             }
 556             parse_convert(tinfo, ptr->children, &f->convert);
 557         }
 558         else if (!XML_STRCMP(ptr->name, "store"))
 559         {
 560             /*
 561               <store name="F">
 562               <xslt stylesheet="some.xsl"/>
 563               <xslt stylesheet="some.xsl"/>
 564               </retrieve>
 565             */
 566             struct filter_store *f =
 567                 odr_malloc(tinfo->odr_config, sizeof(*f));
 568
 569             tinfo->store = f;
 570             f->convert = 0;
 571             parse_convert(tinfo, ptr->children, &f->convert);
 572         }
 573         else if (!XML_STRCMP(ptr->name, "input"))
 574         {
 575             /*
 576               <input syntax="xml">
 577               <xmlreader level="1"/>
 578               </input>
 579               <input syntax="usmarc">
 580               <marc inputcharset="marc-8"/>
 581               </input>
 582             */
 583             struct _xmlAttr *attr;
 584             const char  *syntax = 0;
 585             const char *name = 0;
 586             for (attr = ptr->properties; attr; attr = attr->next)
 587             {
 588                 if (attr_content(attr, "syntax", &syntax))
 589                     ;
 590                 else if (attr_content(attr, "name", &name))
 591                     ;
 592                 else
 593                 {
 594                     dom_log(YLOG_WARN, tinfo, ptr,
 595                             "bad attribute @%s,  expected @syntax|@name",
 596                             attr->name);
 597                 }
 598             }
 599             parse_input(tinfo, ptr->children, syntax, name);
 600         }
 601         else
 602         {
 603             dom_log(YLOG_WARN, tinfo, ptr,
 604                     "bad element <%s>, "
 605                     "expected <extract>|<input>|<retrieve>|<store>",
 606                     ptr->name);
 607             return ZEBRA_FAIL;
 608         }
 609     }
 610     return ZEBRA_OK;
 611 }
 612
 613 static struct filter_retrieve *lookup_retrieve(struct filter_info *tinfo,
 614                                                const char *est)
 615 {
 616     struct filter_retrieve *f = tinfo->retrieve_list;
 617
 618     /* return first schema if no est is provided */
 619     if (!est)
 620         return f;
 621     for (; f; f = f->next)
 622     {
 623         /* find requested schema */
 624         if (est)
 625         {
 626             if (f->identifier && !strcmp(f->identifier, est))
 627                 return f;
 628             if (f->name && !strcmp(f->name, est))
 629                 return f;
 630         }
 631     }
 632     return 0;
 633 }
 634
 635 static ZEBRA_RES filter_config(void *clientData, Res res, const char *args)
 636 {
 637     struct filter_info *tinfo = clientData;
 638     if (!args || !*args)
 639     {
 640         yaz_log(YLOG_WARN, "dom filter: need config file");
 641         return ZEBRA_FAIL;
 642     }
 643
 644     if (tinfo->fname && !strcmp(args, tinfo->fname))
 645         return ZEBRA_OK;
 646
 647     tinfo->profile_path = res_get(res, "profilePath");
 648
 649     destroy_dom(tinfo);
 650     return parse_dom(tinfo, args);
 651 }
 652
 653 static void filter_destroy(void *clientData)
 654 {
 655     struct filter_info *tinfo = clientData;
 656     destroy_dom(tinfo);
 657     odr_destroy(tinfo->odr_config);
 658     odr_destroy(tinfo->odr_record);
 659     xfree(tinfo);
 660 }
 661
 662 static int ioread_ex(void *context, char *buffer, int len)
 663 {
 664     struct recExtractCtrl *p = context;
 665     return p->stream->readf(p->stream, buffer, len);
 666 }
 667
 668 static int ioclose_ex(void *context)
 669 {
 670     return 0;
 671 }
 672
 673
 674 /* DOM filter style indexing */
 675 static int attr_content_xml(struct _xmlAttr *attr, const char *name,
 676                             xmlChar **dst_content)
 677 {
 678     if (0 == XML_STRCMP(attr->name, name) && attr->children
 679         && attr->children->type == XML_TEXT_NODE)
 680     {
 681         *dst_content = (attr->children->content);
 682         return 1;
 683     }
 684     return 0;
 685 }
 686
 687
 688 /* DOM filter style indexing */
 689 static void index_value_of(struct filter_info *tinfo,
 690                            struct recExtractCtrl *extctr,
 691                            RecWord* recword,
 692                            xmlNodePtr node,
 693                            xmlChar * index_p)
 694 {
 695     xmlChar *text = xmlNodeGetContent(node);
 696     size_t text_len = strlen((const char *)text);
 697
 698
 699     /* if there is no text, we do not need to proceed */
 700     if (text_len)
 701     {
 702         xmlChar *look = index_p;
 703         xmlChar *bval;
 704         xmlChar *eval;
 705
 706         xmlChar index[256];
 707         xmlChar type[256];
 708
 709         /* assingning text to be indexed */
 710         recword->term_buf = (const char *)text;
 711         recword->term_len = text_len;
 712
 713         /* parsing all index name/type pairs */
 714         /* may not start with ' ' or ':' */
 715         while (*look && ' ' != *look && ':' != *look)
 716         {
 717             /* setting name and type to zero */
 718             *index = '\0';
 719             *type = '\0';
 720
 721             /* parsing one index name */
 722             bval = look;
 723             while (*look && ':' != *look && ' ' != *look)
 724             {
 725                 look++;
 726             }
 727             eval = look;
 728             strncpy((char *)index, (const char *)bval, eval - bval);
 729             index[eval - bval] = '\0';
 730
 731
 732             /* parsing one index type, if existing */
 733             if (':' == *look)
 734             {
 735                 look++;
 736
 737                 bval = look;
 738                 while (*look && ' ' != *look)
 739                 {
 740                     look++;
 741                 }
 742                 eval = look;
 743                 strncpy((char *)type, (const char *)bval, eval - bval);
 744                 type[eval - bval] = '\0';
 745             }
 746
 747             /* actually indexing the text given */
 748             dom_log(YLOG_DEBUG, tinfo, 0,
 749                     "INDEX '%s:%s' '%s'",
 750                     index, type, text);
 751
 752             recword->index_name = (const char *)index;
 753             if (type && *type)
 754                 recword->index_type = *type;
 755             (extctr->tokenAdd)(recword);
 756
 757             /* eat whitespaces */
 758             if (*look && ' ' == *look && *(look+1))
 759             {
 760                 look++;
 761             }
 762         }
 763     }
 764
 765     xmlFree(text);
 766 }
 767
 768
 769 /* DOM filter style indexing */
 770 static void set_record_info(struct filter_info *tinfo,
 771                             struct recExtractCtrl *extctr,
 772                             xmlChar * id_p,
 773                             xmlChar * rank_p,
 774                             xmlChar * type_p)
 775 {
 776     dom_log(YLOG_DEBUG, tinfo, 0,
 777             "RECORD id=%s rank=%s type=%s",
 778             id_p, rank_p, type_p);
 779
 780     if (id_p)
 781         sscanf((const char *)id_p, "%255s", extctr->match_criteria);
 782
 783     if (rank_p)
 784         extctr->staticrank = atozint((const char *)rank_p);
 785
 786     /*     if (!strcmp("update", type_str)) */
 787     /*         index_node(tinfo, ctrl, ptr, recword); */
 788     /*     else if (!strcmp("delete", type_str)) */
 789     /*         dom_log(YLOG_WARN, tinfo, ptr, "dom filter delete: to be implemented"); */
 790     /*     else */
 791     /*         dom_log(YLOG_WARN, tinfo, ptr, "dom filter: unknown record type '%s'",  */
 792     /*                 type_str); */
 793
 794 }
 795
 796
 797 /* DOM filter style indexing */
 798 static void process_xml_element_zebra_node(struct filter_info *tinfo,
 799                                            struct recExtractCtrl *extctr,
 800                                            RecWord* recword,
 801                                            xmlNodePtr node)
 802 {
 803     if (node->type == XML_ELEMENT_NODE && node->ns && node->ns->href
 804         && 0 == XML_STRCMP(node->ns->href, zebra_dom_ns))
 805     {
 806          if (0 == XML_STRCMP(node->name, "index"))
 807          {
 808             xmlChar *index_p = 0;
 809
 810             struct _xmlAttr *attr;
 811             for (attr = node->properties; attr; attr = attr->next)
 812             {
 813                 if (attr_content_xml(attr, "name", &index_p))
 814                 {
 815                     index_value_of(tinfo, extctr, recword,node, index_p);
 816                 }
 817                 else
 818                 {
 819                     dom_log(YLOG_WARN, tinfo, node,
 820                             "bad attribute @%s, expected @name",
 821                             attr->name);
 822                 }
 823             }
 824         }
 825         else if (0 == XML_STRCMP(node->name, "record"))
 826         {
 827             xmlChar *id_p = 0;
 828             xmlChar *rank_p = 0;
 829             xmlChar *type_p = 0;
 830
 831             struct _xmlAttr *attr;
 832             for (attr = node->properties; attr; attr = attr->next)
 833             {
 834                 if (attr_content_xml(attr, "id", &id_p))
 835                     ;
 836                 else if (attr_content_xml(attr, "rank", &rank_p))
 837                     ;
 838                 else if (attr_content_xml(attr, "type", &type_p))
 839                     ;
 840                 else
 841                 {
 842                     dom_log(YLOG_WARN, tinfo, node,
 843                             "bad attribute @%s, expected @id|@rank|@type",
 844                             attr->name);
 845                 }
 846
 847                 if (type_p && 0 != strcmp("update", (const char *)type_p))
 848                 {
 849                     dom_log(YLOG_WARN, tinfo, node,
 850                             "attribute @%s, only implemented '@type='update'",
 851                             attr->name);
 852                 }
 853             }
 854             set_record_info(tinfo, extctr, id_p, rank_p, type_p);
 855         }
 856         else
 857         {
 858             dom_log(YLOG_WARN, tinfo, node,
 859                     "bad element <%s>,"
 860                     " expected <record>|<index> in namespace '%s'",
 861                     node->name, zebra_dom_ns);
 862         }
 863     }
 864 }
 865
 866
 867 /* DOM filter style indexing */
 868 static void process_xml_pi_node(struct filter_info *tinfo,
 869                                 struct recExtractCtrl *extctr,
 870                                 xmlNodePtr node,
 871                                 xmlChar **index_pp)
 872 {
 873     /* if right PI name, continue parsing PI */
 874     if (0 == strcmp(zebra_pi_name, (const char *)node->name))
 875     {
 876         xmlChar *pi_p =  node->content;
 877         xmlChar *look = pi_p;
 878
 879         xmlChar *bval;
 880         xmlChar *eval;
 881
 882         /* parsing PI record instructions */
 883         if (0 == strncmp((const char *)look, "record", 6))
 884         {
 885             xmlChar id[256];
 886             xmlChar rank[256];
 887             xmlChar type[256];
 888
 889             *id = '\0';
 890             *rank = '\0';
 891             *type = '\0';
 892
 893             look += 6;
 894
 895             /* eat whitespace */
 896             while (*look && ' ' == *look && *(look+1))
 897                 look++;
 898
 899             /* parse possible id */
 900             if (*look && 0 == strncmp((const char *)look, "id=", 3))
 901             {
 902                 look += 3;
 903                 bval = look;
 904                 while (*look && ' ' != *look)
 905                     look++;
 906                 eval = look;
 907                 strncpy((char *)id, (const char *)bval, eval - bval);
 908                 id[eval - bval] = '\0';
 909             }
 910
 911             /* eat whitespace */
 912             while (*look && ' ' == *look && *(look+1))
 913                 look++;
 914
 915             /* parse possible rank */
 916             if (*look && 0 == strncmp((const char *)look, "rank=", 5))
 917             {
 918                 look += 6;
 919                 bval = look;
 920                 while (*look && ' ' != *look)
 921                     look++;
 922                 eval = look;
 923                 strncpy((char *)rank, (const char *)bval, eval - bval);
 924                 rank[eval - bval] = '\0';
 925             }
 926
 927             /* eat whitespace */
 928             while (*look && ' ' == *look && *(look+1))
 929                 look++;
 930
 931             if (look && '\0' != *look)
 932             {
 933                 dom_log(YLOG_WARN, tinfo, node,
 934                         "content '%s', can not parse '%s'",
 935                         pi_p, look);
 936             }
 937             else
 938                 set_record_info(tinfo, extctr, id, rank, 0);
 939
 940         }
 941         /* parsing index instruction */
 942         else if (0 == strncmp((const char *)look, "index", 5))
 943         {
 944             look += 5;
 945
 946             /* eat whitespace */
 947             while (*look && ' ' == *look && *(look+1))
 948                 look++;
 949
 950             /* export index instructions to outside */
 951             *index_pp = look;
 952         }
 953         else
 954         {
 955             dom_log(YLOG_WARN, tinfo, node,
 956                     "content '%s', can not parse '%s'",
 957                     pi_p, look);
 958         }
 959     }
 960 }
 961
 962 /* DOM filter style indexing */
 963 static void process_xml_element_node(struct filter_info *tinfo,
 964                                      struct recExtractCtrl *extctr,
 965                                      RecWord* recword,
 966                                      xmlNodePtr node)
 967 {
 968     /* remember indexing instruction from PI to next element node */
 969     xmlChar *index_p = 0;
 970
 971     /* check if we are an element node in the special zebra namespace
 972        and either set record data or index value-of node content*/
 973     process_xml_element_zebra_node(tinfo, extctr, recword, node);
 974
 975     /* loop through kid nodes */
 976     for (node = node->children; node; node = node->next)
 977     {
 978         /* check and set PI record and index index instructions */
 979         if (node->type == XML_PI_NODE)
 980         {
 981             process_xml_pi_node(tinfo, extctr, node, &index_p);
 982         }
 983         else if (node->type == XML_ELEMENT_NODE)
 984         {
 985             /* if there was a PI index instruction before this element */
 986             if (index_p)
 987             {
 988                 index_value_of(tinfo, extctr, recword, node, index_p);
 989                 index_p = 0;
 990             }
 991             process_xml_element_node(tinfo, extctr, recword,node);
 992         }
 993         else
 994             continue;
 995     }
 996 }
 997
 998
 999 /* DOM filter style indexing */
1000 static void extract_dom_doc_node(struct filter_info *tinfo,
1001                                  struct recExtractCtrl *extctr,
1002                                  xmlDocPtr doc)
1003 {
1004     xmlChar *buf_out;
1005     int len_out;
1006
1007     /* only need to do the initialization once, reuse recword for all terms */
1008     RecWord recword;
1009     (*extctr->init)(extctr, &recword);
1010
1011     if (extctr->flagShowRecords)
1012     {
1013         xmlDocDumpMemory(doc, &buf_out, &len_out);
1014         fwrite(buf_out, len_out, 1, stdout);
1015         xmlFree(buf_out);
1016     }
1017
1018     process_xml_element_node(tinfo, extctr, &recword, (xmlNodePtr)doc);
1019 }
1020
1021
1022
1023
1024 static int convert_extract_doc(struct filter_info *tinfo,
1025                                struct filter_input *input,
1026                                struct recExtractCtrl *p,
1027                                xmlDocPtr doc)
1028
1029 {
1030     xmlChar *buf_out;
1031     int len_out;
1032     const char *params[10];
1033     xsltStylesheetPtr last_xsp = 0;
1034     xmlDocPtr store_doc = 0;
1035
1036     params[0] = 0;
1037     set_param_str(params, "schema", zebra_dom_ns, tinfo->odr_record);
1038
1039     /* input conversion */
1040     perform_convert(tinfo, input->convert, params, &doc, 0);
1041
1042     if (tinfo->store)
1043     {
1044         /* store conversion */
1045         store_doc = xmlCopyDoc(doc, 1);
1046         perform_convert(tinfo, tinfo->store->convert,
1047                         params, &store_doc, &last_xsp);
1048     }
1049
1050     if (last_xsp)
1051         xsltSaveResultToString(&buf_out, &len_out,
1052                                store_doc ? store_doc : doc, last_xsp);
1053     else
1054         xmlDocDumpMemory(store_doc ? store_doc : doc, &buf_out, &len_out);
1055     if (p->flagShowRecords)
1056         fwrite(buf_out, len_out, 1, stdout);
1057     (*p->setStoreData)(p, buf_out, len_out);
1058     xmlFree(buf_out);
1059
1060     if (store_doc)
1061         xmlFreeDoc(store_doc);
1062
1063     /* extract conversion */
1064     perform_convert(tinfo, tinfo->extract->convert, params, &doc, 0);
1065
1066     /* finally, do the indexing */
1067     if (doc)
1068     {
1069         extract_dom_doc_node(tinfo, p, doc);
1070         /* extract_doc_alvis(tinfo, p, doc); */
1071         xmlFreeDoc(doc);
1072     }
1073
1074     return RECCTRL_EXTRACT_OK;
1075 }
1076
1077 static int extract_xml_split(struct filter_info *tinfo,
1078                              struct filter_input *input,
1079                              struct recExtractCtrl *p)
1080 {
1081     int ret;
1082
1083     if (p->first_record)
1084     {
1085         if (input->u.xmlreader.reader)
1086             xmlFreeTextReader(input->u.xmlreader.reader);
1087         input->u.xmlreader.reader = xmlReaderForIO(ioread_ex, ioclose_ex,
1088                                                    p /* I/O handler */,
1089                                                    0 /* URL */,
1090                                                    0 /* encoding */,
1091                                                    XML_PARSE_XINCLUDE|
1092                                                    XML_PARSE_NOENT);
1093     }
1094     if (!input->u.xmlreader.reader)
1095         return RECCTRL_EXTRACT_ERROR_GENERIC;
1096
1097     ret = xmlTextReaderRead(input->u.xmlreader.reader);
1098     while (ret == 1)
1099     {
1100         int type = xmlTextReaderNodeType(input->u.xmlreader.reader);
1101         int depth = xmlTextReaderDepth(input->u.xmlreader.reader);
1102         if (type == XML_READER_TYPE_ELEMENT &&
1103             input->u.xmlreader.split_level == depth)
1104         {
1105             xmlNodePtr ptr
1106                 = xmlTextReaderExpand(input->u.xmlreader.reader);
1107             if (ptr)
1108             {
1109                 xmlNodePtr ptr2 = xmlCopyNode(ptr, 1);
1110                 xmlDocPtr doc = xmlNewDoc((const xmlChar*) "1.0");
1111
1112                 xmlDocSetRootElement(doc, ptr2);
1113
1114                 return convert_extract_doc(tinfo, input, p, doc);
1115             }
1116             else
1117             {
1118                 xmlFreeTextReader(input->u.xmlreader.reader);
1119                 input->u.xmlreader.reader = 0;
1120                 return RECCTRL_EXTRACT_ERROR_GENERIC;
1121             }
1122         }
1123         ret = xmlTextReaderRead(input->u.xmlreader.reader);
1124     }
1125     xmlFreeTextReader(input->u.xmlreader.reader);
1126     input->u.xmlreader.reader = 0;
1127     return RECCTRL_EXTRACT_EOF;
1128 }
1129
1130 static int extract_xml_full(struct filter_info *tinfo,
1131                             struct filter_input *input,
1132                             struct recExtractCtrl *p)
1133 {
1134     if (p->first_record) /* only one record per stream */
1135     {
1136         xmlDocPtr doc = xmlReadIO(ioread_ex, ioclose_ex,
1137                                   p /* I/O handler */,
1138                                   0 /* URL */,
1139                                   0 /* encoding */,
1140                                   XML_PARSE_XINCLUDE|XML_PARSE_NOENT);
1141         if (!doc)
1142         {
1143             return RECCTRL_EXTRACT_ERROR_GENERIC;
1144         }
1145         return convert_extract_doc(tinfo, input, p, doc);
1146     }
1147     else
1148         return RECCTRL_EXTRACT_EOF;
1149 }
1150
1151 static int extract_iso2709(struct filter_info *tinfo,
1152                            struct filter_input *input,
1153                            struct recExtractCtrl *p)
1154 {
1155     char buf[100000];
1156     int record_length;
1157     int read_bytes, r;
1158
1159     if (p->stream->readf(p->stream, buf, 5) != 5)
1160         return RECCTRL_EXTRACT_EOF;
1161     while (*buf < '0' || *buf > '9')
1162     {
1163         int i;
1164
1165         dom_log(YLOG_WARN, tinfo, 0,
1166                 "MARC: Skipping bad byte %d (0x%02X)",
1167                 *buf & 0xff, *buf & 0xff);
1168         for (i = 0; i<4; i++)
1169             buf[i] = buf[i+1];
1170
1171         if (p->stream->readf(p->stream, buf+4, 1) != 1)
1172             return RECCTRL_EXTRACT_EOF;
1173     }
1174     record_length = atoi_n (buf, 5);
1175     if (record_length < 25)
1176     {
1177         dom_log(YLOG_WARN, tinfo, 0,
1178                 "MARC record length < 25, is %d",  record_length);
1179         return RECCTRL_EXTRACT_ERROR_GENERIC;
1180     }
1181     read_bytes = p->stream->readf(p->stream, buf+5, record_length-5);
1182     if (read_bytes < record_length-5)
1183     {
1184         dom_log(YLOG_WARN, tinfo, 0,
1185                 "couldn't read whole MARC record");
1186         return RECCTRL_EXTRACT_ERROR_GENERIC;
1187     }
1188     r = yaz_marc_read_iso2709(input->u.marc.handle,  buf, record_length);
1189     if (r < record_length)
1190     {
1191         dom_log (YLOG_WARN, tinfo, 0,
1192                  "parsing of MARC record failed r=%d length=%d",
1193                  r, record_length);
1194         return RECCTRL_EXTRACT_ERROR_GENERIC;
1195     }
1196     else
1197     {
1198         xmlDocPtr rdoc;
1199         xmlNode *root_ptr;
1200         yaz_marc_write_xml(input->u.marc.handle, &root_ptr, 0, 0, 0);
1201         rdoc = xmlNewDoc((const xmlChar*) "1.0");
1202         xmlDocSetRootElement(rdoc, root_ptr);
1203         return convert_extract_doc(tinfo, input, p, rdoc);
1204     }
1205     return RECCTRL_EXTRACT_OK;
1206 }
1207
1208 static int filter_extract(void *clientData, struct recExtractCtrl *p)
1209 {
1210     struct filter_info *tinfo = clientData;
1211     struct filter_input *input = tinfo->input_list;
1212
1213     if (!input)
1214         return RECCTRL_EXTRACT_ERROR_GENERIC;
1215
1216     odr_reset(tinfo->odr_record);
1217     switch(input->type)
1218     {
1219     case DOM_INPUT_XMLREADER:
1220         if (input->u.xmlreader.split_level == 0)
1221             return extract_xml_full(tinfo, input, p);
1222         else
1223             return extract_xml_split(tinfo, input, p);
1224         break;
1225     case DOM_INPUT_MARC:
1226         return extract_iso2709(tinfo, input, p);
1227     }
1228     return RECCTRL_EXTRACT_ERROR_GENERIC;
1229 }
1230
1231 static int ioread_ret(void *context, char *buffer, int len)
1232 {
1233     struct recRetrieveCtrl *p = context;
1234     return p->stream->readf(p->stream, buffer, len);
1235 }
1236
1237 static int ioclose_ret(void *context)
1238 {
1239     return 0;
1240 }
1241
1242 static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p)
1243 {
1244     /* const char *esn = zebra_dom_ns; */
1245     const char *esn = 0;
1246     const char *params[32];
1247     struct filter_info *tinfo = clientData;
1248     xmlDocPtr doc;
1249     struct filter_retrieve *retrieve;
1250     xsltStylesheetPtr last_xsp = 0;
1251
1252     if (p->comp)
1253     {
1254         if (p->comp->which == Z_RecordComp_simple
1255             && p->comp->u.simple->which == Z_ElementSetNames_generic)
1256         {
1257             esn = p->comp->u.simple->u.generic;
1258         }
1259         else if (p->comp->which == Z_RecordComp_complex
1260                  && p->comp->u.complex->generic->elementSpec
1261                  && p->comp->u.complex->generic->elementSpec->which ==
1262                  Z_ElementSpec_elementSetName)
1263         {
1264             esn = p->comp->u.complex->generic->elementSpec->u.elementSetName;
1265         }
1266     }
1267     retrieve = lookup_retrieve(tinfo, esn);
1268     if (!retrieve)
1269     {
1270         p->diagnostic =
1271             YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
1272         return 0;
1273     }
1274
1275     params[0] = 0;
1276     set_param_int(params, "id", p->localno, p->odr);
1277     if (p->fname)
1278         set_param_str(params, "filename", p->fname, p->odr);
1279     if (p->staticrank >= 0)
1280         set_param_int(params, "rank", p->staticrank, p->odr);
1281
1282     if (esn)
1283         set_param_str(params, "schema", esn, p->odr);
1284     else
1285         if (retrieve->name)
1286             set_param_str(params, "schema", retrieve->name, p->odr);
1287         else if (retrieve->identifier)
1288             set_param_str(params, "schema", retrieve->identifier, p->odr);
1289         else
1290             set_param_str(params, "schema", "", p->odr);
1291
1292     if (p->score >= 0)
1293         set_param_int(params, "score", p->score, p->odr);
1294     set_param_int(params, "size", p->recordSize, p->odr);
1295
1296     doc = xmlReadIO(ioread_ret, ioclose_ret, p /* I/O handler */,
1297                     0 /* URL */,
1298                     0 /* encoding */,
1299                     XML_PARSE_XINCLUDE|XML_PARSE_NOENT);
1300     if (!doc)
1301     {
1302         p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1303         return 0;
1304     }
1305
1306     /* retrieve conversion */
1307     perform_convert(tinfo, retrieve->convert, params, &doc, &last_xsp);
1308     if (!doc)
1309     {
1310         p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1311     }
1312     else if (p->input_format == VAL_NONE || p->input_format == VAL_TEXT_XML)
1313     {
1314         xmlChar *buf_out;
1315         int len_out;
1316
1317         if (last_xsp)
1318             xsltSaveResultToString(&buf_out, &len_out, doc, last_xsp);
1319         else
1320             xmlDocDumpMemory(doc, &buf_out, &len_out);
1321
1322         p->output_format = VAL_TEXT_XML;
1323         p->rec_len = len_out;
1324         p->rec_buf = odr_malloc(p->odr, p->rec_len);
1325         memcpy(p->rec_buf, buf_out, p->rec_len);
1326         xmlFree(buf_out);
1327     }
1328     else if (p->output_format == VAL_SUTRS)
1329     {
1330         xmlChar *buf_out;
1331         int len_out;
1332
1333         if (last_xsp)
1334             xsltSaveResultToString(&buf_out, &len_out, doc, last_xsp);
1335         else
1336             xmlDocDumpMemory(doc, &buf_out, &len_out);
1337
1338         p->output_format = VAL_SUTRS;
1339         p->rec_len = len_out;
1340         p->rec_buf = odr_malloc(p->odr, p->rec_len);
1341         memcpy(p->rec_buf, buf_out, p->rec_len);
1342
1343         xmlFree(buf_out);
1344     }
1345     else
1346     {
1347         p->diagnostic = YAZ_BIB1_RECORD_SYNTAX_UNSUPP;
1348     }
1349     xmlFreeDoc(doc);
1350     return 0;
1351 }
1352
1353 static struct recType filter_type = {
1354     0,
1355     "dom",
1356     filter_init,
1357     filter_config,
1358     filter_destroy,
1359     filter_extract,
1360     filter_retrieve
1361 };
1362
1363 RecType
1364 #ifdef IDZEBRA_STATIC_DOM
1365 idzebra_filter_dom
1366 #else
1367 idzebra_filter
1368 #endif
1369
1370 [] = {
1371     &filter_type,
1372     0,
1373 };
1374 /*
1375  * Local variables:
1376  * c-basic-offset: 4
1377  * indent-tabs-mode: nil
1378  * End:
1379  * vim: shiftwidth=4 tabstop=8 expandtab
1380  */
1381