index/mod_dom.c

   1 /* $Id: mod_dom.c,v 1.21 2007-02-26 16:12:24 adam Exp $
   2    Copyright (C) 1995-2007
   3    Index Data ApS
   4
   5    This file is part of the Zebra server.
   6
   7    Zebra is free software; you can redistribute it and/or modify it under
   8    the terms of the GNU General Public License as published by the Free
   9    Software Foundation; either version 2, or (at your option) any later
  10    version.
  11
  12    Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15    for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, write to the Free Software
  19    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #include <ctype.h>
  26 #include <stdarg.h>
  27
  28 #include <yaz/diagbib1.h>
  29 #include <yaz/tpath.h>
  30 #include <yaz/snprintf.h>
  31
  32 #include <libxml/xmlversion.h>
  33 #include <libxml/parser.h>
  34 #include <libxml/tree.h>
  35 #include <libxml/xmlIO.h>
  36 #include <libxml/xmlreader.h>
  37 #include <libxslt/transform.h>
  38 #include <libxslt/xsltutils.h>
  39
  40 #if YAZ_HAVE_EXSLT
  41 #include <libexslt/exslt.h>
  42 #endif
  43
  44 #include <idzebra/util.h>
  45 #include <idzebra/recctrl.h>
  46
  47 /* DOM filter style indexing */
  48 #define ZEBRA_DOM_NS "http://indexdata.com/zebra-2.0"
  49 static const char *zebra_dom_ns = ZEBRA_DOM_NS;
  50
  51 /* DOM filter style indexing */
  52 #define ZEBRA_PI_NAME "zebra-2.0"
  53 static const char *zebra_pi_name = ZEBRA_PI_NAME;
  54
  55
  56
  57 struct convert_s {
  58     const char *stylesheet;
  59     xsltStylesheetPtr stylesheet_xsp;
  60     struct convert_s *next;
  61 };
  62
  63 struct filter_extract {
  64     const char *name;
  65     struct convert_s *convert;
  66 };
  67
  68 struct filter_store {
  69     struct convert_s *convert;
  70 };
  71
  72 struct filter_retrieve {
  73     const char *name;
  74     const char *identifier;
  75     struct convert_s *convert;
  76     struct filter_retrieve *next;
  77 };
  78
  79 #define DOM_INPUT_XMLREADER 1
  80 #define DOM_INPUT_MARC 2
  81 struct filter_input {
  82     const char *syntax;
  83     const char *name;
  84     struct convert_s *convert;
  85     int type;
  86     union {
  87         struct {
  88             const char *input_charset;
  89             yaz_marc_t handle;
  90             yaz_iconv_t iconv;
  91         } marc;
  92         struct {
  93             xmlTextReaderPtr reader;
  94             int split_level;
  95         } xmlreader;
  96     } u;
  97     struct filter_input *next;
  98 };
  99
 100 struct filter_info {
 101     char *fname;
 102     char *full_name;
 103     const char *profile_path;
 104     ODR odr_record;
 105     ODR odr_config;
 106     xmlDocPtr doc_config;
 107     struct filter_extract *extract;
 108     struct filter_retrieve *retrieve_list;
 109     struct filter_input *input_list;
 110     struct filter_store *store;
 111 };
 112
 113
 114
 115 #define XML_STRCMP(a,b)   strcmp((char*)a, b)
 116 #define XML_STRLEN(a) strlen((char*)a)
 117
 118
 119 #define FOR_EACH_ELEMENT(ptr) for (; ptr; ptr = ptr->next) if (ptr->type == XML_ELEMENT_NODE)
 120
 121 static void dom_log(int level, struct filter_info *tinfo, xmlNodePtr ptr,
 122                     const char *fmt, ...)
 123 #ifdef __GNUC__
 124     __attribute__ ((format (printf, 4, 5)))
 125 #endif
 126     ;
 127
 128 static void dom_log(int level, struct filter_info *tinfo, xmlNodePtr ptr,
 129                     const char *fmt, ...)
 130 {
 131     va_list ap;
 132     char buf[4096];
 133
 134     va_start(ap, fmt);
 135     yaz_vsnprintf(buf, sizeof(buf)-1, fmt, ap);
 136     if (ptr)
 137     {
 138         yaz_log(level, "%s:%ld: %s", tinfo->fname ? tinfo->fname : "none",
 139                 xmlGetLineNo(ptr), buf);
 140     }
 141     else
 142     {
 143         yaz_log(level, "%s: %s", tinfo->fname ? tinfo->fname : "none", buf);
 144     }
 145     va_end(ap);
 146 }
 147
 148
 149 static void set_param_str(const char **params, const char *name,
 150                           const char *value, ODR odr)
 151 {
 152     char *quoted = odr_malloc(odr, 3 + strlen(value));
 153     sprintf(quoted, "'%s'", value);
 154     while (*params)
 155         params++;
 156     params[0] = name;
 157     params[1] = quoted;
 158     params[2] = 0;
 159 }
 160
 161 static void set_param_int(const char **params, const char *name,
 162                           zint value, ODR odr)
 163 {
 164     char *quoted = odr_malloc(odr, 30); /* 25 digits enough for 2^64 */
 165     while (*params)
 166         params++;
 167     sprintf(quoted, "'" ZINT_FORMAT "'", value);
 168     params[0] = name;
 169     params[1] = quoted;
 170     params[2] = 0;
 171 }
 172
 173 static void *filter_init(Res res, RecType recType)
 174 {
 175     struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo));
 176     tinfo->fname = 0;
 177     tinfo->full_name = 0;
 178     tinfo->profile_path = 0;
 179     tinfo->odr_record = odr_createmem(ODR_ENCODE);
 180     tinfo->odr_config = odr_createmem(ODR_ENCODE);
 181     tinfo->extract = 0;
 182     tinfo->retrieve_list = 0;
 183     tinfo->input_list = 0;
 184     tinfo->store = 0;
 185     tinfo->doc_config = 0;
 186
 187 #if YAZ_HAVE_EXSLT
 188     exsltRegisterAll();
 189 #endif
 190
 191     return tinfo;
 192 }
 193
 194 static int attr_content(struct _xmlAttr *attr, const char *name,
 195                         const char **dst_content)
 196 {
 197     if (!XML_STRCMP(attr->name, name) && attr->children
 198         && attr->children->type == XML_TEXT_NODE)
 199     {
 200         *dst_content = (const char *)(attr->children->content);
 201         return 1;
 202     }
 203     return 0;
 204 }
 205
 206 static void destroy_xsp(struct convert_s *c)
 207 {
 208     while(c)
 209     {
 210         if (c->stylesheet_xsp)
 211             xsltFreeStylesheet(c->stylesheet_xsp);
 212         c = c->next;
 213     }
 214 }
 215
 216 static void destroy_dom(struct filter_info *tinfo)
 217 {
 218     if (tinfo->extract)
 219     {
 220         destroy_xsp(tinfo->extract->convert);
 221         tinfo->extract = 0;
 222     }
 223     if (tinfo->store)
 224     {
 225         destroy_xsp(tinfo->store->convert);
 226         tinfo->store = 0;
 227     }
 228     if (tinfo->input_list)
 229     {
 230         struct filter_input *i_ptr;
 231         for (i_ptr = tinfo->input_list; i_ptr; i_ptr = i_ptr->next)
 232         {
 233             switch(i_ptr->type)
 234             {
 235             case DOM_INPUT_XMLREADER:
 236                 if (i_ptr->u.xmlreader.reader)
 237                     xmlFreeTextReader(i_ptr->u.xmlreader.reader);
 238                 break;
 239             case DOM_INPUT_MARC:
 240                 yaz_iconv_close(i_ptr->u.marc.iconv);
 241                 yaz_marc_destroy(i_ptr->u.marc.handle);
 242                 break;
 243             }
 244             destroy_xsp(i_ptr->convert);
 245         }
 246         tinfo->input_list = 0;
 247     }
 248     if (tinfo->retrieve_list)
 249     {
 250         struct filter_retrieve *r_ptr;
 251         for (r_ptr = tinfo->retrieve_list; r_ptr; r_ptr = r_ptr->next)
 252             destroy_xsp(r_ptr->convert);
 253         tinfo->retrieve_list = 0;
 254     }
 255
 256     if (tinfo->doc_config)
 257     {
 258         xmlFreeDoc(tinfo->doc_config);
 259         tinfo->doc_config = 0;
 260     }
 261     odr_reset(tinfo->odr_config);
 262 }
 263
 264 static ZEBRA_RES parse_convert(struct filter_info *tinfo, xmlNodePtr ptr,
 265                                struct convert_s **l)
 266 {
 267     *l = 0;
 268     FOR_EACH_ELEMENT(ptr) {
 269         if (!XML_STRCMP(ptr->name, "xslt"))
 270         {
 271             struct _xmlAttr *attr;
 272             struct convert_s *p
 273                 = odr_malloc(tinfo->odr_config, sizeof(*p));
 274
 275             p->next = 0;
 276             p->stylesheet = 0;
 277             p->stylesheet_xsp = 0;
 278
 279             for (attr = ptr->properties; attr; attr = attr->next)
 280                 if (attr_content(attr, "stylesheet", &p->stylesheet))
 281                     ;
 282                 else
 283                 {
 284                     dom_log(YLOG_WARN, tinfo, ptr,
 285                             "bad attribute @%s", attr->name);
 286                 }
 287             if (p->stylesheet)
 288             {
 289                 char tmp_xslt_full_name[1024];
 290                 if (!yaz_filepath_resolve(p->stylesheet,
 291                                           tinfo->profile_path,
 292                                           NULL,
 293                                           tmp_xslt_full_name))
 294                 {
 295                     dom_log(YLOG_WARN, tinfo, 0,
 296                             "stylesheet %s not found in "
 297                             "path %s",
 298                             p->stylesheet,
 299                             tinfo->profile_path);
 300                     return ZEBRA_FAIL;
 301                 }
 302
 303                 p->stylesheet_xsp
 304                     = xsltParseStylesheetFile((const xmlChar*)
 305                                               tmp_xslt_full_name);
 306                 if (!p->stylesheet_xsp)
 307                 {
 308                     dom_log(YLOG_WARN, tinfo, 0,
 309                             "could not parse xslt stylesheet %s",
 310                             tmp_xslt_full_name);
 311                     return ZEBRA_FAIL;
 312                 }
 313                 }
 314                 else
 315                 {
 316                     dom_log(YLOG_WARN, tinfo, ptr,
 317                             "missing attribute 'stylesheet' ");
 318                     return ZEBRA_FAIL;
 319                 }
 320                 *l = p;
 321                 l = &p->next;
 322         }
 323         else
 324         {
 325             dom_log(YLOG_WARN, tinfo, ptr,
 326                     "bad element '%s', expected <xslt>", ptr->name);
 327             return ZEBRA_FAIL;
 328         }
 329     }
 330     return ZEBRA_OK;
 331 }
 332
 333 static ZEBRA_RES perform_convert(struct filter_info *tinfo,
 334                                  struct convert_s *convert,
 335                                  const char **params,
 336                                  xmlDocPtr *doc,
 337                                  xsltStylesheetPtr *last_xsp)
 338 {
 339     for (; convert; convert = convert->next)
 340     {
 341         xmlDocPtr res_doc = xsltApplyStylesheet(convert->stylesheet_xsp,
 342                                                 *doc, params);
 343         if (last_xsp)
 344             *last_xsp = convert->stylesheet_xsp;
 345         xmlFreeDoc(*doc);
 346         *doc = res_doc;
 347     }
 348     return ZEBRA_OK;
 349 }
 350
 351 static struct filter_input *new_input(struct filter_info *tinfo, int type)
 352 {
 353     struct filter_input *p;
 354     struct filter_input **np = &tinfo->input_list;
 355     for (;*np; np = &(*np)->next)
 356         ;
 357     p = *np = odr_malloc(tinfo->odr_config, sizeof(*p));
 358     p->next = 0;
 359     p->syntax = 0;
 360     p->name = 0;
 361     p->convert = 0;
 362     p->type = type;
 363     return p;
 364 }
 365
 366 static ZEBRA_RES parse_input(struct filter_info *tinfo, xmlNodePtr ptr,
 367                              const char *syntax, const char *name)
 368 {
 369     FOR_EACH_ELEMENT(ptr) {
 370         if (!XML_STRCMP(ptr->name, "marc"))
 371         {
 372             yaz_iconv_t iconv = 0;
 373             const char *input_charset = "marc-8";
 374             struct _xmlAttr *attr;
 375
 376             for (attr = ptr->properties; attr; attr = attr->next)
 377             {
 378                 if (attr_content(attr, "inputcharset", &input_charset))
 379                     ;
 380                 else
 381                 {
 382                     dom_log(YLOG_WARN, tinfo, ptr,
 383                             "bad attribute @%s, expected @inputcharset",
 384                             attr->name);
 385                 }
 386             }
 387             iconv = yaz_iconv_open("utf-8", input_charset);
 388             if (!iconv)
 389             {
 390                 dom_log(YLOG_WARN, tinfo, ptr,
 391                         "unsupported @charset '%s'", input_charset);
 392                 return ZEBRA_FAIL;
 393             }
 394             else
 395             {
 396                 struct filter_input *p
 397                     = new_input(tinfo, DOM_INPUT_MARC);
 398                 p->u.marc.handle = yaz_marc_create();
 399                 p->u.marc.iconv = iconv;
 400
 401                 yaz_marc_iconv(p->u.marc.handle, p->u.marc.iconv);
 402
 403                 ptr = ptr->next;
 404
 405                 parse_convert(tinfo, ptr, &p->convert);
 406             }
 407             break;
 408
 409         }
 410         else if (!XML_STRCMP(ptr->name, "xmlreader"))
 411         {
 412             struct filter_input *p
 413                 = new_input(tinfo, DOM_INPUT_XMLREADER);
 414             struct _xmlAttr *attr;
 415             const char *level_str = 0;
 416
 417             p->u.xmlreader.split_level = 0;
 418             p->u.xmlreader.reader = 0;
 419
 420             for (attr = ptr->properties; attr; attr = attr->next)
 421             {
 422                 if (attr_content(attr, "level", &level_str))
 423                     ;
 424                 else
 425                 {
 426                     dom_log(YLOG_WARN, tinfo, ptr,
 427                             "bad attribute @%s, expected @level",
 428                             attr->name);
 429                 }
 430             }
 431             if (level_str)
 432                 p->u.xmlreader.split_level = atoi(level_str);
 433
 434             ptr = ptr->next;
 435
 436             parse_convert(tinfo, ptr, &p->convert);
 437             break;
 438         }
 439         else
 440         {
 441             dom_log(YLOG_WARN, tinfo, ptr,
 442                     "bad element <%s>, expected <marc>|<xmlreader>",
 443                     ptr->name);
 444             return ZEBRA_FAIL;
 445         }
 446     }
 447     return ZEBRA_OK;
 448 }
 449
 450 static ZEBRA_RES parse_dom(struct filter_info *tinfo, const char *fname)
 451 {
 452     char tmp_full_name[1024];
 453     xmlNodePtr ptr;
 454     xmlDocPtr doc;
 455
 456     tinfo->fname = odr_strdup(tinfo->odr_config, fname);
 457
 458     if (yaz_filepath_resolve(tinfo->fname, tinfo->profile_path,
 459                              NULL, tmp_full_name))
 460         tinfo->full_name = odr_strdup(tinfo->odr_config, tmp_full_name);
 461     else
 462         tinfo->full_name = odr_strdup(tinfo->odr_config, tinfo->fname);
 463
 464     yaz_log(YLOG_LOG, "%s dom filter: "
 465             "loading config file %s", tinfo->fname, tinfo->full_name);
 466
 467     doc = xmlParseFile(tinfo->full_name);
 468     if (!doc)
 469     {
 470         yaz_log(YLOG_WARN, "%s: dom filter: "
 471                 "failed to parse config file %s",
 472                 tinfo->fname, tinfo->full_name);
 473         return ZEBRA_FAIL;
 474     }
 475     /* save because we store ptrs to the content */
 476     tinfo->doc_config = doc;
 477
 478     ptr = xmlDocGetRootElement(doc);
 479     if (!ptr || ptr->type != XML_ELEMENT_NODE
 480         || XML_STRCMP(ptr->name, "dom"))
 481     {
 482         dom_log(YLOG_WARN, tinfo, ptr,
 483                 "bad root element <%s>, expected root element <dom>",
 484                 ptr->name);
 485         return ZEBRA_FAIL;
 486     }
 487
 488     ptr = ptr->children;
 489     FOR_EACH_ELEMENT(ptr) {
 490         if (!XML_STRCMP(ptr->name, "extract"))
 491         {
 492             /*
 493               <extract name="index">
 494               <xslt stylesheet="first.xsl"/>
 495               <xslt stylesheet="second.xsl"/>
 496               </extract>
 497             */
 498             struct _xmlAttr *attr;
 499             struct filter_extract *f =
 500                 odr_malloc(tinfo->odr_config, sizeof(*f));
 501
 502             tinfo->extract = f;
 503             f->name = 0;
 504             f->convert = 0;
 505             for (attr = ptr->properties; attr; attr = attr->next)
 506             {
 507                 if (attr_content(attr, "name", &f->name))
 508                     ;
 509                 else
 510                 {
 511                     dom_log(YLOG_WARN, tinfo, ptr,
 512                             "bad attribute @%s, expected @name",
 513                             attr->name);
 514                 }
 515             }
 516             parse_convert(tinfo, ptr->children, &f->convert);
 517         }
 518         else if (!XML_STRCMP(ptr->name, "retrieve"))
 519         {
 520             /*
 521                <retrieve name="F">
 522                <xslt stylesheet="some.xsl"/>
 523                <xslt stylesheet="some.xsl"/>
 524                </retrieve>
 525             */
 526             struct _xmlAttr *attr;
 527             struct filter_retrieve **fp = &tinfo->retrieve_list;
 528             struct filter_retrieve *f =
 529                 odr_malloc(tinfo->odr_config, sizeof(*f));
 530
 531             while (*fp)
 532                 fp = &(*fp)->next;
 533
 534             *fp = f;
 535             f->name = 0;
 536             f->identifier = 0;
 537             f->convert = 0;
 538             f->next = 0;
 539
 540             for (attr = ptr->properties; attr; attr = attr->next)
 541             {
 542                 if (attr_content(attr, "identifier",
 543                                  &f->identifier))
 544                     ;
 545                 else if (attr_content(attr, "name", &f->name))
 546                     ;
 547                 else
 548                 {
 549                     dom_log(YLOG_WARN, tinfo, ptr,
 550                             "bad attribute @%s,  expected @identifier|@name",
 551                             attr->name);
 552                 }
 553             }
 554             parse_convert(tinfo, ptr->children, &f->convert);
 555         }
 556         else if (!XML_STRCMP(ptr->name, "store"))
 557         {
 558             /*
 559               <store name="F">
 560               <xslt stylesheet="some.xsl"/>
 561               <xslt stylesheet="some.xsl"/>
 562               </retrieve>
 563             */
 564             struct filter_store *f =
 565                 odr_malloc(tinfo->odr_config, sizeof(*f));
 566
 567             tinfo->store = f;
 568             f->convert = 0;
 569             parse_convert(tinfo, ptr->children, &f->convert);
 570         }
 571         else if (!XML_STRCMP(ptr->name, "input"))
 572         {
 573             /*
 574               <input syntax="xml">
 575               <xmlreader level="1"/>
 576               </input>
 577               <input syntax="usmarc">
 578               <marc inputcharset="marc-8"/>
 579               </input>
 580             */
 581             struct _xmlAttr *attr;
 582             const char  *syntax = 0;
 583             const char *name = 0;
 584             for (attr = ptr->properties; attr; attr = attr->next)
 585             {
 586                 if (attr_content(attr, "syntax", &syntax))
 587                     ;
 588                 else if (attr_content(attr, "name", &name))
 589                     ;
 590                 else
 591                 {
 592                     dom_log(YLOG_WARN, tinfo, ptr,
 593                             "bad attribute @%s,  expected @syntax|@name",
 594                             attr->name);
 595                 }
 596             }
 597             parse_input(tinfo, ptr->children, syntax, name);
 598         }
 599         else
 600         {
 601             dom_log(YLOG_WARN, tinfo, ptr,
 602                     "bad element <%s>, "
 603                     "expected <extract>|<input>|<retrieve>|<store>",
 604                     ptr->name);
 605             return ZEBRA_FAIL;
 606         }
 607     }
 608     return ZEBRA_OK;
 609 }
 610
 611 static struct filter_retrieve *lookup_retrieve(struct filter_info *tinfo,
 612                                                const char *est)
 613 {
 614     struct filter_retrieve *f = tinfo->retrieve_list;
 615
 616     /* return first schema if no est is provided */
 617     if (!est)
 618         return f;
 619     for (; f; f = f->next)
 620     {
 621         /* find requested schema */
 622         if (est)
 623         {
 624             if (f->identifier && !strcmp(f->identifier, est))
 625                 return f;
 626             if (f->name && !strcmp(f->name, est))
 627                 return f;
 628         }
 629     }
 630     return 0;
 631 }
 632
 633 static ZEBRA_RES filter_config(void *clientData, Res res, const char *args)
 634 {
 635     struct filter_info *tinfo = clientData;
 636     if (!args || !*args)
 637     {
 638         yaz_log(YLOG_WARN, "dom filter: need config file");
 639         return ZEBRA_FAIL;
 640     }
 641
 642     if (tinfo->fname && !strcmp(args, tinfo->fname))
 643         return ZEBRA_OK;
 644
 645     tinfo->profile_path = res_get(res, "profilePath");
 646
 647     destroy_dom(tinfo);
 648     return parse_dom(tinfo, args);
 649 }
 650
 651 static void filter_destroy(void *clientData)
 652 {
 653     struct filter_info *tinfo = clientData;
 654     destroy_dom(tinfo);
 655     odr_destroy(tinfo->odr_config);
 656     odr_destroy(tinfo->odr_record);
 657     xfree(tinfo);
 658 }
 659
 660 static int ioread_ex(void *context, char *buffer, int len)
 661 {
 662     struct recExtractCtrl *p = context;
 663     return p->stream->readf(p->stream, buffer, len);
 664 }
 665
 666 static int ioclose_ex(void *context)
 667 {
 668     return 0;
 669 }
 670
 671
 672 /* DOM filter style indexing */
 673 static int attr_content_xml(struct _xmlAttr *attr, const char *name,
 674                             xmlChar **dst_content)
 675 {
 676     if (0 == XML_STRCMP(attr->name, name) && attr->children
 677         && attr->children->type == XML_TEXT_NODE)
 678     {
 679         *dst_content = (attr->children->content);
 680         return 1;
 681     }
 682     return 0;
 683 }
 684
 685
 686 /* DOM filter style indexing */
 687 static void index_value_of(struct filter_info *tinfo,
 688                            struct recExtractCtrl *extctr,
 689                            RecWord* recword,
 690                            xmlNodePtr node,
 691                            xmlChar * index_p)
 692 {
 693     xmlChar *text = xmlNodeGetContent(node);
 694     size_t text_len = strlen((const char *)text);
 695
 696     yaz_log(YLOG_LOG, "Indexing :%.*s:", text_len, text);
 697
 698     /* if there is no text, we do not need to proceed */
 699     if (text_len)
 700     {
 701         xmlChar *look = index_p;
 702         xmlChar *bval;
 703         xmlChar *eval;
 704
 705         xmlChar index[256];
 706         xmlChar type[256];
 707
 708         /* assingning text to be indexed */
 709         recword->term_buf = (const char *)text;
 710         recword->term_len = text_len;
 711
 712         /* parsing all index name/type pairs */
 713         /* may not start with ' ' or ':' */
 714         while (*look && ' ' != *look && ':' != *look)
 715         {
 716             /* setting name and type to zero */
 717             *index = '\0';
 718             *type = '\0';
 719
 720             /* parsing one index name */
 721             bval = look;
 722             while (*look && ':' != *look && ' ' != *look)
 723             {
 724                 look++;
 725             }
 726             eval = look;
 727             strncpy((char *)index, (const char *)bval, eval - bval);
 728             index[eval - bval] = '\0';
 729
 730
 731             /* parsing one index type, if existing */
 732             if (':' == *look)
 733             {
 734                 look++;
 735
 736                 bval = look;
 737                 while (*look && ' ' != *look)
 738                 {
 739                     look++;
 740                 }
 741                 eval = look;
 742                 strncpy((char *)type, (const char *)bval, eval - bval);
 743                 type[eval - bval] = '\0';
 744             }
 745
 746             /* actually indexing the text given */
 747             dom_log(YLOG_DEBUG, tinfo, 0,
 748                     "INDEX '%s:%s' '%s'",
 749                     index ? (const char *) index : "null",
 750                     type ? (const char *) type : "null",
 751                     text ? (const char *) text : "null");
 752
 753             recword->index_name = (const char *)index;
 754             if (type && *type)
 755                 recword->index_type = *type;
 756             (extctr->tokenAdd)(recword);
 757
 758             /* eat whitespaces */
 759             if (*look && ' ' == *look && *(look+1))
 760             {
 761                 look++;
 762             }
 763         }
 764     }
 765
 766     xmlFree(text);
 767 }
 768
 769
 770 /* DOM filter style indexing */
 771 static void set_record_info(struct filter_info *tinfo,
 772                             struct recExtractCtrl *extctr,
 773                             xmlChar * id_p,
 774                             xmlChar * rank_p,
 775                             xmlChar * type_p)
 776 {
 777     dom_log(YLOG_DEBUG, tinfo, 0,
 778             "RECORD id=%s rank=%s type=%s",
 779             id_p ? (const char *) id_p : "null",
 780             rank_p ? (const char *) rank_p : "null",
 781             type_p ? (const char *) type_p : "null");
 782
 783     if (id_p)
 784         sscanf((const char *)id_p, "%255s", extctr->match_criteria);
 785
 786     if (rank_p)
 787         extctr->staticrank = atozint((const char *)rank_p);
 788
 789     /*     if (!strcmp("update", type_str)) */
 790     /*         index_node(tinfo, ctrl, ptr, recword); */
 791     /*     else if (!strcmp("delete", type_str)) */
 792     /*         dom_log(YLOG_WARN, tinfo, ptr, "dom filter delete: to be implemented"); */
 793     /*     else */
 794     /*         dom_log(YLOG_WARN, tinfo, ptr, "dom filter: unknown record type '%s'",  */
 795     /*                 type_str); */
 796
 797 }
 798
 799
 800 /* DOM filter style indexing */
 801 static void process_xml_element_zebra_node(struct filter_info *tinfo,
 802                                            struct recExtractCtrl *extctr,
 803                                            RecWord* recword,
 804                                            xmlNodePtr node)
 805 {
 806     if (node->type == XML_ELEMENT_NODE && node->ns && node->ns->href
 807         && 0 == XML_STRCMP(node->ns->href, zebra_dom_ns))
 808     {
 809          if (0 == XML_STRCMP(node->name, "index"))
 810          {
 811             xmlChar *index_p = 0;
 812
 813             struct _xmlAttr *attr;
 814             for (attr = node->properties; attr; attr = attr->next)
 815             {
 816                 if (attr_content_xml(attr, "name", &index_p))
 817                 {
 818                     index_value_of(tinfo, extctr, recword,node, index_p);
 819                 }
 820                 else
 821                 {
 822                     dom_log(YLOG_WARN, tinfo, node,
 823                             "bad attribute @%s, expected @name",
 824                             attr->name);
 825                 }
 826             }
 827         }
 828         else if (0 == XML_STRCMP(node->name, "record"))
 829         {
 830             xmlChar *id_p = 0;
 831             xmlChar *rank_p = 0;
 832             xmlChar *type_p = 0;
 833
 834             struct _xmlAttr *attr;
 835             for (attr = node->properties; attr; attr = attr->next)
 836             {
 837                 if (attr_content_xml(attr, "id", &id_p))
 838                     ;
 839                 else if (attr_content_xml(attr, "rank", &rank_p))
 840                     ;
 841                 else if (attr_content_xml(attr, "type", &type_p))
 842                     ;
 843                 else
 844                 {
 845                     dom_log(YLOG_WARN, tinfo, node,
 846                             "bad attribute @%s, expected @id|@rank|@type",
 847                             attr->name);
 848                 }
 849
 850                 if (type_p && 0 != strcmp("update", (const char *)type_p))
 851                 {
 852                     dom_log(YLOG_WARN, tinfo, node,
 853                             "attribute @%s, only implemented '@type='update'",
 854                             attr->name);
 855                 }
 856             }
 857             set_record_info(tinfo, extctr, id_p, rank_p, type_p);
 858         }
 859         else
 860         {
 861             dom_log(YLOG_WARN, tinfo, node,
 862                     "bad element <%s>,"
 863                     " expected <record>|<index> in namespace '%s'",
 864                     node->name, zebra_dom_ns);
 865         }
 866     }
 867 }
 868
 869
 870 /* DOM filter style indexing */
 871 static void process_xml_pi_node(struct filter_info *tinfo,
 872                                 struct recExtractCtrl *extctr,
 873                                 xmlNodePtr node,
 874                                 xmlChar **index_pp)
 875 {
 876     /* if right PI name, continue parsing PI */
 877     if (0 == strcmp(zebra_pi_name, (const char *)node->name))
 878     {
 879         xmlChar *pi_p =  node->content;
 880         xmlChar *look = pi_p;
 881
 882         xmlChar *bval;
 883         xmlChar *eval;
 884
 885         /* parsing PI record instructions */
 886         if (0 == strncmp((const char *)look, "record", 6))
 887         {
 888             xmlChar id[256];
 889             xmlChar rank[256];
 890             xmlChar type[256];
 891
 892             *id = '\0';
 893             *rank = '\0';
 894             *type = '\0';
 895
 896             look += 6;
 897
 898             /* eat whitespace */
 899             while (*look && ' ' == *look && *(look+1))
 900                 look++;
 901
 902             /* parse possible id */
 903             if (*look && 0 == strncmp((const char *)look, "id=", 3))
 904             {
 905                 look += 3;
 906                 bval = look;
 907                 while (*look && ' ' != *look)
 908                     look++;
 909                 eval = look;
 910                 strncpy((char *)id, (const char *)bval, eval - bval);
 911                 id[eval - bval] = '\0';
 912             }
 913
 914             /* eat whitespace */
 915             while (*look && ' ' == *look && *(look+1))
 916                 look++;
 917
 918             /* parse possible rank */
 919             if (*look && 0 == strncmp((const char *)look, "rank=", 5))
 920             {
 921                 look += 6;
 922                 bval = look;
 923                 while (*look && ' ' != *look)
 924                     look++;
 925                 eval = look;
 926                 strncpy((char *)rank, (const char *)bval, eval - bval);
 927                 rank[eval - bval] = '\0';
 928             }
 929
 930             /* eat whitespace */
 931             while (*look && ' ' == *look && *(look+1))
 932                 look++;
 933
 934             if (look && '\0' != *look)
 935             {
 936                 dom_log(YLOG_WARN, tinfo, node,
 937                         "content '%s', can not parse '%s'",
 938                         pi_p, look);
 939             }
 940             else
 941                 set_record_info(tinfo, extctr, id, rank, 0);
 942
 943         }
 944         /* parsing index instruction */
 945         else if (0 == strncmp((const char *)look, "index", 5))
 946         {
 947             look += 5;
 948
 949             /* eat whitespace */
 950             while (*look && ' ' == *look && *(look+1))
 951                 look++;
 952
 953             /* export index instructions to outside */
 954             *index_pp = look;
 955         }
 956         else
 957         {
 958             dom_log(YLOG_WARN, tinfo, node,
 959                     "content '%s', can not parse '%s'",
 960                     pi_p, look);
 961         }
 962     }
 963 }
 964
 965 /* DOM filter style indexing */
 966 static void process_xml_element_node(struct filter_info *tinfo,
 967                                      struct recExtractCtrl *extctr,
 968                                      RecWord* recword,
 969                                      xmlNodePtr node)
 970 {
 971     /* remember indexing instruction from PI to next element node */
 972     xmlChar *index_p = 0;
 973
 974     /* check if we are an element node in the special zebra namespace
 975        and either set record data or index value-of node content*/
 976     process_xml_element_zebra_node(tinfo, extctr, recword, node);
 977
 978     /* loop through kid nodes */
 979     for (node = node->children; node; node = node->next)
 980     {
 981         /* check and set PI record and index index instructions */
 982         if (node->type == XML_PI_NODE)
 983         {
 984             process_xml_pi_node(tinfo, extctr, node, &index_p);
 985         }
 986         else if (node->type == XML_ELEMENT_NODE)
 987         {
 988             /* if there was a PI index instruction before this element */
 989             if (index_p)
 990             {
 991                 index_value_of(tinfo, extctr, recword, node, index_p);
 992                 index_p = 0;
 993             }
 994             process_xml_element_node(tinfo, extctr, recword,node);
 995         }
 996         else
 997             continue;
 998     }
 999 }
1000
1001
1002 /* DOM filter style indexing */
1003 static void extract_dom_doc_node(struct filter_info *tinfo,
1004                                  struct recExtractCtrl *extctr,
1005                                  xmlDocPtr doc)
1006 {
1007     xmlChar *buf_out;
1008     int len_out;
1009
1010     /* only need to do the initialization once, reuse recword for all terms */
1011     RecWord recword;
1012     (*extctr->init)(extctr, &recword);
1013
1014     if (extctr->flagShowRecords)
1015     {
1016         xmlDocDumpMemory(doc, &buf_out, &len_out);
1017         fwrite(buf_out, len_out, 1, stdout);
1018         xmlFree(buf_out);
1019     }
1020
1021     process_xml_element_node(tinfo, extctr, &recword, (xmlNodePtr)doc);
1022 }
1023
1024
1025
1026
1027 static int convert_extract_doc(struct filter_info *tinfo,
1028                                struct filter_input *input,
1029                                struct recExtractCtrl *p,
1030                                xmlDocPtr doc)
1031
1032 {
1033     xmlChar *buf_out;
1034     int len_out;
1035     const char *params[10];
1036     xsltStylesheetPtr last_xsp = 0;
1037     xmlDocPtr store_doc = 0;
1038
1039     params[0] = 0;
1040     set_param_str(params, "schema", zebra_dom_ns, tinfo->odr_record);
1041
1042     /* input conversion */
1043     perform_convert(tinfo, input->convert, params, &doc, 0);
1044
1045     if (tinfo->store)
1046     {
1047         /* store conversion */
1048         store_doc = xmlCopyDoc(doc, 1);
1049         perform_convert(tinfo, tinfo->store->convert,
1050                         params, &store_doc, &last_xsp);
1051     }
1052
1053     if (last_xsp)
1054         xsltSaveResultToString(&buf_out, &len_out,
1055                                store_doc ? store_doc : doc, last_xsp);
1056     else
1057         xmlDocDumpMemory(store_doc ? store_doc : doc, &buf_out, &len_out);
1058     if (p->flagShowRecords)
1059         fwrite(buf_out, len_out, 1, stdout);
1060     (*p->setStoreData)(p, buf_out, len_out);
1061     xmlFree(buf_out);
1062
1063     if (store_doc)
1064         xmlFreeDoc(store_doc);
1065
1066     /* extract conversion */
1067     perform_convert(tinfo, tinfo->extract->convert, params, &doc, 0);
1068
1069     /* finally, do the indexing */
1070     if (doc)
1071     {
1072         extract_dom_doc_node(tinfo, p, doc);
1073         /* extract_doc_alvis(tinfo, p, doc); */
1074         xmlFreeDoc(doc);
1075     }
1076
1077     return RECCTRL_EXTRACT_OK;
1078 }
1079
1080 static int extract_xml_split(struct filter_info *tinfo,
1081                              struct filter_input *input,
1082                              struct recExtractCtrl *p)
1083 {
1084     int ret;
1085
1086     if (p->first_record)
1087     {
1088         if (input->u.xmlreader.reader)
1089             xmlFreeTextReader(input->u.xmlreader.reader);
1090         input->u.xmlreader.reader = xmlReaderForIO(ioread_ex, ioclose_ex,
1091                                                    p /* I/O handler */,
1092                                                    0 /* URL */,
1093                                                    0 /* encoding */,
1094                                                    XML_PARSE_XINCLUDE|
1095                                                    XML_PARSE_NOENT);
1096     }
1097     if (!input->u.xmlreader.reader)
1098         return RECCTRL_EXTRACT_ERROR_GENERIC;
1099
1100     ret = xmlTextReaderRead(input->u.xmlreader.reader);
1101     while (ret == 1)
1102     {
1103         int type = xmlTextReaderNodeType(input->u.xmlreader.reader);
1104         int depth = xmlTextReaderDepth(input->u.xmlreader.reader);
1105         if (type == XML_READER_TYPE_ELEMENT &&
1106             input->u.xmlreader.split_level == depth)
1107         {
1108             xmlNodePtr ptr
1109                 = xmlTextReaderExpand(input->u.xmlreader.reader);
1110             if (ptr)
1111             {
1112                 xmlNodePtr ptr2 = xmlCopyNode(ptr, 1);
1113                 xmlDocPtr doc = xmlNewDoc((const xmlChar*) "1.0");
1114
1115                 xmlDocSetRootElement(doc, ptr2);
1116
1117                 return convert_extract_doc(tinfo, input, p, doc);
1118             }
1119             else
1120             {
1121                 xmlFreeTextReader(input->u.xmlreader.reader);
1122                 input->u.xmlreader.reader = 0;
1123                 return RECCTRL_EXTRACT_ERROR_GENERIC;
1124             }
1125         }
1126         ret = xmlTextReaderRead(input->u.xmlreader.reader);
1127     }
1128     xmlFreeTextReader(input->u.xmlreader.reader);
1129     input->u.xmlreader.reader = 0;
1130     return RECCTRL_EXTRACT_EOF;
1131 }
1132
1133 static int extract_xml_full(struct filter_info *tinfo,
1134                             struct filter_input *input,
1135                             struct recExtractCtrl *p)
1136 {
1137     if (p->first_record) /* only one record per stream */
1138     {
1139         xmlDocPtr doc = xmlReadIO(ioread_ex, ioclose_ex,
1140                                   p /* I/O handler */,
1141                                   0 /* URL */,
1142                                   0 /* encoding */,
1143                                   XML_PARSE_XINCLUDE|XML_PARSE_NOENT);
1144         if (!doc)
1145         {
1146             return RECCTRL_EXTRACT_ERROR_GENERIC;
1147         }
1148         return convert_extract_doc(tinfo, input, p, doc);
1149     }
1150     else
1151         return RECCTRL_EXTRACT_EOF;
1152 }
1153
1154 static int extract_iso2709(struct filter_info *tinfo,
1155                            struct filter_input *input,
1156                            struct recExtractCtrl *p)
1157 {
1158     char buf[100000];
1159     int record_length;
1160     int read_bytes, r;
1161
1162     if (p->stream->readf(p->stream, buf, 5) != 5)
1163         return RECCTRL_EXTRACT_EOF;
1164     while (*buf < '0' || *buf > '9')
1165     {
1166         int i;
1167
1168         dom_log(YLOG_WARN, tinfo, 0,
1169                 "MARC: Skipping bad byte %d (0x%02X)",
1170                 *buf & 0xff, *buf & 0xff);
1171         for (i = 0; i<4; i++)
1172             buf[i] = buf[i+1];
1173
1174         if (p->stream->readf(p->stream, buf+4, 1) != 1)
1175             return RECCTRL_EXTRACT_EOF;
1176     }
1177     record_length = atoi_n (buf, 5);
1178     if (record_length < 25)
1179     {
1180         dom_log(YLOG_WARN, tinfo, 0,
1181                 "MARC record length < 25, is %d",  record_length);
1182         return RECCTRL_EXTRACT_ERROR_GENERIC;
1183     }
1184     read_bytes = p->stream->readf(p->stream, buf+5, record_length-5);
1185     if (read_bytes < record_length-5)
1186     {
1187         dom_log(YLOG_WARN, tinfo, 0,
1188                 "couldn't read whole MARC record");
1189         return RECCTRL_EXTRACT_ERROR_GENERIC;
1190     }
1191     r = yaz_marc_read_iso2709(input->u.marc.handle,  buf, record_length);
1192     if (r < record_length)
1193     {
1194         dom_log (YLOG_WARN, tinfo, 0,
1195                  "parsing of MARC record failed r=%d length=%d",
1196                  r, record_length);
1197         return RECCTRL_EXTRACT_ERROR_GENERIC;
1198     }
1199     else
1200     {
1201         xmlDocPtr rdoc;
1202         xmlNode *root_ptr;
1203         yaz_marc_write_xml(input->u.marc.handle, &root_ptr, 0, 0, 0);
1204         rdoc = xmlNewDoc((const xmlChar*) "1.0");
1205         xmlDocSetRootElement(rdoc, root_ptr);
1206         return convert_extract_doc(tinfo, input, p, rdoc);
1207     }
1208     return RECCTRL_EXTRACT_OK;
1209 }
1210
1211 static int filter_extract(void *clientData, struct recExtractCtrl *p)
1212 {
1213     struct filter_info *tinfo = clientData;
1214     struct filter_input *input = tinfo->input_list;
1215
1216     if (!input)
1217         return RECCTRL_EXTRACT_ERROR_GENERIC;
1218
1219     odr_reset(tinfo->odr_record);
1220     switch(input->type)
1221     {
1222     case DOM_INPUT_XMLREADER:
1223         if (input->u.xmlreader.split_level == 0)
1224             return extract_xml_full(tinfo, input, p);
1225         else
1226             return extract_xml_split(tinfo, input, p);
1227         break;
1228     case DOM_INPUT_MARC:
1229         return extract_iso2709(tinfo, input, p);
1230     }
1231     return RECCTRL_EXTRACT_ERROR_GENERIC;
1232 }
1233
1234 static int ioread_ret(void *context, char *buffer, int len)
1235 {
1236     struct recRetrieveCtrl *p = context;
1237     return p->stream->readf(p->stream, buffer, len);
1238 }
1239
1240 static int ioclose_ret(void *context)
1241 {
1242     return 0;
1243 }
1244
1245 static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p)
1246 {
1247     /* const char *esn = zebra_dom_ns; */
1248     const char *esn = 0;
1249     const char *params[32];
1250     struct filter_info *tinfo = clientData;
1251     xmlDocPtr doc;
1252     struct filter_retrieve *retrieve;
1253     xsltStylesheetPtr last_xsp = 0;
1254
1255     if (p->comp)
1256     {
1257         if (p->comp->which == Z_RecordComp_simple
1258             && p->comp->u.simple->which == Z_ElementSetNames_generic)
1259         {
1260             esn = p->comp->u.simple->u.generic;
1261         }
1262         else if (p->comp->which == Z_RecordComp_complex
1263                  && p->comp->u.complex->generic->elementSpec
1264                  && p->comp->u.complex->generic->elementSpec->which ==
1265                  Z_ElementSpec_elementSetName)
1266         {
1267             esn = p->comp->u.complex->generic->elementSpec->u.elementSetName;
1268         }
1269     }
1270     retrieve = lookup_retrieve(tinfo, esn);
1271     if (!retrieve)
1272     {
1273         p->diagnostic =
1274             YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
1275         return 0;
1276     }
1277
1278     params[0] = 0;
1279     set_param_int(params, "id", p->localno, p->odr);
1280     if (p->fname)
1281         set_param_str(params, "filename", p->fname, p->odr);
1282     if (p->staticrank >= 0)
1283         set_param_int(params, "rank", p->staticrank, p->odr);
1284
1285     if (esn)
1286         set_param_str(params, "schema", esn, p->odr);
1287     else
1288         if (retrieve->name)
1289             set_param_str(params, "schema", retrieve->name, p->odr);
1290         else if (retrieve->identifier)
1291             set_param_str(params, "schema", retrieve->identifier, p->odr);
1292         else
1293             set_param_str(params, "schema", "", p->odr);
1294
1295     if (p->score >= 0)
1296         set_param_int(params, "score", p->score, p->odr);
1297     set_param_int(params, "size", p->recordSize, p->odr);
1298
1299     doc = xmlReadIO(ioread_ret, ioclose_ret, p /* I/O handler */,
1300                     0 /* URL */,
1301                     0 /* encoding */,
1302                     XML_PARSE_XINCLUDE|XML_PARSE_NOENT);
1303     if (!doc)
1304     {
1305         p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1306         return 0;
1307     }
1308
1309     /* retrieve conversion */
1310     perform_convert(tinfo, retrieve->convert, params, &doc, &last_xsp);
1311     if (!doc)
1312     {
1313         p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1314     }
1315     else if (p->input_format == VAL_NONE || p->input_format == VAL_TEXT_XML)
1316     {
1317         xmlChar *buf_out;
1318         int len_out;
1319
1320         if (last_xsp)
1321             xsltSaveResultToString(&buf_out, &len_out, doc, last_xsp);
1322         else
1323             xmlDocDumpMemory(doc, &buf_out, &len_out);
1324
1325         p->output_format = VAL_TEXT_XML;
1326         p->rec_len = len_out;
1327         p->rec_buf = odr_malloc(p->odr, p->rec_len);
1328         memcpy(p->rec_buf, buf_out, p->rec_len);
1329         xmlFree(buf_out);
1330     }
1331     else if (p->output_format == VAL_SUTRS)
1332     {
1333         xmlChar *buf_out;
1334         int len_out;
1335
1336         if (last_xsp)
1337             xsltSaveResultToString(&buf_out, &len_out, doc, last_xsp);
1338         else
1339             xmlDocDumpMemory(doc, &buf_out, &len_out);
1340
1341         p->output_format = VAL_SUTRS;
1342         p->rec_len = len_out;
1343         p->rec_buf = odr_malloc(p->odr, p->rec_len);
1344         memcpy(p->rec_buf, buf_out, p->rec_len);
1345
1346         xmlFree(buf_out);
1347     }
1348     else
1349     {
1350         p->diagnostic = YAZ_BIB1_RECORD_SYNTAX_UNSUPP;
1351     }
1352     xmlFreeDoc(doc);
1353     return 0;
1354 }
1355
1356 static struct recType filter_type = {
1357     0,
1358     "dom",
1359     filter_init,
1360     filter_config,
1361     filter_destroy,
1362     filter_extract,
1363     filter_retrieve
1364 };
1365
1366 RecType
1367 #ifdef IDZEBRA_STATIC_DOM
1368 idzebra_filter_dom
1369 #else
1370 idzebra_filter
1371 #endif
1372
1373 [] = {
1374     &filter_type,
1375     0,
1376 };
1377 /*
1378  * Local variables:
1379  * c-basic-offset: 4
1380  * indent-tabs-mode: nil
1381  * End:
1382  * vim: shiftwidth=4 tabstop=8 expandtab
1383  */
1384