index/mod_dom.c

   1 /* $Id: mod_dom.c,v 1.17 2007-02-23 11:10:37 adam Exp $
   2    Copyright (C) 1995-2007
   3    Index Data ApS
   4
   5    This file is part of the Zebra server.
   6
   7    Zebra is free software; you can redistribute it and/or modify it under
   8    the terms of the GNU General Public License as published by the Free
   9    Software Foundation; either version 2, or (at your option) any later
  10    version.
  11
  12    Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15    for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, write to the Free Software
  19    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #include <ctype.h>
  26 #include <stdarg.h>
  27
  28 #include <yaz/diagbib1.h>
  29 #include <yaz/tpath.h>
  30 #include <yaz/snprintf.h>
  31
  32 #include <libxml/xmlversion.h>
  33 #include <libxml/parser.h>
  34 #include <libxml/tree.h>
  35 #include <libxml/xmlIO.h>
  36 #include <libxml/xmlreader.h>
  37 #include <libxslt/transform.h>
  38 #include <libxslt/xsltutils.h>
  39
  40 #if YAZ_HAVE_EXSLT
  41 #include <libexslt/exslt.h>
  42 #endif
  43
  44 #include <idzebra/util.h>
  45 #include <idzebra/recctrl.h>
  46
  47 /* DOM filter style indexing */
  48 #define ZEBRA_DOM_NS "http://indexdata.com/zebra-2.0"
  49 static const char *zebra_dom_ns = ZEBRA_DOM_NS;
  50
  51 /* DOM filter style indexing */
  52 #define ZEBRA_PI_NAME "zebra-2.0"
  53 static const char *zebra_pi_name = ZEBRA_PI_NAME;
  54
  55
  56
  57 struct convert_s {
  58     const char *stylesheet;
  59     xsltStylesheetPtr stylesheet_xsp;
  60     struct convert_s *next;
  61 };
  62
  63 struct filter_extract {
  64     const char *name;
  65     struct convert_s *convert;
  66 };
  67
  68 struct filter_store {
  69     struct convert_s *convert;
  70 };
  71
  72 struct filter_retrieve {
  73     const char *name;
  74     const char *identifier;
  75     struct convert_s *convert;
  76     struct filter_retrieve *next;
  77 };
  78
  79 #define DOM_INPUT_XMLREADER 1
  80 #define DOM_INPUT_MARC 2
  81 struct filter_input {
  82     const char *syntax;
  83     const char *name;
  84     struct convert_s *convert;
  85     int type;
  86     union {
  87         struct {
  88             const char *input_charset;
  89             yaz_marc_t handle;
  90             yaz_iconv_t iconv;
  91         } marc;
  92         struct {
  93             xmlTextReaderPtr reader;
  94             int split_level;
  95         } xmlreader;
  96     } u;
  97     struct filter_input *next;
  98 };
  99
 100 struct filter_info {
 101     char *fname;
 102     char *full_name;
 103     const char *profile_path;
 104     ODR odr_record;
 105     ODR odr_config;
 106     xmlDocPtr doc_config;
 107     struct filter_extract *extract;
 108     struct filter_retrieve *retrieve_list;
 109     struct filter_input *input_list;
 110     struct filter_store *store;
 111 };
 112
 113
 114
 115 #define XML_STRCMP(a,b)   strcmp((char*)a, b)
 116 #define XML_STRLEN(a) strlen((char*)a)
 117
 118 static void dom_log(int level, struct filter_info *tinfo, xmlNodePtr ptr,
 119                     const char *fmt, ...)
 120 #ifdef __GNUC__
 121     __attribute__ ((format (printf, 4, 5)))
 122 #endif
 123     ;
 124
 125 static void dom_log(int level, struct filter_info *tinfo, xmlNodePtr ptr,
 126                     const char *fmt, ...)
 127 {
 128     va_list ap;
 129     char buf[4096];
 130     xmlChar *node_path = 0;
 131
 132     if (ptr)
 133         node_path = xmlGetNodePath(ptr);
 134
 135     va_start(ap, fmt);
 136     yaz_vsnprintf(buf, sizeof(buf)-1, fmt, ap);
 137     yaz_log(level, "%s: dom filter %s%s: %s",
 138             tinfo->fname ? tinfo->fname : "none",
 139             node_path ? "in " : "",
 140             node_path ? (const char *) node_path : "", buf);
 141
 142     if (node_path)
 143         xmlFree(node_path);
 144     va_end(ap);
 145 }
 146
 147
 148 static void set_param_str(const char **params, const char *name,
 149                           const char *value, ODR odr)
 150 {
 151     char *quoted = odr_malloc(odr, 3 + strlen(value));
 152     sprintf(quoted, "'%s'", value);
 153     while (*params)
 154         params++;
 155     params[0] = name;
 156     params[1] = quoted;
 157     params[2] = 0;
 158 }
 159
 160 static void set_param_int(const char **params, const char *name,
 161                           zint value, ODR odr)
 162 {
 163     char *quoted = odr_malloc(odr, 30); /* 25 digits enough for 2^64 */
 164     while (*params)
 165         params++;
 166     sprintf(quoted, "'" ZINT_FORMAT "'", value);
 167     params[0] = name;
 168     params[1] = quoted;
 169     params[2] = 0;
 170 }
 171
 172 static void *filter_init(Res res, RecType recType)
 173 {
 174     struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo));
 175     tinfo->fname = 0;
 176     tinfo->full_name = 0;
 177     tinfo->profile_path = 0;
 178     tinfo->odr_record = odr_createmem(ODR_ENCODE);
 179     tinfo->odr_config = odr_createmem(ODR_ENCODE);
 180     tinfo->extract = 0;
 181     tinfo->retrieve_list = 0;
 182     tinfo->input_list = 0;
 183     tinfo->store = 0;
 184     tinfo->doc_config = 0;
 185
 186 #if YAZ_HAVE_EXSLT
 187     exsltRegisterAll();
 188 #endif
 189
 190     return tinfo;
 191 }
 192
 193 static int attr_content(struct _xmlAttr *attr, const char *name,
 194                         const char **dst_content)
 195 {
 196     if (!XML_STRCMP(attr->name, name) && attr->children
 197         && attr->children->type == XML_TEXT_NODE)
 198     {
 199         *dst_content = (const char *)(attr->children->content);
 200         return 1;
 201     }
 202     return 0;
 203 }
 204
 205 static void destroy_xsp(struct convert_s *c)
 206 {
 207     while(c)
 208     {
 209         if (c->stylesheet_xsp)
 210             xsltFreeStylesheet(c->stylesheet_xsp);
 211         c = c->next;
 212     }
 213 }
 214
 215 static void destroy_dom(struct filter_info *tinfo)
 216 {
 217     if (tinfo->extract)
 218     {
 219         destroy_xsp(tinfo->extract->convert);
 220         tinfo->extract = 0;
 221     }
 222     if (tinfo->store)
 223     {
 224         destroy_xsp(tinfo->store->convert);
 225         tinfo->store = 0;
 226     }
 227     if (tinfo->input_list)
 228     {
 229         struct filter_input *i_ptr;
 230         for (i_ptr = tinfo->input_list; i_ptr; i_ptr = i_ptr->next)
 231         {
 232             switch(i_ptr->type)
 233             {
 234             case DOM_INPUT_XMLREADER:
 235                 if (i_ptr->u.xmlreader.reader)
 236                     xmlFreeTextReader(i_ptr->u.xmlreader.reader);
 237                 break;
 238             case DOM_INPUT_MARC:
 239                 yaz_iconv_close(i_ptr->u.marc.iconv);
 240                 yaz_marc_destroy(i_ptr->u.marc.handle);
 241                 break;
 242             }
 243             destroy_xsp(i_ptr->convert);
 244         }
 245         tinfo->input_list = 0;
 246     }
 247     if (tinfo->retrieve_list)
 248     {
 249         struct filter_retrieve *r_ptr;
 250         for (r_ptr = tinfo->retrieve_list; r_ptr; r_ptr = r_ptr->next)
 251             destroy_xsp(r_ptr->convert);
 252         tinfo->retrieve_list = 0;
 253     }
 254
 255     if (tinfo->doc_config)
 256     {
 257         xmlFreeDoc(tinfo->doc_config);
 258         tinfo->doc_config = 0;
 259     }
 260     odr_reset(tinfo->odr_config);
 261 }
 262
 263 static ZEBRA_RES parse_convert(struct filter_info *tinfo, xmlNodePtr ptr,
 264                                struct convert_s **l)
 265 {
 266     *l = 0;
 267     for(; ptr; ptr = ptr->next)
 268     {
 269         if (ptr->type != XML_ELEMENT_NODE)
 270             continue;
 271         if (!XML_STRCMP(ptr->name, "xslt"))
 272         {
 273             struct _xmlAttr *attr;
 274             struct convert_s *p
 275                 = odr_malloc(tinfo->odr_config, sizeof(*p));
 276
 277             p->next = 0;
 278             p->stylesheet = 0;
 279             p->stylesheet_xsp = 0;
 280
 281             for (attr = ptr->properties; attr; attr = attr->next)
 282                 if (attr_content(attr, "stylesheet", &p->stylesheet))
 283                     ;
 284                 else
 285                 {
 286                     dom_log(YLOG_WARN, tinfo, ptr,
 287                             "bad attribute @%s", attr->name);
 288                 }
 289             if (p->stylesheet)
 290             {
 291                 char tmp_xslt_full_name[1024];
 292                 if (!yaz_filepath_resolve(p->stylesheet,
 293                                           tinfo->profile_path,
 294                                           NULL,
 295                                           tmp_xslt_full_name))
 296                 {
 297                     dom_log(YLOG_WARN, tinfo, 0,
 298                             "stylesheet %s not found in "
 299                             "path %s",
 300                             p->stylesheet,
 301                             tinfo->profile_path);
 302                     return ZEBRA_FAIL;
 303                 }
 304
 305                 p->stylesheet_xsp
 306                     = xsltParseStylesheetFile((const xmlChar*)
 307                                               tmp_xslt_full_name);
 308                 if (!p->stylesheet_xsp)
 309                 {
 310                     dom_log(YLOG_WARN, tinfo, 0,
 311                             "could not parse xslt stylesheet %s",
 312                             tmp_xslt_full_name);
 313                     return ZEBRA_FAIL;
 314                 }
 315             }
 316             else
 317             {
 318                 dom_log(YLOG_WARN, tinfo, ptr,
 319                         "missing attribute 'stylesheet' ");
 320                 return ZEBRA_FAIL;
 321             }
 322             *l = p;
 323             l = &p->next;
 324         }
 325         else
 326         {
 327             dom_log(YLOG_WARN, tinfo, ptr,
 328                     "bad node '%s'", ptr->name);
 329             return ZEBRA_FAIL;
 330         }
 331     }
 332     return ZEBRA_OK;
 333 }
 334
 335 static ZEBRA_RES perform_convert(struct filter_info *tinfo,
 336                                  struct convert_s *convert,
 337                                  const char **params,
 338                                  xmlDocPtr *doc,
 339                                  xsltStylesheetPtr *last_xsp)
 340 {
 341     for (; convert; convert = convert->next)
 342     {
 343         xmlDocPtr res_doc = xsltApplyStylesheet(convert->stylesheet_xsp,
 344                                                 *doc, params);
 345         if (last_xsp)
 346             *last_xsp = convert->stylesheet_xsp;
 347         xmlFreeDoc(*doc);
 348         *doc = res_doc;
 349     }
 350     return ZEBRA_OK;
 351 }
 352
 353 static struct filter_input *new_input(struct filter_info *tinfo, int type)
 354 {
 355     struct filter_input *p;
 356     struct filter_input **np = &tinfo->input_list;
 357     for (;*np; np = &(*np)->next)
 358         ;
 359     p = *np = odr_malloc(tinfo->odr_config, sizeof(*p));
 360     p->next = 0;
 361     p->syntax = 0;
 362     p->name = 0;
 363     p->convert = 0;
 364     p->type = type;
 365     return p;
 366 }
 367
 368 static ZEBRA_RES parse_input(struct filter_info *tinfo, xmlNodePtr ptr,
 369                              const char *syntax, const char *name)
 370 {
 371     for (; ptr; ptr = ptr->next)
 372     {
 373         if (ptr->type != XML_ELEMENT_NODE)
 374             continue;
 375         if (!XML_STRCMP(ptr->name, "marc"))
 376         {
 377             yaz_iconv_t iconv = 0;
 378             const char *input_charset = "marc-8";
 379             struct _xmlAttr *attr;
 380
 381             for (attr = ptr->properties; attr; attr = attr->next)
 382             {
 383                 if (attr_content(attr, "charset", &input_charset))
 384                     ;
 385                 else
 386                 {
 387                     dom_log(YLOG_WARN, tinfo, ptr,
 388                             "bad attribute @%s, expected @charset",
 389                             attr->name);
 390                 }
 391             }
 392             iconv = yaz_iconv_open("utf-8", input_charset);
 393             if (!iconv)
 394             {
 395                 dom_log(YLOG_WARN, tinfo, ptr,
 396                         "unsupported @charset '%s'", input_charset);
 397                 return ZEBRA_FAIL;
 398             }
 399             else
 400             {
 401                 struct filter_input *p
 402                     = new_input(tinfo, DOM_INPUT_MARC);
 403                 p->u.marc.handle = yaz_marc_create();
 404                 p->u.marc.iconv = iconv;
 405
 406                 yaz_marc_iconv(p->u.marc.handle, p->u.marc.iconv);
 407
 408                 ptr = ptr->next;
 409
 410                 parse_convert(tinfo, ptr, &p->convert);
 411             }
 412             break;
 413
 414         }
 415         else if (!XML_STRCMP(ptr->name, "xmlreader"))
 416         {
 417             struct filter_input *p
 418                 = new_input(tinfo, DOM_INPUT_XMLREADER);
 419             struct _xmlAttr *attr;
 420             const char *level_str = 0;
 421
 422             p->u.xmlreader.split_level = 0;
 423             p->u.xmlreader.reader = 0;
 424
 425             for (attr = ptr->properties; attr; attr = attr->next)
 426             {
 427                 if (attr_content(attr, "level", &level_str))
 428                     ;
 429                 else
 430                 {
 431                     dom_log(YLOG_WARN, tinfo, ptr,
 432                             "bad attribute @%s, expected @level",
 433                             attr->name);
 434                 }
 435             }
 436             if (level_str)
 437                 p->u.xmlreader.split_level = atoi(level_str);
 438
 439             ptr = ptr->next;
 440
 441             parse_convert(tinfo, ptr, &p->convert);
 442             break;
 443         }
 444         else
 445         {
 446             dom_log(YLOG_WARN, tinfo, ptr,
 447                     "bad element <%s>, expected <marc>|<xmlreader>",
 448                     ptr->name);
 449             return ZEBRA_FAIL;
 450         }
 451     }
 452     return ZEBRA_OK;
 453 }
 454
 455 static ZEBRA_RES parse_dom(struct filter_info *tinfo, const char *fname)
 456 {
 457     char tmp_full_name[1024];
 458     xmlNodePtr ptr;
 459     xmlDocPtr doc;
 460
 461     tinfo->fname = odr_strdup(tinfo->odr_config, fname);
 462
 463     if (yaz_filepath_resolve(tinfo->fname, tinfo->profile_path,
 464                              NULL, tmp_full_name))
 465         tinfo->full_name = odr_strdup(tinfo->odr_config, tmp_full_name);
 466     else
 467         tinfo->full_name = odr_strdup(tinfo->odr_config, tinfo->fname);
 468
 469     yaz_log(YLOG_LOG, "%s dom filter: "
 470             "loading config file %s", tinfo->fname, tinfo->full_name);
 471
 472     doc = xmlParseFile(tinfo->full_name);
 473     if (!doc)
 474     {
 475         yaz_log(YLOG_WARN, "%s: dom filter: "
 476                 "failed to parse config file %s",
 477                 tinfo->fname, tinfo->full_name);
 478         return ZEBRA_FAIL;
 479     }
 480     /* save because we store ptrs to the content */
 481     tinfo->doc_config = doc;
 482
 483     ptr = xmlDocGetRootElement(doc);
 484     if (!ptr || ptr->type != XML_ELEMENT_NODE
 485         || XML_STRCMP(ptr->name, "dom"))
 486     {
 487         dom_log(YLOG_WARN, tinfo, ptr,
 488                 "bad root element <%s>, expected root element <dom>",
 489                 ptr->name);
 490         return ZEBRA_FAIL;
 491     }
 492
 493     for (ptr = ptr->children; ptr; ptr = ptr->next)
 494     {
 495         if (ptr->type != XML_ELEMENT_NODE)
 496             continue;
 497         if (!XML_STRCMP(ptr->name, "extract"))
 498         {
 499             /*
 500               <extract name="index">
 501               <xslt stylesheet="first.xsl"/>
 502               <xslt stylesheet="second.xsl"/>
 503               </extract>
 504             */
 505             struct _xmlAttr *attr;
 506             struct filter_extract *f =
 507                 odr_malloc(tinfo->odr_config, sizeof(*f));
 508
 509             tinfo->extract = f;
 510             f->name = 0;
 511             f->convert = 0;
 512             for (attr = ptr->properties; attr; attr = attr->next)
 513             {
 514                 if (attr_content(attr, "name", &f->name))
 515                     ;
 516                 else
 517                 {
 518                     dom_log(YLOG_WARN, tinfo, ptr,
 519                             "bad attribute @%s, expected @name",
 520                             attr->name);
 521                 }
 522             }
 523             parse_convert(tinfo, ptr->children, &f->convert);
 524         }
 525         else if (!XML_STRCMP(ptr->name, "retrieve"))
 526         {
 527             /*
 528                <retrieve name="F">
 529                <xslt stylesheet="some.xsl"/>
 530                <xslt stylesheet="some.xsl"/>
 531                </retrieve>
 532             */
 533             struct _xmlAttr *attr;
 534             struct filter_retrieve **fp = &tinfo->retrieve_list;
 535             struct filter_retrieve *f =
 536                 odr_malloc(tinfo->odr_config, sizeof(*f));
 537
 538             while (*fp)
 539                 fp = &(*fp)->next;
 540
 541             *fp = f;
 542             f->name = 0;
 543             f->identifier = 0;
 544             f->convert = 0;
 545             f->next = 0;
 546
 547             for (attr = ptr->properties; attr; attr = attr->next)
 548             {
 549                 if (attr_content(attr, "identifier",
 550                                  &f->identifier))
 551                     ;
 552                 else if (attr_content(attr, "name", &f->name))
 553                     ;
 554                 else
 555                 {
 556                     dom_log(YLOG_WARN, tinfo, ptr,
 557                             "bad attribute @%s,  expected @identifier|@name",
 558                             attr->name);
 559                 }
 560             }
 561             parse_convert(tinfo, ptr->children, &f->convert);
 562         }
 563         else if (!XML_STRCMP(ptr->name, "store"))
 564         {
 565             /*
 566               <store name="F">
 567               <xslt stylesheet="some.xsl"/>
 568               <xslt stylesheet="some.xsl"/>
 569               </retrieve>
 570             */
 571             struct filter_store *f =
 572                 odr_malloc(tinfo->odr_config, sizeof(*f));
 573
 574             tinfo->store = f;
 575             f->convert = 0;
 576             parse_convert(tinfo, ptr->children, &f->convert);
 577         }
 578         else if (!XML_STRCMP(ptr->name, "input"))
 579         {
 580             /*
 581               <input syntax="xml">
 582               <xmlreader level="1"/>
 583               </input>
 584               <input syntax="usmarc">
 585               <marc inputcharset="marc-8"/>
 586               </input>
 587             */
 588             struct _xmlAttr *attr;
 589             const char  *syntax = 0;
 590             const char *name = 0;
 591             for (attr = ptr->properties; attr; attr = attr->next)
 592             {
 593                 if (attr_content(attr, "syntax", &syntax))
 594                     ;
 595                 else if (attr_content(attr, "name", &name))
 596                     ;
 597                 else
 598                 {
 599                     dom_log(YLOG_WARN, tinfo, ptr,
 600                             "bad attribute @%s,  expected @syntax|@name",
 601                             attr->name);
 602                 }
 603             }
 604             parse_input(tinfo, ptr->children, syntax, name);
 605         }
 606         else
 607         {
 608             dom_log(YLOG_WARN, tinfo, ptr,
 609                     "bad element <%s>, "
 610                     "expected <extract>|<input>|<retrieve>|<store>",
 611                     ptr->name);
 612             return ZEBRA_FAIL;
 613         }
 614     }
 615     return ZEBRA_OK;
 616 }
 617
 618 static struct filter_retrieve *lookup_retrieve(struct filter_info *tinfo,
 619                                                const char *est)
 620 {
 621     struct filter_retrieve *f = tinfo->retrieve_list;
 622
 623     /* return first schema if no est is provided */
 624     if (!est)
 625         return f;
 626     for (; f; f = f->next)
 627     {
 628         /* find requested schema */
 629         if (est)
 630         {
 631             if (f->identifier && !strcmp(f->identifier, est))
 632                 return f;
 633             if (f->name && !strcmp(f->name, est))
 634                 return f;
 635         }
 636     }
 637     return 0;
 638 }
 639
 640 static ZEBRA_RES filter_config(void *clientData, Res res, const char *args)
 641 {
 642     struct filter_info *tinfo = clientData;
 643     if (!args || !*args)
 644     {
 645         yaz_log(YLOG_WARN, "dom filter: need config file");
 646         return ZEBRA_FAIL;
 647     }
 648
 649     if (tinfo->fname && !strcmp(args, tinfo->fname))
 650         return ZEBRA_OK;
 651
 652     tinfo->profile_path = res_get(res, "profilePath");
 653
 654     destroy_dom(tinfo);
 655     return parse_dom(tinfo, args);
 656 }
 657
 658 static void filter_destroy(void *clientData)
 659 {
 660     struct filter_info *tinfo = clientData;
 661     destroy_dom(tinfo);
 662     odr_destroy(tinfo->odr_config);
 663     odr_destroy(tinfo->odr_record);
 664     xfree(tinfo);
 665 }
 666
 667 static int ioread_ex(void *context, char *buffer, int len)
 668 {
 669     struct recExtractCtrl *p = context;
 670     return p->stream->readf(p->stream, buffer, len);
 671 }
 672
 673 static int ioclose_ex(void *context)
 674 {
 675     return 0;
 676 }
 677
 678
 679 /* DOM filter style indexing */
 680 static int attr_content_xml(struct _xmlAttr *attr, const char *name,
 681                             xmlChar **dst_content)
 682 {
 683     if (0 == XML_STRCMP(attr->name, name) && attr->children
 684         && attr->children->type == XML_TEXT_NODE)
 685     {
 686         *dst_content = (attr->children->content);
 687         return 1;
 688     }
 689     return 0;
 690 }
 691
 692
 693 /* DOM filter style indexing */
 694 static void index_value_of(struct filter_info *tinfo,
 695                            struct recExtractCtrl *extctr,
 696                            RecWord* recword,
 697                            xmlNodePtr node,
 698                            xmlChar * index_p)
 699 {
 700     xmlChar *text = xmlNodeGetContent(node);
 701     size_t text_len = strlen((const char *)text);
 702
 703
 704     /* if there is no text, we do not need to proceed */
 705     if (text_len)
 706     {
 707         xmlChar *look = index_p;
 708         xmlChar *bval;
 709         xmlChar *eval;
 710
 711         xmlChar index[256];
 712         xmlChar type[256];
 713
 714         /* assingning text to be indexed */
 715         recword->term_buf = (const char *)text;
 716         recword->term_len = text_len;
 717
 718         /* parsing all index name/type pairs */
 719         /* may not start with ' ' or ':' */
 720         while (*look && ' ' != *look && ':' != *look)
 721         {
 722             /* setting name and type to zero */
 723             *index = '\0';
 724             *type = '\0';
 725
 726             /* parsing one index name */
 727             bval = look;
 728             while (*look && ':' != *look && ' ' != *look)
 729             {
 730                 look++;
 731             }
 732             eval = look;
 733             strncpy((char *)index, (const char *)bval, eval - bval);
 734             index[eval - bval] = '\0';
 735
 736
 737             /* parsing one index type, if existing */
 738             if (':' == *look)
 739             {
 740                 look++;
 741
 742                 bval = look;
 743                 while (*look && ' ' != *look)
 744                 {
 745                     look++;
 746                 }
 747                 eval = look;
 748                 strncpy((char *)type, (const char *)bval, eval - bval);
 749                 type[eval - bval] = '\0';
 750             }
 751
 752             /* actually indexing the text given */
 753             dom_log(YLOG_DEBUG, tinfo, 0,
 754                     "INDEX '%s:%s' '%s'",
 755                     index, type, text);
 756
 757             recword->index_name = (const char *)index;
 758             if (type && *type)
 759                 recword->index_type = *type;
 760             (extctr->tokenAdd)(recword);
 761
 762             /* eat whitespaces */
 763             if (*look && ' ' == *look && *(look+1))
 764             {
 765                 look++;
 766             }
 767         }
 768     }
 769
 770     xmlFree(text);
 771 }
 772
 773
 774 /* DOM filter style indexing */
 775 static void set_record_info(struct filter_info *tinfo,
 776                             struct recExtractCtrl *extctr,
 777                             xmlChar * id_p,
 778                             xmlChar * rank_p,
 779                             xmlChar * type_p)
 780 {
 781     dom_log(YLOG_DEBUG, tinfo, 0,
 782             "RECORD id=%s rank=%s type=%s",
 783             id_p, rank_p, type_p);
 784
 785     if (id_p)
 786         sscanf((const char *)id_p, "%255s", extctr->match_criteria);
 787
 788     if (rank_p)
 789         extctr->staticrank = atozint((const char *)rank_p);
 790
 791     /*     if (!strcmp("update", type_str)) */
 792     /*         index_node(tinfo, ctrl, ptr, recword); */
 793     /*     else if (!strcmp("delete", type_str)) */
 794     /*         dom_log(YLOG_WARN, tinfo, ptr, "dom filter delete: to be implemented"); */
 795     /*     else */
 796     /*         dom_log(YLOG_WARN, tinfo, ptr, "dom filter: unknown record type '%s'",  */
 797     /*                 type_str); */
 798
 799 }
 800
 801
 802 /* DOM filter style indexing */
 803 static void process_xml_element_zebra_node(struct filter_info *tinfo,
 804                                            struct recExtractCtrl *extctr,
 805                                            RecWord* recword,
 806                                            xmlNodePtr node)
 807 {
 808     if (node->type == XML_ELEMENT_NODE && node->ns && node->ns->href
 809         && 0 == XML_STRCMP(node->ns->href, zebra_dom_ns))
 810     {
 811          if (0 == XML_STRCMP(node->name, "index"))
 812          {
 813             xmlChar *index_p = 0;
 814
 815             struct _xmlAttr *attr;
 816             for (attr = node->properties; attr; attr = attr->next)
 817             {
 818                 if (attr_content_xml(attr, "name", &index_p))
 819                 {
 820                     index_value_of(tinfo, extctr, recword,node, index_p);
 821                 }
 822                 else
 823                 {
 824                     dom_log(YLOG_WARN, tinfo, node,
 825                             "bad attribute @%s, expected @name",
 826                             attr->name);
 827                 }
 828             }
 829         }
 830         else if (0 == XML_STRCMP(node->name, "record"))
 831         {
 832             xmlChar *id_p = 0;
 833             xmlChar *rank_p = 0;
 834             xmlChar *type_p = 0;
 835
 836             struct _xmlAttr *attr;
 837             for (attr = node->properties; attr; attr = attr->next)
 838             {
 839                 if (attr_content_xml(attr, "id", &id_p))
 840                     ;
 841                 else if (attr_content_xml(attr, "rank", &rank_p))
 842                     ;
 843                 else if (attr_content_xml(attr, "type", &type_p))
 844                     ;
 845                 else
 846                 {
 847                     dom_log(YLOG_WARN, tinfo, node,
 848                             "bad attribute @%s, expected @id|@rank|@type",
 849                             attr->name);
 850                 }
 851
 852                 if (type_p && 0 != strcmp("update", (const char *)type_p))
 853                 {
 854                     dom_log(YLOG_WARN, tinfo, node,
 855                             "attribute @%s, only implemented '@type='update'",
 856                             attr->name);
 857                 }
 858             }
 859             set_record_info(tinfo, extctr, id_p, rank_p, type_p);
 860         }
 861         else
 862         {
 863             dom_log(YLOG_WARN, tinfo, node,
 864                     "bad element <%s>,"
 865                     " expected <record>|<index> in namespace '%s'",
 866                     node->name, zebra_dom_ns);
 867         }
 868     }
 869 }
 870
 871
 872 /* DOM filter style indexing */
 873 static void process_xml_pi_node(struct filter_info *tinfo,
 874                                 struct recExtractCtrl *extctr,
 875                                 xmlNodePtr node,
 876                                 xmlChar **index_pp)
 877 {
 878     /* if right PI name, continue parsing PI */
 879     if (0 == strcmp(zebra_pi_name, (const char *)node->name))
 880     {
 881         xmlChar *pi_p =  node->content;
 882         xmlChar *look = pi_p;
 883
 884         xmlChar *bval;
 885         xmlChar *eval;
 886
 887         /* parsing PI record instructions */
 888         if (0 == strncmp((const char *)look, "record", 6))
 889         {
 890             xmlChar id[256];
 891             xmlChar rank[256];
 892             xmlChar type[256];
 893
 894             *id = '\0';
 895             *rank = '\0';
 896             *type = '\0';
 897
 898             look += 6;
 899
 900             /* eat whitespace */
 901             while (*look && ' ' == *look && *(look+1))
 902                 look++;
 903
 904             /* parse possible id */
 905             if (*look && 0 == strncmp((const char *)look, "id=", 3))
 906             {
 907                 look += 3;
 908                 bval = look;
 909                 while (*look && ' ' != *look)
 910                     look++;
 911                 eval = look;
 912                 strncpy((char *)id, (const char *)bval, eval - bval);
 913                 id[eval - bval] = '\0';
 914             }
 915
 916             /* eat whitespace */
 917             while (*look && ' ' == *look && *(look+1))
 918                 look++;
 919
 920             /* parse possible rank */
 921             if (*look && 0 == strncmp((const char *)look, "rank=", 5))
 922             {
 923                 look += 6;
 924                 bval = look;
 925                 while (*look && ' ' != *look)
 926                     look++;
 927                 eval = look;
 928                 strncpy((char *)rank, (const char *)bval, eval - bval);
 929                 rank[eval - bval] = '\0';
 930             }
 931
 932             /* eat whitespace */
 933             while (*look && ' ' == *look && *(look+1))
 934                 look++;
 935
 936             if (look && '\0' != *look)
 937             {
 938                 dom_log(YLOG_WARN, tinfo, node,
 939                         "content '%s', can not parse '%s'",
 940                         pi_p, look);
 941             }
 942             else
 943                 set_record_info(tinfo, extctr, id, rank, 0);
 944
 945         }
 946         /* parsing index instruction */
 947         else if (0 == strncmp((const char *)look, "index", 5))
 948         {
 949             look += 5;
 950
 951             /* eat whitespace */
 952             while (*look && ' ' == *look && *(look+1))
 953                 look++;
 954
 955             /* export index instructions to outside */
 956             *index_pp = look;
 957         }
 958         else
 959         {
 960             dom_log(YLOG_WARN, tinfo, node,
 961                     "content '%s', can not parse '%s'",
 962                     pi_p, look);
 963         }
 964     }
 965 }
 966
 967 /* DOM filter style indexing */
 968 static void process_xml_element_node(struct filter_info *tinfo,
 969                                      struct recExtractCtrl *extctr,
 970                                      RecWord* recword,
 971                                      xmlNodePtr node)
 972 {
 973     /* remember indexing instruction from PI to next element node */
 974     xmlChar *index_p = 0;
 975
 976     /* check if we are an element node in the special zebra namespace
 977        and either set record data or index value-of node content*/
 978     process_xml_element_zebra_node(tinfo, extctr, recword, node);
 979
 980     /* loop through kid nodes */
 981     for (node = node->children; node; node = node->next)
 982     {
 983         /* check and set PI record and index index instructions */
 984         if (node->type == XML_PI_NODE)
 985         {
 986             process_xml_pi_node(tinfo, extctr, node, &index_p);
 987         }
 988         else if (node->type == XML_ELEMENT_NODE)
 989         {
 990             /* if there was a PI index instruction before this element */
 991             if (index_p)
 992             {
 993                 index_value_of(tinfo, extctr, recword, node, index_p);
 994                 index_p = 0;
 995             }
 996             process_xml_element_node(tinfo, extctr, recword,node);
 997         }
 998         else
 999             continue;
1000     }
1001 }
1002
1003
1004 /* DOM filter style indexing */
1005 static void extract_dom_doc_node(struct filter_info *tinfo,
1006                                  struct recExtractCtrl *extctr,
1007                                  xmlDocPtr doc)
1008 {
1009     xmlChar *buf_out;
1010     int len_out;
1011
1012     /* only need to do the initialization once, reuse recword for all terms */
1013     RecWord recword;
1014     (*extctr->init)(extctr, &recword);
1015
1016     if (extctr->flagShowRecords)
1017     {
1018         xmlDocDumpMemory(doc, &buf_out, &len_out);
1019         fwrite(buf_out, len_out, 1, stdout);
1020         xmlFree(buf_out);
1021     }
1022
1023     process_xml_element_node(tinfo, extctr, &recword, (xmlNodePtr)doc);
1024 }
1025
1026
1027
1028
1029 static int convert_extract_doc(struct filter_info *tinfo,
1030                                struct filter_input *input,
1031                                struct recExtractCtrl *p,
1032                                xmlDocPtr doc)
1033
1034 {
1035     xmlChar *buf_out;
1036     int len_out;
1037     const char *params[10];
1038     xsltStylesheetPtr last_xsp = 0;
1039     xmlDocPtr store_doc = 0;
1040
1041     params[0] = 0;
1042     set_param_str(params, "schema", zebra_dom_ns, tinfo->odr_record);
1043
1044     /* input conversion */
1045     perform_convert(tinfo, input->convert, params, &doc, 0);
1046
1047     if (tinfo->store)
1048     {
1049         /* store conversion */
1050         store_doc = xmlCopyDoc(doc, 1);
1051         perform_convert(tinfo, tinfo->store->convert,
1052                         params, &store_doc, &last_xsp);
1053     }
1054
1055     if (last_xsp)
1056         xsltSaveResultToString(&buf_out, &len_out,
1057                                store_doc ? store_doc : doc, last_xsp);
1058     else
1059         xmlDocDumpMemory(store_doc ? store_doc : doc, &buf_out, &len_out);
1060     if (p->flagShowRecords)
1061         fwrite(buf_out, len_out, 1, stdout);
1062     (*p->setStoreData)(p, buf_out, len_out);
1063     xmlFree(buf_out);
1064
1065     if (store_doc)
1066         xmlFreeDoc(store_doc);
1067
1068     /* extract conversion */
1069     perform_convert(tinfo, tinfo->extract->convert, params, &doc, 0);
1070
1071     /* finally, do the indexing */
1072     if (doc)
1073     {
1074         extract_dom_doc_node(tinfo, p, doc);
1075         /* extract_doc_alvis(tinfo, p, doc); */
1076         xmlFreeDoc(doc);
1077     }
1078
1079     return RECCTRL_EXTRACT_OK;
1080 }
1081
1082 static int extract_xml_split(struct filter_info *tinfo,
1083                              struct filter_input *input,
1084                              struct recExtractCtrl *p)
1085 {
1086     int ret;
1087
1088     if (p->first_record)
1089     {
1090         if (input->u.xmlreader.reader)
1091             xmlFreeTextReader(input->u.xmlreader.reader);
1092         input->u.xmlreader.reader = xmlReaderForIO(ioread_ex, ioclose_ex,
1093                                                    p /* I/O handler */,
1094                                                    0 /* URL */,
1095                                                    0 /* encoding */,
1096                                                    XML_PARSE_XINCLUDE|
1097                                                    XML_PARSE_NOENT);
1098     }
1099     if (!input->u.xmlreader.reader)
1100         return RECCTRL_EXTRACT_ERROR_GENERIC;
1101
1102     ret = xmlTextReaderRead(input->u.xmlreader.reader);
1103     while (ret == 1)
1104     {
1105         int type = xmlTextReaderNodeType(input->u.xmlreader.reader);
1106         int depth = xmlTextReaderDepth(input->u.xmlreader.reader);
1107         if (type == XML_READER_TYPE_ELEMENT &&
1108             input->u.xmlreader.split_level == depth)
1109         {
1110             xmlNodePtr ptr
1111                 = xmlTextReaderExpand(input->u.xmlreader.reader);
1112             if (ptr)
1113             {
1114                 xmlNodePtr ptr2 = xmlCopyNode(ptr, 1);
1115                 xmlDocPtr doc = xmlNewDoc((const xmlChar*) "1.0");
1116
1117                 xmlDocSetRootElement(doc, ptr2);
1118
1119                 return convert_extract_doc(tinfo, input, p, doc);
1120             }
1121             else
1122             {
1123                 xmlFreeTextReader(input->u.xmlreader.reader);
1124                 input->u.xmlreader.reader = 0;
1125                 return RECCTRL_EXTRACT_ERROR_GENERIC;
1126             }
1127         }
1128         ret = xmlTextReaderRead(input->u.xmlreader.reader);
1129     }
1130     xmlFreeTextReader(input->u.xmlreader.reader);
1131     input->u.xmlreader.reader = 0;
1132     return RECCTRL_EXTRACT_EOF;
1133 }
1134
1135 static int extract_xml_full(struct filter_info *tinfo,
1136                             struct filter_input *input,
1137                             struct recExtractCtrl *p)
1138 {
1139     if (p->first_record) /* only one record per stream */
1140     {
1141         xmlDocPtr doc = xmlReadIO(ioread_ex, ioclose_ex,
1142                                   p /* I/O handler */,
1143                                   0 /* URL */,
1144                                   0 /* encoding */,
1145                                   XML_PARSE_XINCLUDE|XML_PARSE_NOENT);
1146         if (!doc)
1147         {
1148             return RECCTRL_EXTRACT_ERROR_GENERIC;
1149         }
1150         return convert_extract_doc(tinfo, input, p, doc);
1151     }
1152     else
1153         return RECCTRL_EXTRACT_EOF;
1154 }
1155
1156 static int extract_iso2709(struct filter_info *tinfo,
1157                            struct filter_input *input,
1158                            struct recExtractCtrl *p)
1159 {
1160     char buf[100000];
1161     int record_length;
1162     int read_bytes, r;
1163
1164     if (p->stream->readf(p->stream, buf, 5) != 5)
1165         return RECCTRL_EXTRACT_EOF;
1166     while (*buf < '0' || *buf > '9')
1167     {
1168         int i;
1169
1170         dom_log(YLOG_WARN, tinfo, 0,
1171                 "MARC: Skipping bad byte %d (0x%02X)",
1172                 *buf & 0xff, *buf & 0xff);
1173         for (i = 0; i<4; i++)
1174             buf[i] = buf[i+1];
1175
1176         if (p->stream->readf(p->stream, buf+4, 1) != 1)
1177             return RECCTRL_EXTRACT_EOF;
1178     }
1179     record_length = atoi_n (buf, 5);
1180     if (record_length < 25)
1181     {
1182         dom_log(YLOG_WARN, tinfo, 0,
1183                 "MARC record length < 25, is %d",  record_length);
1184         return RECCTRL_EXTRACT_ERROR_GENERIC;
1185     }
1186     read_bytes = p->stream->readf(p->stream, buf+5, record_length-5);
1187     if (read_bytes < record_length-5)
1188     {
1189         dom_log(YLOG_WARN, tinfo, 0,
1190                 "couldn't read whole MARC record");
1191         return RECCTRL_EXTRACT_ERROR_GENERIC;
1192     }
1193     r = yaz_marc_read_iso2709(input->u.marc.handle,  buf, record_length);
1194     if (r < record_length)
1195     {
1196         dom_log (YLOG_WARN, tinfo, 0,
1197                  "parsing of MARC record failed r=%d length=%d",
1198                  r, record_length);
1199         return RECCTRL_EXTRACT_ERROR_GENERIC;
1200     }
1201     else
1202     {
1203         xmlDocPtr rdoc;
1204         xmlNode *root_ptr;
1205         yaz_marc_write_xml(input->u.marc.handle, &root_ptr, 0, 0, 0);
1206         rdoc = xmlNewDoc((const xmlChar*) "1.0");
1207         xmlDocSetRootElement(rdoc, root_ptr);
1208         return convert_extract_doc(tinfo, input, p, rdoc);
1209     }
1210     return RECCTRL_EXTRACT_OK;
1211 }
1212
1213 static int filter_extract(void *clientData, struct recExtractCtrl *p)
1214 {
1215     struct filter_info *tinfo = clientData;
1216     struct filter_input *input = tinfo->input_list;
1217
1218     if (!input)
1219         return RECCTRL_EXTRACT_ERROR_GENERIC;
1220
1221     odr_reset(tinfo->odr_record);
1222     switch(input->type)
1223     {
1224     case DOM_INPUT_XMLREADER:
1225         if (input->u.xmlreader.split_level == 0)
1226             return extract_xml_full(tinfo, input, p);
1227         else
1228             return extract_xml_split(tinfo, input, p);
1229         break;
1230     case DOM_INPUT_MARC:
1231         return extract_iso2709(tinfo, input, p);
1232     }
1233     return RECCTRL_EXTRACT_ERROR_GENERIC;
1234 }
1235
1236 static int ioread_ret(void *context, char *buffer, int len)
1237 {
1238     struct recRetrieveCtrl *p = context;
1239     return p->stream->readf(p->stream, buffer, len);
1240 }
1241
1242 static int ioclose_ret(void *context)
1243 {
1244     return 0;
1245 }
1246
1247 static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p)
1248 {
1249     /* const char *esn = zebra_dom_ns; */
1250     const char *esn = 0;
1251     const char *params[32];
1252     struct filter_info *tinfo = clientData;
1253     xmlDocPtr doc;
1254     struct filter_retrieve *retrieve;
1255     xsltStylesheetPtr last_xsp = 0;
1256
1257     if (p->comp)
1258     {
1259         if (p->comp->which == Z_RecordComp_simple
1260             && p->comp->u.simple->which == Z_ElementSetNames_generic)
1261         {
1262             esn = p->comp->u.simple->u.generic;
1263         }
1264         else if (p->comp->which == Z_RecordComp_complex
1265                  && p->comp->u.complex->generic->elementSpec
1266                  && p->comp->u.complex->generic->elementSpec->which ==
1267                  Z_ElementSpec_elementSetName)
1268         {
1269             esn = p->comp->u.complex->generic->elementSpec->u.elementSetName;
1270         }
1271     }
1272     retrieve = lookup_retrieve(tinfo, esn);
1273     if (!retrieve)
1274     {
1275         p->diagnostic =
1276             YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
1277         return 0;
1278     }
1279
1280     params[0] = 0;
1281     set_param_int(params, "id", p->localno, p->odr);
1282     if (p->fname)
1283         set_param_str(params, "filename", p->fname, p->odr);
1284     if (p->staticrank >= 0)
1285         set_param_int(params, "rank", p->staticrank, p->odr);
1286
1287     if (esn)
1288         set_param_str(params, "schema", esn, p->odr);
1289     else
1290         if (retrieve->name)
1291             set_param_str(params, "schema", retrieve->name, p->odr);
1292         else if (retrieve->identifier)
1293             set_param_str(params, "schema", retrieve->identifier, p->odr);
1294         else
1295             set_param_str(params, "schema", "", p->odr);
1296
1297     if (p->score >= 0)
1298         set_param_int(params, "score", p->score, p->odr);
1299     set_param_int(params, "size", p->recordSize, p->odr);
1300
1301     doc = xmlReadIO(ioread_ret, ioclose_ret, p /* I/O handler */,
1302                     0 /* URL */,
1303                     0 /* encoding */,
1304                     XML_PARSE_XINCLUDE|XML_PARSE_NOENT);
1305     if (!doc)
1306     {
1307         p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1308         return 0;
1309     }
1310
1311     /* retrieve conversion */
1312     perform_convert(tinfo, retrieve->convert, params, &doc, &last_xsp);
1313     if (!doc)
1314     {
1315         p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1316     }
1317     else if (p->input_format == VAL_NONE || p->input_format == VAL_TEXT_XML)
1318     {
1319         xmlChar *buf_out;
1320         int len_out;
1321
1322         if (last_xsp)
1323             xsltSaveResultToString(&buf_out, &len_out, doc, last_xsp);
1324         else
1325             xmlDocDumpMemory(doc, &buf_out, &len_out);
1326
1327         p->output_format = VAL_TEXT_XML;
1328         p->rec_len = len_out;
1329         p->rec_buf = odr_malloc(p->odr, p->rec_len);
1330         memcpy(p->rec_buf, buf_out, p->rec_len);
1331         xmlFree(buf_out);
1332     }
1333     else if (p->output_format == VAL_SUTRS)
1334     {
1335         xmlChar *buf_out;
1336         int len_out;
1337
1338         if (last_xsp)
1339             xsltSaveResultToString(&buf_out, &len_out, doc, last_xsp);
1340         else
1341             xmlDocDumpMemory(doc, &buf_out, &len_out);
1342
1343         p->output_format = VAL_SUTRS;
1344         p->rec_len = len_out;
1345         p->rec_buf = odr_malloc(p->odr, p->rec_len);
1346         memcpy(p->rec_buf, buf_out, p->rec_len);
1347
1348         xmlFree(buf_out);
1349     }
1350     else
1351     {
1352         p->diagnostic = YAZ_BIB1_RECORD_SYNTAX_UNSUPP;
1353     }
1354     xmlFreeDoc(doc);
1355     return 0;
1356 }
1357
1358 static struct recType filter_type = {
1359     0,
1360     "dom",
1361     filter_init,
1362     filter_config,
1363     filter_destroy,
1364     filter_extract,
1365     filter_retrieve
1366 };
1367
1368 RecType
1369 #ifdef IDZEBRA_STATIC_DOM
1370 idzebra_filter_dom
1371 #else
1372 idzebra_filter
1373 #endif
1374
1375 [] = {
1376     &filter_type,
1377     0,
1378 };
1379 /*
1380  * Local variables:
1381  * c-basic-offset: 4
1382  * indent-tabs-mode: nil
1383  * End:
1384  * vim: shiftwidth=4 tabstop=8 expandtab
1385  */
1386