renamed record filter module sources
authorAdam Dickmeiss <adam@indexdata.dk>
Thu, 11 Sep 2008 13:57:59 +0000 (15:57 +0200)
committerAdam Dickmeiss <adam@indexdata.dk>
Thu, 11 Sep 2008 13:57:59 +0000 (15:57 +0200)
16 files changed:
index/Makefile.am
index/alvis.c [deleted file]
index/marcread.c [deleted file]
index/mod_alvis.c [new file with mode: 0644]
index/mod_grs_marc.c [new file with mode: 0644]
index/mod_grs_regx.c [new file with mode: 0644]
index/mod_grs_sgml.c [new file with mode: 0644]
index/mod_grs_xml.c [new file with mode: 0644]
index/mod_safari.c [new file with mode: 0644]
index/mod_text.c [new file with mode: 0644]
index/rectext.c [deleted file]
index/regxread.c [deleted file]
index/safari.c [deleted file]
index/sgmlread.c [deleted file]
index/xmlread.c [deleted file]
win/makefile

index 88e8bd1..688e112 100644 (file)
@@ -20,27 +20,27 @@ modlibdir=$(libdir)/$(PACKAGE)$(PACKAGE_SUFFIX)/modules
 tabdatadir = $(datadir)/$(PACKAGE)$(PACKAGE_SUFFIX)/tab
 
 # The shared modules 
-mod_grs_regx_la_SOURCES = regxread.c
+mod_grs_regx_la_SOURCES = mod_grs_regx.c
 mod_grs_regx_la_LDFLAGS = -rpath $(modlibdir) -module -avoid-version
 mod_grs_regx_la_LADD = $(TCL_LIB)
 mod_grs_regx_la_LIBADD = $(zebralib) $(mod_grs_regx_la_LADD)
 
-mod_grs_xml_la_SOURCES = xmlread.c
+mod_grs_xml_la_SOURCES = mod_grs_xml.c
 mod_grs_xml_la_LDFLAGS = -rpath $(modlibdir) -module -avoid-version
 mod_grs_xml_la_LADD = $(EXPAT_LIBS)
 mod_grs_xml_la_LIBADD = $(zebralib) $(mod_grs_xml_la_LADD)
 
-mod_grs_marc_la_SOURCES = marcread.c marcomp.c marcomp.h inline.c inline.h
+mod_grs_marc_la_SOURCES = mod_grs_marc.c marcomp.c marcomp.h inline.c inline.h
 mod_grs_marc_la_LDFLAGS = -rpath $(modlibdir) -module -avoid-version
 mod_grs_marc_la_LADD =
 mod_grs_marc_la_LIBADD = $(zebralib) $(mod_grs_marc_la_LADD)
 
-mod_safari_la_SOURCES = safari.c
+mod_safari_la_SOURCES = mod_safari.c
 mod_safari_la_LDFLAGS = -rpath $(modlibdir) -module -avoid-version
 mod_safari_la_LADD =
 mod_safari_la_LIBADD = $(zebralib) $(mod_safari_la_LADD)
 
-mod_alvis_la_SOURCES = alvis.c
+mod_alvis_la_SOURCES = mod_alvis.c
 mod_alvis_la_LDFLAGS = -rpath $(modlibdir) -module -avoid-version
 mod_alvis_la_LADD =
 mod_alvis_la_LIBADD = $(zebralib) $(mod_alvis_la_LADD)
@@ -50,7 +50,7 @@ mod_dom_la_LDFLAGS = -rpath $(modlibdir) -module -avoid-version
 mod_dom_la_LADD =
 mod_dom_la_LIBADD = $(zebralib) $(mod_dom_la_LADD)
 
-mod_text_la_SOURCES = rectext.c
+mod_text_la_SOURCES = mod_text.c
 mod_text_la_LDFLAGS = -rpath $(modlibdir) -module -avoid-version
 mod_text_la_LADD =
 mod_text_la_LIBADD = $(zebralib) $(mod_text_la_LADD)
@@ -82,7 +82,7 @@ libidzebra_2_0_la_LDFLAGS=-export-dynamic -version-info $(ZEBRALIBS_VERSION_INFO
 libidzebra_2_0_la_SOURCES = \
   recctrl.c \
   recgrs.c \
-  sgmlread.c \
+  mod_grs_sgml.c \
   attribute.c \
   compact.c \
   dir.c dirs.c \
diff --git a/index/alvis.c b/index/alvis.c
deleted file mode 100644 (file)
index a752890..0000000
+++ /dev/null
@@ -1,729 +0,0 @@
-/* This file is part of the Zebra server.
-   Copyright (C) 1995-2008 Index Data
-
-Zebra is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 2, or (at your option) any later
-version.
-
-Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-
-*/
-
-#include <stdio.h>
-#include <assert.h>
-#include <ctype.h>
-
-#include <yaz/diagbib1.h>
-#include <yaz/tpath.h>
-#include <yaz/oid_db.h>
-
-#include <libxml/xmlversion.h>
-#include <libxml/parser.h>
-#include <libxml/tree.h>
-#include <libxml/xmlIO.h>
-#include <libxml/xmlreader.h>
-#include <libxslt/transform.h>
-#include <libxslt/xsltutils.h>
-
-#if YAZ_HAVE_EXSLT
-#include <libexslt/exslt.h>
-#endif
-
-#include <idzebra/util.h>
-#include <idzebra/recctrl.h>
-
-struct filter_schema {
-    const char *name;
-    const char *identifier;
-    const char *stylesheet;
-    struct filter_schema *next;
-    const char *default_schema;
-    /* char default_schema; */
-    xsltStylesheetPtr stylesheet_xsp;
-};
-
-struct filter_info {
-    xmlDocPtr doc;
-    char *fname;
-    char *full_name;
-    const char *profile_path;
-    int split_level;
-    const char *split_path;
-    ODR odr;
-    struct filter_schema *schemas;
-    xmlTextReaderPtr reader;
-};
-
-#define ZEBRA_SCHEMA_XSLT_NS "http://indexdata.dk/zebra/xslt/1"
-
-#define XML_STRCMP(a,b)   strcmp((char*)a, b)
-#define XML_STRLEN(a) strlen((char*)a)
-
-static const char *zebra_xslt_ns = ZEBRA_SCHEMA_XSLT_NS;
-
-static void set_param_str(const char **params, const char *name,
-                         const char *value, ODR odr)
-{
-    char *quoted = odr_malloc(odr, 3 + strlen(value));
-    sprintf(quoted, "'%s'", value);
-    while (*params)
-       params++;
-    params[0] = name;
-    params[1] = quoted;
-    params[2] = 0;
-}
-
-static void set_param_int(const char **params, const char *name,
-                         zint value, ODR odr)
-{
-    char *quoted = odr_malloc(odr, 30); /* 25 digits enough for 2^64 */
-    while (*params)
-       params++;
-    sprintf(quoted, "'" ZINT_FORMAT "'", value);
-    params[0] = name;
-    params[1] = quoted;
-    params[2] = 0;
-}
-
-#define ENABLE_INPUT_CALLBACK 0
-
-#if ENABLE_INPUT_CALLBACK
-static int zebra_xmlInputMatchCallback (char const *filename)
-{
-    yaz_log(YLOG_LOG, "match %s", filename);
-    return 0;
-}
-
-static void * zebra_xmlInputOpenCallback (char const *filename)
-{
-    return 0;
-}
-
-static int zebra_xmlInputReadCallback (void * context, char * buffer, int len)
-{
-    return 0;
-}
-
-static int zebra_xmlInputCloseCallback (void * context)
-{
-    return 0;
-}
-#endif
-
-static void *filter_init(Res res, RecType recType)
-{
-    struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo));
-    tinfo->reader = 0;
-    tinfo->fname = 0;
-    tinfo->full_name = 0;
-    tinfo->profile_path = 0;
-    tinfo->split_level = 0;
-    tinfo->split_path = 0;
-    tinfo->odr = odr_createmem(ODR_ENCODE);
-    tinfo->doc = 0;
-    tinfo->schemas = 0;
-
-#if YAZ_HAVE_EXSLT
-    exsltRegisterAll(); 
-#endif
-
-#if ENABLE_INPUT_CALLBACK
-    xmlRegisterDefaultInputCallbacks();
-    xmlRegisterInputCallbacks(zebra_xmlInputMatchCallback,
-                             zebra_xmlInputOpenCallback,
-                             zebra_xmlInputReadCallback,
-                             zebra_xmlInputCloseCallback);
-#endif
-    return tinfo;
-}
-
-static int attr_content(struct _xmlAttr *attr, const char *name,
-                       const char **dst_content)
-{
-    if (!XML_STRCMP(attr->name, name) && attr->children 
-        && attr->children->type == XML_TEXT_NODE)
-    {
-       *dst_content = (const char *)(attr->children->content);
-       return 1;
-    }
-    return 0;
-}
-
-static void destroy_schemas(struct filter_info *tinfo)
-{
-    struct filter_schema *schema = tinfo->schemas;
-    while (schema)
-    {
-       struct filter_schema *schema_next = schema->next;
-       if (schema->stylesheet_xsp)
-           xsltFreeStylesheet(schema->stylesheet_xsp);
-       xfree(schema);
-       schema = schema_next;
-    }
-    tinfo->schemas = 0;
-    xfree(tinfo->fname);
-    if (tinfo->doc)
-       xmlFreeDoc(tinfo->doc);    
-    tinfo->doc = 0;
-}
-
-static ZEBRA_RES create_schemas(struct filter_info *tinfo, const char *fname)
-{
-    char tmp_full_name[1024];
-    xmlNodePtr ptr;
-    tinfo->fname = xstrdup(fname);
-    
-    if (yaz_filepath_resolve(tinfo->fname, tinfo->profile_path, 
-                             NULL, tmp_full_name))
-        tinfo->full_name = xstrdup(tmp_full_name);
-    else
-        tinfo->full_name = xstrdup(tinfo->fname);
-    
-    yaz_log(YLOG_LOG, "alvis filter: loading config file %s", tinfo->full_name);
-    
-    tinfo->doc = xmlParseFile(tinfo->full_name);
-    
-    if (!tinfo->doc)
-    {
-        yaz_log(YLOG_WARN, "alvis filter: could not parse config file %s", 
-                tinfo->full_name);
-        
-       return ZEBRA_FAIL;
-    }
-    
-    ptr = xmlDocGetRootElement(tinfo->doc);
-    if (!ptr || ptr->type != XML_ELEMENT_NODE 
-        || XML_STRCMP(ptr->name, "schemaInfo"))
-    {
-        yaz_log(YLOG_WARN, 
-                "alvis filter:  config file %s :" 
-                " expected root element <schemaInfo>", 
-                tinfo->full_name);  
-        return ZEBRA_FAIL;
-    }
-
-    for (ptr = ptr->children; ptr; ptr = ptr->next)
-    {
-       if (ptr->type != XML_ELEMENT_NODE)
-           continue;
-       if (!XML_STRCMP(ptr->name, "schema"))
-       {  
-           struct _xmlAttr *attr;
-           struct filter_schema *schema = xmalloc(sizeof(*schema));
-           schema->name = 0;
-           schema->identifier = 0;
-           schema->stylesheet = 0;
-           schema->default_schema = 0;
-           schema->next = tinfo->schemas;
-           schema->stylesheet_xsp = 0;
-           tinfo->schemas = schema;
-           for (attr = ptr->properties; attr; attr = attr->next)
-           {
-               attr_content(attr, "identifier", &schema->identifier);
-               attr_content(attr, "name", &schema->name);
-               attr_content(attr, "stylesheet", &schema->stylesheet);
-               attr_content(attr, "default", &schema->default_schema);
-           }
-            /*yaz_log(YLOG_LOG, "XSLT add %s %s %s", 
-              schema->name, schema->identifier, schema->stylesheet); */
-
-            /* find requested schema */
-
-           if (schema->stylesheet)
-            {
-                char tmp_xslt_full_name[1024];
-                if (!yaz_filepath_resolve(schema->stylesheet, tinfo->profile_path, 
-                                          NULL, tmp_xslt_full_name)) 
-                {
-                    yaz_log(YLOG_WARN, 
-                            "alvis filter: stylesheet %s not found in path %s",
-                            schema->stylesheet, tinfo->profile_path);
-                    return ZEBRA_FAIL;
-                }
-                schema->stylesheet_xsp 
-                    = xsltParseStylesheetFile((const xmlChar*) tmp_xslt_full_name);
-                if (!schema->stylesheet_xsp)
-                {
-                    yaz_log(YLOG_WARN, 
-                            "alvis filter: could not parse xslt stylesheet %s", 
-                            tmp_xslt_full_name);
-                    return ZEBRA_FAIL;
-                }
-            }
-       }
-       else if (!XML_STRCMP(ptr->name, "split"))
-       {
-           struct _xmlAttr *attr;
-           for (attr = ptr->properties; attr; attr = attr->next)
-           {
-                const char *split_level_str = 0;
-               attr_content(attr, "level", &split_level_str);
-                tinfo->split_level = 
-                    split_level_str ? atoi(split_level_str) : 0;
-           }
-       }
-       else
-       {
-           yaz_log(YLOG_WARN, "Bad element %s in %s", ptr->name, fname);
-           return ZEBRA_FAIL;
-       }
-    }
-    return ZEBRA_OK;
-}
-
-static struct filter_schema *lookup_schema(struct filter_info *tinfo,
-                                          const char *est)
-{
-    struct filter_schema *schema;
-
-    for (schema = tinfo->schemas; schema; schema = schema->next)
-    { 
-        /* find requested schema */
-       if (est) 
-       {    
-           if (schema->identifier && !strcmp(schema->identifier, est))
-                return schema;
-            
-           if (schema->name && !strcmp(schema->name, est))
-               return schema;
-       } 
-        /* or return default schema if defined */
-        else if (schema->default_schema)
-           return schema;
-    }
-
-    /* return first schema if no default schema defined */
-    if (tinfo->schemas)
-        return tinfo->schemas;
-    
-    return 0;
-}
-
-static ZEBRA_RES filter_config(void *clientData, Res res, const char *args)
-{
-    struct filter_info *tinfo = clientData;
-    if (!args || !*args)
-    {
-        yaz_log(YLOG_WARN, "alvis filter: need config file");
-        return ZEBRA_FAIL;
-    }
-
-    if (tinfo->fname && !strcmp(args, tinfo->fname))
-       return ZEBRA_OK;
-    
-    tinfo->profile_path = res_get(res, "profilePath");
-    yaz_log(YLOG_LOG, "alvis filter: profilePath %s", tinfo->profile_path);
-
-    destroy_schemas(tinfo);
-    return create_schemas(tinfo, args);
-}
-
-static void filter_destroy(void *clientData)
-{
-    struct filter_info *tinfo = clientData;
-    destroy_schemas(tinfo);
-    xfree(tinfo->full_name);
-    if (tinfo->reader)
-       xmlFreeTextReader(tinfo->reader);
-    odr_destroy(tinfo->odr);
-    xfree(tinfo);
-}
-
-static int ioread_ex(void *context, char *buffer, int len)
-{
-    struct recExtractCtrl *p = context;
-    return p->stream->readf(p->stream, buffer, len);
-}
-
-static int ioclose_ex(void *context)
-{
-    return 0;
-}
-
-static void index_cdata(struct filter_info *tinfo, struct recExtractCtrl *ctrl,
-                       xmlNodePtr ptr, RecWord *recWord)
-{
-    for(; ptr; ptr = ptr->next)
-    {
-       index_cdata(tinfo, ctrl, ptr->children, recWord);
-       if (ptr->type != XML_TEXT_NODE)
-           continue;
-       recWord->term_buf = (const char *)ptr->content;
-       recWord->term_len = XML_STRLEN(ptr->content);
-       (*ctrl->tokenAdd)(recWord);
-    }
-}
-
-static void index_node(struct filter_info *tinfo,  struct recExtractCtrl *ctrl,
-                      xmlNodePtr ptr, RecWord *recWord)
-{
-    for(; ptr; ptr = ptr->next)
-    {
-       index_node(tinfo, ctrl, ptr->children, recWord);
-       if (ptr->type != XML_ELEMENT_NODE || !ptr->ns ||
-           XML_STRCMP(ptr->ns->href, zebra_xslt_ns))
-           continue;
-       if (!XML_STRCMP(ptr->name, "index"))
-       {
-           const char *name_str = 0;
-           const char *type_str = 0;
-           const char *xpath_str = 0;
-           struct _xmlAttr *attr;
-           for (attr = ptr->properties; attr; attr = attr->next)
-           {
-               attr_content(attr, "name", &name_str);
-               attr_content(attr, "xpath", &xpath_str);
-               attr_content(attr, "type", &type_str);
-           }
-           if (name_str)
-           {
-               const char *prev_type = recWord->index_type; /* save default type */
-
-               if (type_str && *type_str)
-                   recWord->index_type = (const char *) type_str; /* type was given */
-               recWord->index_name = name_str;
-               index_cdata(tinfo, ctrl, ptr->children, recWord);
-
-               recWord->index_type = prev_type;     /* restore it again */
-           }
-       }
-    }
-}
-
-static void index_record(struct filter_info *tinfo,struct recExtractCtrl *ctrl,
-                        xmlNodePtr ptr, RecWord *recWord)
-{
-    const char *type_str = "update";
-
-    if (ptr && ptr->type == XML_ELEMENT_NODE && ptr->ns &&
-       !XML_STRCMP(ptr->ns->href, zebra_xslt_ns)
-       && !XML_STRCMP(ptr->name, "record"))
-    {
-       const char *id_str = 0;
-       const char *rank_str = 0;
-       struct _xmlAttr *attr;
-       for (attr = ptr->properties; attr; attr = attr->next)
-       {
-           attr_content(attr, "type", &type_str);
-           attr_content(attr, "id", &id_str);
-           attr_content(attr, "rank", &rank_str);
-       }
-       if (id_str)
-           sscanf(id_str, "%255s", ctrl->match_criteria);
-
-       if (rank_str)
-           ctrl->staticrank = atozint(rank_str);
-       ptr = ptr->children;
-    }
-
-    if (!strcmp("update", type_str))
-        index_node(tinfo, ctrl, ptr, recWord);
-    else if (!strcmp("delete", type_str))
-         yaz_log(YLOG_WARN, "alvis filter delete: to be implemented");
-    else
-         yaz_log(YLOG_WARN, "alvis filter: unknown record type '%s'", 
-                 type_str);
-}
-    
-static int extract_doc(struct filter_info *tinfo, struct recExtractCtrl *p,
-                      xmlDocPtr doc)
-{
-    RecWord recWord;
-    const char *params[10];
-    xmlChar *buf_out;
-    int len_out;
-
-    struct filter_schema *schema = lookup_schema(tinfo, zebra_xslt_ns);
-
-    params[0] = 0;
-    set_param_str(params, "schema", zebra_xslt_ns, tinfo->odr);
-
-    (*p->init)(p, &recWord);
-
-    if (schema && schema->stylesheet_xsp)
-    {
-       xmlNodePtr root_ptr;
-       xmlDocPtr resDoc = 
-           xsltApplyStylesheet(schema->stylesheet_xsp,
-                               doc, params);
-       if (p->flagShowRecords)
-       {
-           xmlDocDumpMemory(resDoc, &buf_out, &len_out);
-           fwrite(buf_out, len_out, 1, stdout);
-           xmlFree(buf_out);
-       }
-       root_ptr = xmlDocGetRootElement(resDoc);
-       if (root_ptr)
-           index_record(tinfo, p, root_ptr, &recWord);
-       else
-       {
-           yaz_log(YLOG_WARN, "No root for index XML record."
-                   " split_level=%d stylesheet=%s",
-                   tinfo->split_level, schema->stylesheet);
-       }
-       xmlFreeDoc(resDoc);
-    }
-    xmlDocDumpMemory(doc, &buf_out, &len_out);
-    if (p->flagShowRecords)
-       fwrite(buf_out, len_out, 1, stdout);
-    if (p->setStoreData)
-        (*p->setStoreData)(p, buf_out, len_out);
-    xmlFree(buf_out);
-    
-    xmlFreeDoc(doc);
-    return RECCTRL_EXTRACT_OK;
-}
-
-static int extract_split(struct filter_info *tinfo, struct recExtractCtrl *p)
-{
-    int ret;
-
-    if (p->first_record)
-    {
-       if (tinfo->reader)
-           xmlFreeTextReader(tinfo->reader);
-       tinfo->reader = xmlReaderForIO(ioread_ex, ioclose_ex,
-                                      p /* I/O handler */,
-                                      0 /* URL */, 
-                                      0 /* encoding */,
-                                      XML_PARSE_XINCLUDE
-                                       | XML_PARSE_NOENT
-                                       | XML_PARSE_NONET);
-    }
-    if (!tinfo->reader)
-       return RECCTRL_EXTRACT_ERROR_GENERIC;
-
-    ret = xmlTextReaderRead(tinfo->reader);
-    while (ret == 1)
-    {
-       int type = xmlTextReaderNodeType(tinfo->reader);
-       int depth = xmlTextReaderDepth(tinfo->reader);
-       if (type == XML_READER_TYPE_ELEMENT && tinfo->split_level == depth)
-       {
-           xmlNodePtr ptr = xmlTextReaderExpand(tinfo->reader);
-            if (ptr)
-            {
-                xmlNodePtr ptr2 = xmlCopyNode(ptr, 1);
-                xmlDocPtr doc = xmlNewDoc((const xmlChar*) "1.0");
-                
-                xmlDocSetRootElement(doc, ptr2);
-                
-                return extract_doc(tinfo, p, doc);
-            }
-            else
-            {
-                xmlFreeTextReader(tinfo->reader);
-                tinfo->reader = 0;
-                return RECCTRL_EXTRACT_ERROR_GENERIC;
-            }
-       }
-       ret = xmlTextReaderRead(tinfo->reader);
-    }
-    xmlFreeTextReader(tinfo->reader);
-    tinfo->reader = 0;
-    return RECCTRL_EXTRACT_EOF;
-}
-
-static int extract_full(struct filter_info *tinfo, struct recExtractCtrl *p)
-{
-    if (p->first_record) /* only one record per stream */
-    {
-       xmlDocPtr doc = xmlReadIO(ioread_ex, ioclose_ex, p /* I/O handler */,
-                                 0 /* URL */,
-                                 0 /* encoding */,
-                                 XML_PARSE_XINCLUDE
-                                 | XML_PARSE_NOENT
-                                 | XML_PARSE_NONET);
-       if (!doc)
-           return RECCTRL_EXTRACT_ERROR_GENERIC;
-       /* else {
-           xmlNodePtr root = xmlDocGetRootElement(doc);
-            if (!root)
-                return RECCTRL_EXTRACT_ERROR_GENERIC;
-                } */
-       
-       return extract_doc(tinfo, p, doc);
-    }
-    else
-       return RECCTRL_EXTRACT_EOF;
-}
-
-static int filter_extract(void *clientData, struct recExtractCtrl *p)
-{
-    struct filter_info *tinfo = clientData;
-
-    odr_reset(tinfo->odr);
-    if (tinfo->split_level == 0 || p->setStoreData == 0)
-        return extract_full(tinfo, p);
-    else
-        return extract_split(tinfo, p);
-}
-
-static int ioread_ret(void *context, char *buffer, int len)
-{
-    struct recRetrieveCtrl *p = context;
-    return p->stream->readf(p->stream, buffer, len);
-}
-
-static int ioclose_ret(void *context)
-{
-    return 0;
-}
-
-static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p)
-{
-    /* const char *esn = zebra_xslt_ns; */
-    const char *esn = 0;
-    const char *params[32];
-    struct filter_info *tinfo = clientData;
-    xmlDocPtr resDoc;
-    xmlDocPtr doc;
-    struct filter_schema *schema;
-
-    if (p->comp)
-    {
-       if (p->comp->which == Z_RecordComp_simple
-           && p->comp->u.simple->which == Z_ElementSetNames_generic)
-       {
-           esn = p->comp->u.simple->u.generic;
-       }
-       else if (p->comp->which == Z_RecordComp_complex 
-                && p->comp->u.complex->generic->elementSpec
-                && p->comp->u.complex->generic->elementSpec->which ==
-                Z_ElementSpec_elementSetName)
-       {
-           esn = p->comp->u.complex->generic->elementSpec->u.elementSetName;
-       }
-    }
-    schema = lookup_schema(tinfo, esn);
-    if (!schema)
-    {
-       p->diagnostic =
-           YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
-       return 0;
-    }
-
-    params[0] = 0;
-    set_param_int(params, "id", p->localno, p->odr);
-    if (p->fname)
-       set_param_str(params, "filename", p->fname, p->odr);
-    if (p->staticrank >= 0)
-       set_param_int(params, "rank", p->staticrank, p->odr);
-
-    if (esn)
-        set_param_str(params, "schema", esn, p->odr);
-    else
-        if (schema->name)
-            set_param_str(params, "schema", schema->name, p->odr);
-        else if (schema->identifier)
-            set_param_str(params, "schema", schema->identifier, p->odr);
-        else
-            set_param_str(params, "schema", "", p->odr);
-
-    if (p->score >= 0)
-       set_param_int(params, "score", p->score, p->odr);
-    set_param_int(params, "size", p->recordSize, p->odr);
-
-    doc = xmlReadIO(ioread_ret, ioclose_ret, p /* I/O handler */,
-                   0 /* URL */,
-                   0 /* encoding */,
-                   XML_PARSE_XINCLUDE | XML_PARSE_NOENT | XML_PARSE_NONET);
-    if (!doc)
-    {
-       p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
-       return 0;
-    }
-
-    if (!schema->stylesheet_xsp)
-       resDoc = doc;
-    else
-    {
-       resDoc = xsltApplyStylesheet(schema->stylesheet_xsp,
-                                    doc, params);
-       xmlFreeDoc(doc);
-    }
-    if (!resDoc)
-    {
-       p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
-    }
-    else if (!p->input_format 
-             || !oid_oidcmp(p->input_format, yaz_oid_recsyn_xml))
-    {
-       xmlChar *buf_out;
-       int len_out;
-
-        if (schema->stylesheet_xsp)
-            xsltSaveResultToString(&buf_out, &len_out, resDoc,
-                                   schema->stylesheet_xsp);    
-        else
-           xmlDocDumpMemory(resDoc, &buf_out, &len_out);            
-
-       p->output_format = yaz_oid_recsyn_xml;
-       p->rec_len = len_out;
-       p->rec_buf = odr_malloc(p->odr, p->rec_len);
-       memcpy(p->rec_buf, buf_out, p->rec_len);
-       xmlFree(buf_out);
-    }
-    else if (!oid_oidcmp(p->output_format, yaz_oid_recsyn_sutrs))
-    {
-       xmlChar *buf_out;
-       int len_out;
-
-        if (schema->stylesheet_xsp)
-            xsltSaveResultToString(&buf_out, &len_out, resDoc,
-                                   schema->stylesheet_xsp);
-        else
-           xmlDocDumpMemory(resDoc, &buf_out, &len_out);            
-
-       p->output_format = yaz_oid_recsyn_sutrs;
-       p->rec_len = len_out;
-       p->rec_buf = odr_malloc(p->odr, p->rec_len);
-       memcpy(p->rec_buf, buf_out, p->rec_len);
-       
-       xmlFree(buf_out);
-    }
-    else
-    {
-       p->diagnostic = YAZ_BIB1_RECORD_SYNTAX_UNSUPP;
-    }
-    xmlFreeDoc(resDoc);
-    return 0;
-}
-
-static struct recType filter_type = {
-    0,
-    "alvis",
-    filter_init,
-    filter_config,
-    filter_destroy,
-    filter_extract,
-    filter_retrieve
-};
-
-RecType
-#ifdef IDZEBRA_STATIC_ALVIS
-idzebra_filter_alvis
-#else
-idzebra_filter
-#endif
-
-[] = {
-    &filter_type,
-    0,
-};
-/*
- * Local variables:
- * c-basic-offset: 4
- * indent-tabs-mode: nil
- * End:
- * vim: shiftwidth=4 tabstop=8 expandtab
- */
-
diff --git a/index/marcread.c b/index/marcread.c
deleted file mode 100644 (file)
index 52b59e8..0000000
+++ /dev/null
@@ -1,914 +0,0 @@
-/* This file is part of the Zebra server.
-   Copyright (C) 1995-2008 Index Data
-
-Zebra is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 2, or (at your option) any later
-version.
-
-Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-
-*/
-
-#include <stdio.h>
-#include <ctype.h>
-#include <assert.h>
-
-#include <yaz/yaz-util.h>
-#include <yaz/marcdisp.h>
-#include <idzebra/recgrs.h>
-#include "marcomp.h"
-#include "inline.h"
-
-#define MARC_DEBUG 0
-#define MARCOMP_DEBUG 0
-
-struct marc_info {
-    char type[256];
-};
-
-static data1_node *grs_read_iso2709 (struct grs_read_info *p, int marc_xml)
-{
-    struct marc_info *mi = (struct marc_info*) p->clientData;
-    char buf[100000];
-    int entry_p;
-    int record_length;
-    int indicator_length;
-    int identifier_length;
-    int base_address;
-    int end_of_directory;
-    int length_data_entry;
-    int length_starting;
-    int length_implementation;
-    int read_bytes;
-#if MARC_DEBUG
-    FILE *outf = stdout;
-#endif
-    data1_node *res_root, *res_top;
-    char *absynName;
-    data1_marctab *marctab;
-
-    if (p->stream->readf(p->stream, buf, 5) != 5)
-        return NULL;
-    while (*buf < '0' || *buf > '9')
-    {
-       int i;
-
-       yaz_log(YLOG_WARN, "MARC: Skipping bad byte %d (0x%02X)",
-               *buf & 0xff, *buf & 0xff);
-       for (i = 0; i<4; i++)
-           buf[i] = buf[i+1];
-
-       if (p->stream->readf(p->stream, buf+4, 1) != 1)
-           return NULL;
-    }
-    record_length = atoi_n (buf, 5);
-    if (record_length < 25)
-    {
-        yaz_log (YLOG_WARN, "MARC record length < 25, is %d", record_length);
-        return NULL;
-    }
-
-    read_bytes = p->stream->readf(p->stream, buf+5, record_length-5);
-    if (read_bytes < record_length-5)
-    {
-        yaz_log (YLOG_WARN, "Couldn't read whole MARC record");
-        return NULL;
-    }
-    /* skip until we meet a record separator */
-    while (buf[record_length-1] != ISO2709_RS)
-    {
-        if (record_length > sizeof(buf)-2)
-            break;
-        read_bytes = p->stream->readf(p->stream, buf+record_length, 1);
-        if (read_bytes != 1)
-            break;
-        record_length++;
-    }
-    /* read one byte ahead to see if there is more ... */
-    read_bytes = p->stream->readf(p->stream, buf+record_length, 1);
-    if (read_bytes == 1)
-    {
-        off_t cur_offset = p->stream->tellf(p->stream);
-       if (p->stream->endf)
-        {
-            off_t end_offset = cur_offset - 1;
-           p->stream->endf(p->stream, &end_offset);
-        }
-    }
-
-    absynName = mi->type;
-    res_root = data1_mk_root (p->dh, p->mem, absynName);
-    if (!res_root)
-    {
-        yaz_log (YLOG_WARN, "cannot read MARC without an abstract syntax");
-        return 0;
-    }
-    if (marc_xml)
-    {
-       data1_node *lead;
-       const char *attr[] = { "xmlns", "http://www.loc.gov/MARC21/slim", 0};
-                        
-       res_top = data1_mk_tag (p->dh, p->mem, "record", attr, res_root);
-
-       lead = data1_mk_tag(p->dh, p->mem, "leader", 0, res_top);
-       data1_mk_text_n(p->dh, p->mem, buf, 24, lead);
-    }
-    else
-       res_top = data1_mk_tag (p->dh, p->mem, absynName, 0, res_root);
-
-    if ((marctab = data1_absyn_getmarctab(p->dh, res_root)))
-    {
-       memcpy(marctab->leader, buf, 24);
-        memcpy(marctab->implementation_codes, buf+6, 4);
-        marctab->implementation_codes[4] = '\0';
-        memcpy(marctab->user_systems, buf+17, 3);
-        marctab->user_systems[3] = '\0';
-    }
-
-    if (marctab && marctab->force_indicator_length >= 0)
-       indicator_length = marctab->force_indicator_length;
-    else
-       indicator_length = atoi_n (buf+10, 1);
-    if (marctab && marctab->force_identifier_length >= 0)
-       identifier_length = marctab->force_identifier_length;
-    else
-       identifier_length = atoi_n (buf+11, 1);
-    base_address = atoi_n (buf+12, 5);
-
-    length_data_entry = atoi_n (buf+20, 1);
-    length_starting = atoi_n (buf+21, 1);
-    length_implementation = atoi_n (buf+22, 1);
-
-    for (entry_p = 24; buf[entry_p] != ISO2709_FS; )
-    {
-        int l = 3 + length_data_entry + length_starting;
-        if (entry_p + l >= record_length)
-        {
-           yaz_log(YLOG_WARN, "MARC: Directory offset %d: end of record.",
-                   entry_p);
-           return 0;
-        }
-        /* check for digits in length info */
-        while (--l >= 3)
-            if (!isdigit(*(const unsigned char *) (buf + entry_p+l)))
-                break;
-        if (l >= 3)
-        {
-            /* not all digits, so stop directory scan */
-           yaz_log(YLOG_LOG, "MARC: Bad directory");
-            break;
-        }
-        entry_p += 3 + length_data_entry + length_starting;
-    }
-    end_of_directory = entry_p;
-    if (base_address != entry_p+1)
-    {
-       yaz_log(YLOG_WARN, "MARC: Base address does not follow directory");
-    }
-    for (entry_p = 24; entry_p != end_of_directory; )
-    {
-        int data_length;
-        int data_offset;
-        int end_offset;
-        int i, i0;
-        char tag[4];
-        data1_node *res;
-        data1_node *parent = res_top;
-
-        memcpy (tag, buf+entry_p, 3);
-        entry_p += 3;
-        tag[3] = '\0';
-
-       if (marc_xml)
-           res = parent;
-       else
-           res = data1_mk_tag_n (p->dh, p->mem, tag, 3, 0 /* attr */, parent);
-       
-#if MARC_DEBUG
-        fprintf (outf, "%s ", tag);
-#endif
-        data_length = atoi_n (buf+entry_p, length_data_entry);
-        entry_p += length_data_entry;
-        data_offset = atoi_n (buf+entry_p, length_starting);
-        entry_p += length_starting;
-        i = data_offset + base_address;
-        end_offset = i+data_length-1;
-
-       if (data_length <= 0 || data_offset < 0 || end_offset >= record_length)
-       {
-           yaz_log(YLOG_WARN, "MARC: Bad offsets in data. Skipping rest");
-           break;
-       }
-       
-        if (memcmp (tag, "00", 2) && indicator_length)
-        {
-            /* generate indicator node */
-           if (marc_xml)
-           {
-               const char *attr[10];
-               int j;
-
-               attr[0] = "tag";
-               attr[1] = tag;
-               attr[2] = 0;
-
-               res = data1_mk_tag(p->dh, p->mem, "datafield", attr, res);
-
-               for (j = 0; j<indicator_length; j++)
-               {
-                   char str1[18], str2[2];
-                   sprintf (str1, "ind%d", j+1);
-                   str2[0] = buf[i+j];
-                   str2[1] = '\0';
-
-                   attr[0] = str1;
-                   attr[1] = str2;
-                   
-                   data1_tag_add_attr (p->dh, p->mem, res, attr);
-               }
-           }
-           else
-           {
-#if MARC_DEBUG
-               int j;
-#endif
-               res = data1_mk_tag_n (p->dh, p->mem, 
-                                     buf+i, indicator_length, 0 /* attr */, res);
-#if MARC_DEBUG
-               for (j = 0; j<indicator_length; j++)
-                   fprintf (outf, "%c", buf[j+i]);
-#endif
-           }
-           i += indicator_length;
-        } 
-       else
-       {
-           if (marc_xml)
-           {
-               const char *attr[10];
-               
-               attr[0] = "tag";
-               attr[1] = tag;
-               attr[2] = 0;
-               
-               res = data1_mk_tag(p->dh, p->mem, "controlfield", attr, res);
-           }
-       }
-        parent = res;
-        /* traverse sub fields */
-        i0 = i;
-        while (buf[i] != ISO2709_RS && buf[i] != ISO2709_FS && i < end_offset)
-        {
-           if (memcmp (tag, "00", 2) && identifier_length)
-            {
-               data1_node *res;
-               if (marc_xml)
-               {
-                   int j;
-                   const char *attr[3];
-                   char code[10];
-                   
-                   for (j = 1; j<identifier_length && j < 9; j++)
-                       code[j-1] = buf[i+j];
-                   code[j-1] = 0;
-                   attr[0] = "code";
-                   attr[1] = code;
-                   attr[2] = 0;
-                   res = data1_mk_tag(p->dh, p->mem, "subfield",
-                                      attr, parent);
-               }
-               else
-               {
-                   res = data1_mk_tag_n (p->dh, p->mem,
-                                          buf+i+1, identifier_length-1, 
-                                          0 /* attr */, parent);
-               }
-#if MARC_DEBUG
-                fprintf (outf, " $"); 
-                for (j = 1; j<identifier_length; j++)
-                    fprintf (outf, "%c", buf[j+i]);
-                fprintf (outf, " ");
-#endif
-                i += identifier_length;
-                i0 = i;
-               while (buf[i] != ISO2709_RS && buf[i] != ISO2709_IDFS &&
-                    buf[i] != ISO2709_FS && i < end_offset)
-               {
-#if MARC_DEBUG
-                   fprintf (outf, "%c", buf[i]);
-#endif
-                   i++;
-               }
-               data1_mk_text_n (p->dh, p->mem, buf + i0, i - i0, res);
-               i0 = i;
-            }
-            else
-            {
-#if MARC_DEBUG
-                fprintf (outf, "%c", buf[i]);
-#endif
-                i++;
-            }
-        }
-        if (i > i0)
-       {
-            data1_mk_text_n (p->dh, p->mem, buf + i0, i - i0, parent);
-       }
-#if MARC_DEBUG
-        fprintf (outf, "\n");
-        if (i < end_offset)
-            fprintf (outf, "-- separator but not at end of field\n");
-        if (buf[i] != ISO2709_RS && buf[i] != ISO2709_FS)
-            fprintf (outf, "-- no separator at end of field\n");
-#endif
-    }
-    return res_root;
-}
-
-/*
- * Locate some data under this node. This routine should handle variants
- * prettily.
- */
-static char *get_data(data1_node *n, int *len)
-{
-    char *r;
-
-    while (n)
-    {
-        if (n->which == DATA1N_data)
-        {
-            int i;
-            *len = n->u.data.len;
-
-            for (i = 0; i<*len; i++)
-                if (!d1_isspace(n->u.data.data[i]))
-                    break;
-            while (*len && d1_isspace(n->u.data.data[*len - 1]))
-                (*len)--;
-            *len = *len - i;
-            if (*len > 0)
-                return n->u.data.data + i;
-        }
-        if (n->which == DATA1N_tag)
-            n = n->child;
-       else if (n->which == DATA1N_data)
-            n = n->next;
-       else
-            break;     
-    }
-    r = "";
-    *len = strlen(r);
-    return r;
-}
-
-static data1_node *lookup_subfield(data1_node *node, const char *name)
-{
-    data1_node *p;
-    
-    for (p=node; p; p=p->next)
-    {
-       if (!yaz_matchstr(p->u.tag.tag, name))
-           return p;
-    }
-    return 0;
-}
-
-static inline_subfield *lookup_inline_subfield(inline_subfield *pisf,
-                                              const char *name)
-{
-    inline_subfield *p;
-    
-    for (p=pisf; p; p=p->next)
-    {
-       if (!yaz_matchstr(p->name, name))
-           return p;
-    }
-    return 0;
-}
-
-static inline_subfield *cat_inline_subfield(mc_subfield *psf, WRBUF buf,
-                                           inline_subfield *pisf)
-{
-    mc_subfield *p;
-    
-    for (p = psf; p && pisf; p = p->next)
-    {
-       if (p->which == MC_SF)
-       {
-           inline_subfield *found = lookup_inline_subfield(pisf, p->name);
-           
-           if (found)
-           {
-               if (strcmp(p->prefix, "_"))
-               {
-                   wrbuf_puts(buf, " ");
-                   wrbuf_puts(buf, p->prefix);
-               }
-               if (p->interval.start == -1)
-               {
-                   wrbuf_puts(buf, found->data);
-               }
-               else
-               {
-                   wrbuf_write(buf, found->data+p->interval.start,
-                               p->interval.end-p->interval.start);
-                   wrbuf_puts(buf, "");
-               }
-               if (strcmp(p->suffix, "_"))
-               {
-                   wrbuf_puts(buf, p->suffix);
-                   wrbuf_puts(buf, " ");
-               }
-#if MARCOMP_DEBUG
-               yaz_log(YLOG_LOG, "cat_inline_subfield(): add subfield $%s", found->name);
-#endif         
-               pisf = found->next;
-           }
-       }
-       else if (p->which == MC_SFVARIANT)
-       {
-           inline_subfield *next;
-           
-           do {
-               next = cat_inline_subfield(p->u.child, buf, pisf);
-               if (next == pisf)
-                   break;
-               pisf = next;
-           } while (pisf);
-       }
-       else if (p->which == MC_SFGROUP)
-       {
-           mc_subfield *pp;
-           int found;
-           
-           for (pp = p->u.child, found = 0; pp; pp = pp->next)
-           {
-               if (!yaz_matchstr(pisf->name, p->name))
-               {
-                   found = 1;
-                   break;
-               }
-           }
-           if (found)
-           {
-               wrbuf_puts(buf, " (");
-               pisf = cat_inline_subfield(p->u.child, buf, pisf);
-               wrbuf_puts(buf, ") ");
-           }
-       }
-    }
-    return pisf; 
-}
-
-static void cat_inline_field(mc_field *pf, WRBUF buf, data1_node *subfield)
-{    
-    if (!pf || !subfield)
-       return;
-
-    for (;subfield;)
-    {
-       int len;
-       inline_field *pif=NULL;
-       data1_node *psubf;
-       
-       if (yaz_matchstr(subfield->u.tag.tag, "1"))
-       {
-           subfield = subfield->next;
-           continue;
-       }
-       
-       psubf = subfield;
-       pif = inline_mk_field();
-       do
-       {
-           int i;
-           if ((i=inline_parse(pif, psubf->u.tag.tag, get_data(psubf, &len)))<0)
-           {
-               yaz_log(YLOG_WARN, "inline subfield ($%s): parse error",
-                   psubf->u.tag.tag);
-               inline_destroy_field(pif);
-               return; 
-           }
-           psubf = psubf->next;
-       } while (psubf && yaz_matchstr(psubf->u.tag.tag, "1"));
-       
-       subfield = psubf;
-       
-       if (pif && !yaz_matchstr(pif->name, pf->name))
-       {
-           if (!pf->list && pif->list)
-           {
-               wrbuf_puts(buf, pif->list->data);
-           }
-           else
-           {
-               int ind1, ind2;
-
-               /*
-                   check indicators
-               */
-
-               ind1 = (pif->ind1[0] == ' ') ? '_':pif->ind1[0];
-               ind2 = (pif->ind2[0] == ' ') ? '_':pif->ind2[0];
-    
-               if (((pf->ind1[0] == '.') || (ind1 == pf->ind1[0])) &&
-                   ((pf->ind2[0] == '.') || (ind2 == pf->ind2[0])))
-               {
-                   cat_inline_subfield(pf->list, buf, pif->list);
-                   
-                   /*
-                       add separator for inline fields
-                   */
-                   if (wrbuf_len(buf))
-                   {
-                       wrbuf_puts(buf, "\n");
-                   }
-               }
-               else
-               {
-                   yaz_log(YLOG_WARN, "In-line field %s missed -- indicators do not match", pif->name);
-               }
-           }
-       }
-       inline_destroy_field(pif);
-    }
-#if MARCOMP_DEBUG    
-    yaz_log(YLOG_LOG, "cat_inline_field(): got buffer {%s}", buf);
-#endif
-}
-
-static data1_node *cat_subfield(mc_subfield *psf, WRBUF buf,
-                               data1_node *subfield)
-{
-    mc_subfield *p;
-    
-    for (p = psf; p && subfield; p = p->next)
-    {
-       if (p->which == MC_SF)
-       {
-           data1_node *found = lookup_subfield(subfield, p->name);
-           
-           if (found)
-           {
-               int len;
-               
-               if (strcmp(p->prefix, "_"))
-               {
-                   wrbuf_puts(buf, " ");
-                   wrbuf_puts(buf, p->prefix);
-               }
-               
-               if (p->u.in_line)
-               {
-                   cat_inline_field(p->u.in_line, buf, found);
-               }
-               else if (p->interval.start == -1)
-               {
-                   wrbuf_puts(buf, get_data(found, &len));
-               }
-               else
-               {
-                   wrbuf_write(buf, get_data(found, &len)+p->interval.start,
-                       p->interval.end-p->interval.start);
-                   wrbuf_puts(buf, "");
-               }
-               if (strcmp(p->suffix, "_"))
-               {
-                   wrbuf_puts(buf, p->suffix);
-                   wrbuf_puts(buf, " ");
-               }
-#if MARCOMP_DEBUG              
-               yaz_log(YLOG_LOG, "cat_subfield(): add subfield $%s", found->u.tag.tag);
-#endif         
-               subfield = found->next;
-           }
-       }
-       else if (p->which == MC_SFVARIANT)
-       {
-           data1_node *next;
-           do {
-               next = cat_subfield(p->u.child, buf, subfield);
-               if (next == subfield)
-                   break;
-               subfield = next;
-           } while (subfield);
-       }
-       else if (p->which == MC_SFGROUP)
-       {
-           mc_subfield *pp;
-           int found;
-           
-           for (pp = p->u.child, found = 0; pp; pp = pp->next)
-           {
-               if (!yaz_matchstr(subfield->u.tag.tag, pp->name))
-               {
-                   found = 1;
-                   break;
-               }
-           }
-           if (found)
-           {
-               wrbuf_puts(buf, " (");
-               subfield = cat_subfield(p->u.child, buf, subfield);
-               wrbuf_puts(buf, ") ");
-           }
-       }
-    }
-    return subfield;
-}
-
-static data1_node *cat_field(struct grs_read_info *p, mc_field *pf,
-                            WRBUF buf, data1_node *field)
-{
-    data1_node *subfield;
-    int ind1, ind2;
-    
-    if (!pf || !field)
-       return 0;
-
-    
-    if (yaz_matchstr(field->u.tag.tag, pf->name))
-       return field->next;
-
-    subfield = field->child;
-    
-    if (!subfield)
-       return field->next;
-
-    /*
-       check subfield without indicators
-    */
-    
-    if (!pf->list && subfield->which == DATA1N_data)
-    {
-       int len;
-       
-       if (pf->interval.start == -1)
-       {
-           wrbuf_puts(buf, get_data(field, &len));
-       }
-       else
-       {
-           wrbuf_write(buf, get_data(field, &len)+pf->interval.start,
-                       pf->interval.end-pf->interval.start);
-           wrbuf_puts(buf, "");
-       }
-#if MARCOMP_DEBUG
-        yaz_log(YLOG_LOG, "cat_field(): got buffer {%s}", buf);
-#endif
-       return field->next;
-    }
-    
-    /*
-       check indicators
-    */
-
-    ind1 = (subfield->u.tag.tag[0] == ' ') ? '_':subfield->u.tag.tag[0];
-    ind2 = (subfield->u.tag.tag[1] == ' ') ? '_':subfield->u.tag.tag[1];
-    
-    if (!(
-       ((pf->ind1[0] == '.') || (ind1 == pf->ind1[0])) &&
-       ((pf->ind2[0] == '.') || (ind2 == pf->ind2[0]))
-       ))
-    {
-#if MARCOMP_DEBUG
-       yaz_log(YLOG_WARN, "Field %s missed -- does not match indicators", field->u.tag.tag);
-#endif
-       return field->next;
-    }
-    
-    subfield = subfield->child;
-    
-    if (!subfield)
-       return field->next;
-
-    cat_subfield(pf->list, buf, subfield);
-
-#if MARCOMP_DEBUG    
-    yaz_log(YLOG_LOG, "cat_field(): got buffer {%s}", buf);
-#endif
-    
-    return field->next;    
-}
-
-static int is_empty(char *s)
-{
-    char *p = s;
-    
-    for (p = s; *p; p++)
-    {
-       if (!isspace(*(unsigned char *)p))
-           return 0;
-    }
-    return 1;
-}
-
-static void parse_data1_tree(struct grs_read_info *p, const char *mc_stmnt,
-                            data1_node *root)
-{
-    data1_marctab *marctab = data1_absyn_getmarctab(p->dh, root);
-    data1_node *top = root->child;
-    data1_node *field;
-    mc_context *c;
-    mc_field *pf;
-    WRBUF buf;
-    
-    c = mc_mk_context(mc_stmnt+3);
-    
-    if (!c)
-       return;
-       
-    pf = mc_getfield(c);
-    
-    if (!pf)
-    {
-       mc_destroy_context(c);
-       return;
-    }
-    buf = wrbuf_alloc();
-#if MARCOMP_DEBUG    
-    yaz_log(YLOG_LOG, "parse_data1_tree(): statement -{%s}", mc_stmnt);
-#endif
-    if (!yaz_matchstr(pf->name, "ldr"))
-    {
-       data1_node *new;
-#if MARCOMP_DEBUG
-       yaz_log(YLOG_LOG,"parse_data1_tree(): try LEADER from {%d} to {%d} positions",
-           pf->interval.start, pf->interval.end);
-#endif 
-        if (marctab)
-        {
-            new = data1_mk_tag_n(p->dh, p->mem, mc_stmnt, strlen(mc_stmnt), 0, top);
-            data1_mk_text_n(p->dh, p->mem, marctab->leader+pf->interval.start,
-                            pf->interval.end-pf->interval.start+1, new);
-        }
-    }
-    else
-    {
-       field=top->child;
-       
-       while(field)
-       {
-           if (!yaz_matchstr(field->u.tag.tag, pf->name))
-           {
-               data1_node *new;
-               char *pb;
-#if MARCOMP_DEBUG              
-               yaz_log(YLOG_LOG, "parse_data1_tree(): try field {%s}", field->u.tag.tag);
-#endif         
-               wrbuf_rewind(buf);
-               wrbuf_puts(buf, "");
-
-               field = cat_field(p, pf, buf, field);
-               
-               wrbuf_cstr(buf);
-                pb = wrbuf_buf(buf);
-               for (pb = strtok(pb, "\n"); pb; pb = strtok(NULL, "\n"))
-               {
-                    if (!is_empty(pb))
-                    {
-                        new = data1_mk_tag_n(p->dh, p->mem, mc_stmnt, strlen(mc_stmnt), 0, top);
-                        data1_mk_text_n(p->dh, p->mem, pb, strlen(pb), new);
-                    }
-               }
-           }
-           else
-           {
-               field = field->next;
-           }
-       }
-    }
-    mc_destroy_field(pf);
-    mc_destroy_context(c);
-    wrbuf_destroy(buf);
-}
-
-data1_node *grs_read_marcxml(struct grs_read_info *p)
-{
-    data1_node *root = grs_read_iso2709(p, 1);
-    data1_element *e;
-
-    if (!root)
-       return 0;
-       
-    for (e = data1_absyn_getelements(p->dh, root); e; e=e->next)
-    {
-       data1_tag *tag = e->tag;
-       
-       if (tag && tag->which == DATA1T_string &&
-           !yaz_matchstr(tag->value.string, "mc?"))
-               parse_data1_tree(p, tag->value.string, root);
-    }
-    return root;
-}
-
-data1_node *grs_read_marc(struct grs_read_info *p)
-{
-    data1_node *root = grs_read_iso2709(p, 0);
-    data1_element *e;
-
-    if (!root)
-       return 0;
-       
-    for (e = data1_absyn_getelements(p->dh, root); e; e=e->next)
-    {
-       data1_tag *tag = e->tag;
-       
-       if (tag && tag->which == DATA1T_string &&
-           !yaz_matchstr(tag->value.string, "mc?"))
-               parse_data1_tree(p, tag->value.string, root);
-    }
-    return root;
-}
-
-static void *init_marc(Res res, RecType rt)
-{
-    struct marc_info *p = xmalloc(sizeof(*p));
-    strcpy(p->type, "");
-    return p;
-}
-
-static ZEBRA_RES config_marc(void *clientData, Res res, const char *args)
-{
-    struct marc_info *p = (struct marc_info*) clientData;
-    if (strlen(args) < sizeof(p->type))
-       strcpy(p->type, args);
-    return ZEBRA_OK;
-}
-
-static void destroy_marc(void *clientData)
-{
-    struct marc_info *p = (struct marc_info*) clientData;
-    xfree (p);
-}
-
-
-static int extract_marc(void *clientData, struct recExtractCtrl *ctrl)
-{
-    return zebra_grs_extract(clientData, ctrl, grs_read_marc);
-}
-
-static int retrieve_marc(void *clientData, struct recRetrieveCtrl *ctrl)
-{
-    return zebra_grs_retrieve(clientData, ctrl, grs_read_marc);
-}
-
-static struct recType marc_type = {
-    0,
-    "grs.marc",
-    init_marc,
-    config_marc,
-    destroy_marc,
-    extract_marc,
-    retrieve_marc,
-};
-
-static int extract_marcxml(void *clientData, struct recExtractCtrl *ctrl)
-{
-    return zebra_grs_extract(clientData, ctrl, grs_read_marcxml);
-}
-
-static int retrieve_marcxml(void *clientData, struct recRetrieveCtrl *ctrl)
-{
-    return zebra_grs_retrieve(clientData, ctrl, grs_read_marcxml);
-}
-
-static struct recType marcxml_type = {
-    0,
-    "grs.marcxml",
-    init_marc,
-    config_marc,
-    destroy_marc,
-    extract_marcxml,
-    retrieve_marcxml,
-};
-
-RecType
-#ifdef IDZEBRA_STATIC_GRS_MARC
-idzebra_filter_grs_marc
-#else
-idzebra_filter
-#endif
-
-[] = {
-    &marc_type,
-    &marcxml_type,
-    0,
-};
-    
-/*
- * Local variables:
- * c-basic-offset: 4
- * indent-tabs-mode: nil
- * End:
- * vim: shiftwidth=4 tabstop=8 expandtab
- */
-
diff --git a/index/mod_alvis.c b/index/mod_alvis.c
new file mode 100644 (file)
index 0000000..a752890
--- /dev/null
@@ -0,0 +1,729 @@
+/* This file is part of the Zebra server.
+   Copyright (C) 1995-2008 Index Data
+
+Zebra is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2, or (at your option) any later
+version.
+
+Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+#include <stdio.h>
+#include <assert.h>
+#include <ctype.h>
+
+#include <yaz/diagbib1.h>
+#include <yaz/tpath.h>
+#include <yaz/oid_db.h>
+
+#include <libxml/xmlversion.h>
+#include <libxml/parser.h>
+#include <libxml/tree.h>
+#include <libxml/xmlIO.h>
+#include <libxml/xmlreader.h>
+#include <libxslt/transform.h>
+#include <libxslt/xsltutils.h>
+
+#if YAZ_HAVE_EXSLT
+#include <libexslt/exslt.h>
+#endif
+
+#include <idzebra/util.h>
+#include <idzebra/recctrl.h>
+
+struct filter_schema {
+    const char *name;
+    const char *identifier;
+    const char *stylesheet;
+    struct filter_schema *next;
+    const char *default_schema;
+    /* char default_schema; */
+    xsltStylesheetPtr stylesheet_xsp;
+};
+
+struct filter_info {
+    xmlDocPtr doc;
+    char *fname;
+    char *full_name;
+    const char *profile_path;
+    int split_level;
+    const char *split_path;
+    ODR odr;
+    struct filter_schema *schemas;
+    xmlTextReaderPtr reader;
+};
+
+#define ZEBRA_SCHEMA_XSLT_NS "http://indexdata.dk/zebra/xslt/1"
+
+#define XML_STRCMP(a,b)   strcmp((char*)a, b)
+#define XML_STRLEN(a) strlen((char*)a)
+
+static const char *zebra_xslt_ns = ZEBRA_SCHEMA_XSLT_NS;
+
+static void set_param_str(const char **params, const char *name,
+                         const char *value, ODR odr)
+{
+    char *quoted = odr_malloc(odr, 3 + strlen(value));
+    sprintf(quoted, "'%s'", value);
+    while (*params)
+       params++;
+    params[0] = name;
+    params[1] = quoted;
+    params[2] = 0;
+}
+
+static void set_param_int(const char **params, const char *name,
+                         zint value, ODR odr)
+{
+    char *quoted = odr_malloc(odr, 30); /* 25 digits enough for 2^64 */
+    while (*params)
+       params++;
+    sprintf(quoted, "'" ZINT_FORMAT "'", value);
+    params[0] = name;
+    params[1] = quoted;
+    params[2] = 0;
+}
+
+#define ENABLE_INPUT_CALLBACK 0
+
+#if ENABLE_INPUT_CALLBACK
+static int zebra_xmlInputMatchCallback (char const *filename)
+{
+    yaz_log(YLOG_LOG, "match %s", filename);
+    return 0;
+}
+
+static void * zebra_xmlInputOpenCallback (char const *filename)
+{
+    return 0;
+}
+
+static int zebra_xmlInputReadCallback (void * context, char * buffer, int len)
+{
+    return 0;
+}
+
+static int zebra_xmlInputCloseCallback (void * context)
+{
+    return 0;
+}
+#endif
+
+static void *filter_init(Res res, RecType recType)
+{
+    struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo));
+    tinfo->reader = 0;
+    tinfo->fname = 0;
+    tinfo->full_name = 0;
+    tinfo->profile_path = 0;
+    tinfo->split_level = 0;
+    tinfo->split_path = 0;
+    tinfo->odr = odr_createmem(ODR_ENCODE);
+    tinfo->doc = 0;
+    tinfo->schemas = 0;
+
+#if YAZ_HAVE_EXSLT
+    exsltRegisterAll(); 
+#endif
+
+#if ENABLE_INPUT_CALLBACK
+    xmlRegisterDefaultInputCallbacks();
+    xmlRegisterInputCallbacks(zebra_xmlInputMatchCallback,
+                             zebra_xmlInputOpenCallback,
+                             zebra_xmlInputReadCallback,
+                             zebra_xmlInputCloseCallback);
+#endif
+    return tinfo;
+}
+
+static int attr_content(struct _xmlAttr *attr, const char *name,
+                       const char **dst_content)
+{
+    if (!XML_STRCMP(attr->name, name) && attr->children 
+        && attr->children->type == XML_TEXT_NODE)
+    {
+       *dst_content = (const char *)(attr->children->content);
+       return 1;
+    }
+    return 0;
+}
+
+static void destroy_schemas(struct filter_info *tinfo)
+{
+    struct filter_schema *schema = tinfo->schemas;
+    while (schema)
+    {
+       struct filter_schema *schema_next = schema->next;
+       if (schema->stylesheet_xsp)
+           xsltFreeStylesheet(schema->stylesheet_xsp);
+       xfree(schema);
+       schema = schema_next;
+    }
+    tinfo->schemas = 0;
+    xfree(tinfo->fname);
+    if (tinfo->doc)
+       xmlFreeDoc(tinfo->doc);    
+    tinfo->doc = 0;
+}
+
+static ZEBRA_RES create_schemas(struct filter_info *tinfo, const char *fname)
+{
+    char tmp_full_name[1024];
+    xmlNodePtr ptr;
+    tinfo->fname = xstrdup(fname);
+    
+    if (yaz_filepath_resolve(tinfo->fname, tinfo->profile_path, 
+                             NULL, tmp_full_name))
+        tinfo->full_name = xstrdup(tmp_full_name);
+    else
+        tinfo->full_name = xstrdup(tinfo->fname);
+    
+    yaz_log(YLOG_LOG, "alvis filter: loading config file %s", tinfo->full_name);
+    
+    tinfo->doc = xmlParseFile(tinfo->full_name);
+    
+    if (!tinfo->doc)
+    {
+        yaz_log(YLOG_WARN, "alvis filter: could not parse config file %s", 
+                tinfo->full_name);
+        
+       return ZEBRA_FAIL;
+    }
+    
+    ptr = xmlDocGetRootElement(tinfo->doc);
+    if (!ptr || ptr->type != XML_ELEMENT_NODE 
+        || XML_STRCMP(ptr->name, "schemaInfo"))
+    {
+        yaz_log(YLOG_WARN, 
+                "alvis filter:  config file %s :" 
+                " expected root element <schemaInfo>", 
+                tinfo->full_name);  
+        return ZEBRA_FAIL;
+    }
+
+    for (ptr = ptr->children; ptr; ptr = ptr->next)
+    {
+       if (ptr->type != XML_ELEMENT_NODE)
+           continue;
+       if (!XML_STRCMP(ptr->name, "schema"))
+       {  
+           struct _xmlAttr *attr;
+           struct filter_schema *schema = xmalloc(sizeof(*schema));
+           schema->name = 0;
+           schema->identifier = 0;
+           schema->stylesheet = 0;
+           schema->default_schema = 0;
+           schema->next = tinfo->schemas;
+           schema->stylesheet_xsp = 0;
+           tinfo->schemas = schema;
+           for (attr = ptr->properties; attr; attr = attr->next)
+           {
+               attr_content(attr, "identifier", &schema->identifier);
+               attr_content(attr, "name", &schema->name);
+               attr_content(attr, "stylesheet", &schema->stylesheet);
+               attr_content(attr, "default", &schema->default_schema);
+           }
+            /*yaz_log(YLOG_LOG, "XSLT add %s %s %s", 
+              schema->name, schema->identifier, schema->stylesheet); */
+
+            /* find requested schema */
+
+           if (schema->stylesheet)
+            {
+                char tmp_xslt_full_name[1024];
+                if (!yaz_filepath_resolve(schema->stylesheet, tinfo->profile_path, 
+                                          NULL, tmp_xslt_full_name)) 
+                {
+                    yaz_log(YLOG_WARN, 
+                            "alvis filter: stylesheet %s not found in path %s",
+                            schema->stylesheet, tinfo->profile_path);
+                    return ZEBRA_FAIL;
+                }
+                schema->stylesheet_xsp 
+                    = xsltParseStylesheetFile((const xmlChar*) tmp_xslt_full_name);
+                if (!schema->stylesheet_xsp)
+                {
+                    yaz_log(YLOG_WARN, 
+                            "alvis filter: could not parse xslt stylesheet %s", 
+                            tmp_xslt_full_name);
+                    return ZEBRA_FAIL;
+                }
+            }
+       }
+       else if (!XML_STRCMP(ptr->name, "split"))
+       {
+           struct _xmlAttr *attr;
+           for (attr = ptr->properties; attr; attr = attr->next)
+           {
+                const char *split_level_str = 0;
+               attr_content(attr, "level", &split_level_str);
+                tinfo->split_level = 
+                    split_level_str ? atoi(split_level_str) : 0;
+           }
+       }
+       else
+       {
+           yaz_log(YLOG_WARN, "Bad element %s in %s", ptr->name, fname);
+           return ZEBRA_FAIL;
+       }
+    }
+    return ZEBRA_OK;
+}
+
+static struct filter_schema *lookup_schema(struct filter_info *tinfo,
+                                          const char *est)
+{
+    struct filter_schema *schema;
+
+    for (schema = tinfo->schemas; schema; schema = schema->next)
+    { 
+        /* find requested schema */
+       if (est) 
+       {    
+           if (schema->identifier && !strcmp(schema->identifier, est))
+                return schema;
+            
+           if (schema->name && !strcmp(schema->name, est))
+               return schema;
+       } 
+        /* or return default schema if defined */
+        else if (schema->default_schema)
+           return schema;
+    }
+
+    /* return first schema if no default schema defined */
+    if (tinfo->schemas)
+        return tinfo->schemas;
+    
+    return 0;
+}
+
+static ZEBRA_RES filter_config(void *clientData, Res res, const char *args)
+{
+    struct filter_info *tinfo = clientData;
+    if (!args || !*args)
+    {
+        yaz_log(YLOG_WARN, "alvis filter: need config file");
+        return ZEBRA_FAIL;
+    }
+
+    if (tinfo->fname && !strcmp(args, tinfo->fname))
+       return ZEBRA_OK;
+    
+    tinfo->profile_path = res_get(res, "profilePath");
+    yaz_log(YLOG_LOG, "alvis filter: profilePath %s", tinfo->profile_path);
+
+    destroy_schemas(tinfo);
+    return create_schemas(tinfo, args);
+}
+
+static void filter_destroy(void *clientData)
+{
+    struct filter_info *tinfo = clientData;
+    destroy_schemas(tinfo);
+    xfree(tinfo->full_name);
+    if (tinfo->reader)
+       xmlFreeTextReader(tinfo->reader);
+    odr_destroy(tinfo->odr);
+    xfree(tinfo);
+}
+
+static int ioread_ex(void *context, char *buffer, int len)
+{
+    struct recExtractCtrl *p = context;
+    return p->stream->readf(p->stream, buffer, len);
+}
+
+static int ioclose_ex(void *context)
+{
+    return 0;
+}
+
+static void index_cdata(struct filter_info *tinfo, struct recExtractCtrl *ctrl,
+                       xmlNodePtr ptr, RecWord *recWord)
+{
+    for(; ptr; ptr = ptr->next)
+    {
+       index_cdata(tinfo, ctrl, ptr->children, recWord);
+       if (ptr->type != XML_TEXT_NODE)
+           continue;
+       recWord->term_buf = (const char *)ptr->content;
+       recWord->term_len = XML_STRLEN(ptr->content);
+       (*ctrl->tokenAdd)(recWord);
+    }
+}
+
+static void index_node(struct filter_info *tinfo,  struct recExtractCtrl *ctrl,
+                      xmlNodePtr ptr, RecWord *recWord)
+{
+    for(; ptr; ptr = ptr->next)
+    {
+       index_node(tinfo, ctrl, ptr->children, recWord);
+       if (ptr->type != XML_ELEMENT_NODE || !ptr->ns ||
+           XML_STRCMP(ptr->ns->href, zebra_xslt_ns))
+           continue;
+       if (!XML_STRCMP(ptr->name, "index"))
+       {
+           const char *name_str = 0;
+           const char *type_str = 0;
+           const char *xpath_str = 0;
+           struct _xmlAttr *attr;
+           for (attr = ptr->properties; attr; attr = attr->next)
+           {
+               attr_content(attr, "name", &name_str);
+               attr_content(attr, "xpath", &xpath_str);
+               attr_content(attr, "type", &type_str);
+           }
+           if (name_str)
+           {
+               const char *prev_type = recWord->index_type; /* save default type */
+
+               if (type_str && *type_str)
+                   recWord->index_type = (const char *) type_str; /* type was given */
+               recWord->index_name = name_str;
+               index_cdata(tinfo, ctrl, ptr->children, recWord);
+
+               recWord->index_type = prev_type;     /* restore it again */
+           }
+       }
+    }
+}
+
+static void index_record(struct filter_info *tinfo,struct recExtractCtrl *ctrl,
+                        xmlNodePtr ptr, RecWord *recWord)
+{
+    const char *type_str = "update";
+
+    if (ptr && ptr->type == XML_ELEMENT_NODE && ptr->ns &&
+       !XML_STRCMP(ptr->ns->href, zebra_xslt_ns)
+       && !XML_STRCMP(ptr->name, "record"))
+    {
+       const char *id_str = 0;
+       const char *rank_str = 0;
+       struct _xmlAttr *attr;
+       for (attr = ptr->properties; attr; attr = attr->next)
+       {
+           attr_content(attr, "type", &type_str);
+           attr_content(attr, "id", &id_str);
+           attr_content(attr, "rank", &rank_str);
+       }
+       if (id_str)
+           sscanf(id_str, "%255s", ctrl->match_criteria);
+
+       if (rank_str)
+           ctrl->staticrank = atozint(rank_str);
+       ptr = ptr->children;
+    }
+
+    if (!strcmp("update", type_str))
+        index_node(tinfo, ctrl, ptr, recWord);
+    else if (!strcmp("delete", type_str))
+         yaz_log(YLOG_WARN, "alvis filter delete: to be implemented");
+    else
+         yaz_log(YLOG_WARN, "alvis filter: unknown record type '%s'", 
+                 type_str);
+}
+    
+static int extract_doc(struct filter_info *tinfo, struct recExtractCtrl *p,
+                      xmlDocPtr doc)
+{
+    RecWord recWord;
+    const char *params[10];
+    xmlChar *buf_out;
+    int len_out;
+
+    struct filter_schema *schema = lookup_schema(tinfo, zebra_xslt_ns);
+
+    params[0] = 0;
+    set_param_str(params, "schema", zebra_xslt_ns, tinfo->odr);
+
+    (*p->init)(p, &recWord);
+
+    if (schema && schema->stylesheet_xsp)
+    {
+       xmlNodePtr root_ptr;
+       xmlDocPtr resDoc = 
+           xsltApplyStylesheet(schema->stylesheet_xsp,
+                               doc, params);
+       if (p->flagShowRecords)
+       {
+           xmlDocDumpMemory(resDoc, &buf_out, &len_out);
+           fwrite(buf_out, len_out, 1, stdout);
+           xmlFree(buf_out);
+       }
+       root_ptr = xmlDocGetRootElement(resDoc);
+       if (root_ptr)
+           index_record(tinfo, p, root_ptr, &recWord);
+       else
+       {
+           yaz_log(YLOG_WARN, "No root for index XML record."
+                   " split_level=%d stylesheet=%s",
+                   tinfo->split_level, schema->stylesheet);
+       }
+       xmlFreeDoc(resDoc);
+    }
+    xmlDocDumpMemory(doc, &buf_out, &len_out);
+    if (p->flagShowRecords)
+       fwrite(buf_out, len_out, 1, stdout);
+    if (p->setStoreData)
+        (*p->setStoreData)(p, buf_out, len_out);
+    xmlFree(buf_out);
+    
+    xmlFreeDoc(doc);
+    return RECCTRL_EXTRACT_OK;
+}
+
+static int extract_split(struct filter_info *tinfo, struct recExtractCtrl *p)
+{
+    int ret;
+
+    if (p->first_record)
+    {
+       if (tinfo->reader)
+           xmlFreeTextReader(tinfo->reader);
+       tinfo->reader = xmlReaderForIO(ioread_ex, ioclose_ex,
+                                      p /* I/O handler */,
+                                      0 /* URL */, 
+                                      0 /* encoding */,
+                                      XML_PARSE_XINCLUDE
+                                       | XML_PARSE_NOENT
+                                       | XML_PARSE_NONET);
+    }
+    if (!tinfo->reader)
+       return RECCTRL_EXTRACT_ERROR_GENERIC;
+
+    ret = xmlTextReaderRead(tinfo->reader);
+    while (ret == 1)
+    {
+       int type = xmlTextReaderNodeType(tinfo->reader);
+       int depth = xmlTextReaderDepth(tinfo->reader);
+       if (type == XML_READER_TYPE_ELEMENT && tinfo->split_level == depth)
+       {
+           xmlNodePtr ptr = xmlTextReaderExpand(tinfo->reader);
+            if (ptr)
+            {
+                xmlNodePtr ptr2 = xmlCopyNode(ptr, 1);
+                xmlDocPtr doc = xmlNewDoc((const xmlChar*) "1.0");
+                
+                xmlDocSetRootElement(doc, ptr2);
+                
+                return extract_doc(tinfo, p, doc);
+            }
+            else
+            {
+                xmlFreeTextReader(tinfo->reader);
+                tinfo->reader = 0;
+                return RECCTRL_EXTRACT_ERROR_GENERIC;
+            }
+       }
+       ret = xmlTextReaderRead(tinfo->reader);
+    }
+    xmlFreeTextReader(tinfo->reader);
+    tinfo->reader = 0;
+    return RECCTRL_EXTRACT_EOF;
+}
+
+static int extract_full(struct filter_info *tinfo, struct recExtractCtrl *p)
+{
+    if (p->first_record) /* only one record per stream */
+    {
+       xmlDocPtr doc = xmlReadIO(ioread_ex, ioclose_ex, p /* I/O handler */,
+                                 0 /* URL */,
+                                 0 /* encoding */,
+                                 XML_PARSE_XINCLUDE
+                                 | XML_PARSE_NOENT
+                                 | XML_PARSE_NONET);
+       if (!doc)
+           return RECCTRL_EXTRACT_ERROR_GENERIC;
+       /* else {
+           xmlNodePtr root = xmlDocGetRootElement(doc);
+            if (!root)
+                return RECCTRL_EXTRACT_ERROR_GENERIC;
+                } */
+       
+       return extract_doc(tinfo, p, doc);
+    }
+    else
+       return RECCTRL_EXTRACT_EOF;
+}
+
+static int filter_extract(void *clientData, struct recExtractCtrl *p)
+{
+    struct filter_info *tinfo = clientData;
+
+    odr_reset(tinfo->odr);
+    if (tinfo->split_level == 0 || p->setStoreData == 0)
+        return extract_full(tinfo, p);
+    else
+        return extract_split(tinfo, p);
+}
+
+static int ioread_ret(void *context, char *buffer, int len)
+{
+    struct recRetrieveCtrl *p = context;
+    return p->stream->readf(p->stream, buffer, len);
+}
+
+static int ioclose_ret(void *context)
+{
+    return 0;
+}
+
+static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p)
+{
+    /* const char *esn = zebra_xslt_ns; */
+    const char *esn = 0;
+    const char *params[32];
+    struct filter_info *tinfo = clientData;
+    xmlDocPtr resDoc;
+    xmlDocPtr doc;
+    struct filter_schema *schema;
+
+    if (p->comp)
+    {
+       if (p->comp->which == Z_RecordComp_simple
+           && p->comp->u.simple->which == Z_ElementSetNames_generic)
+       {
+           esn = p->comp->u.simple->u.generic;
+       }
+       else if (p->comp->which == Z_RecordComp_complex 
+                && p->comp->u.complex->generic->elementSpec
+                && p->comp->u.complex->generic->elementSpec->which ==
+                Z_ElementSpec_elementSetName)
+       {
+           esn = p->comp->u.complex->generic->elementSpec->u.elementSetName;
+       }
+    }
+    schema = lookup_schema(tinfo, esn);
+    if (!schema)
+    {
+       p->diagnostic =
+           YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
+       return 0;
+    }
+
+    params[0] = 0;
+    set_param_int(params, "id", p->localno, p->odr);
+    if (p->fname)
+       set_param_str(params, "filename", p->fname, p->odr);
+    if (p->staticrank >= 0)
+       set_param_int(params, "rank", p->staticrank, p->odr);
+
+    if (esn)
+        set_param_str(params, "schema", esn, p->odr);
+    else
+        if (schema->name)
+            set_param_str(params, "schema", schema->name, p->odr);
+        else if (schema->identifier)
+            set_param_str(params, "schema", schema->identifier, p->odr);
+        else
+            set_param_str(params, "schema", "", p->odr);
+
+    if (p->score >= 0)
+       set_param_int(params, "score", p->score, p->odr);
+    set_param_int(params, "size", p->recordSize, p->odr);
+
+    doc = xmlReadIO(ioread_ret, ioclose_ret, p /* I/O handler */,
+                   0 /* URL */,
+                   0 /* encoding */,
+                   XML_PARSE_XINCLUDE | XML_PARSE_NOENT | XML_PARSE_NONET);
+    if (!doc)
+    {
+       p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
+       return 0;
+    }
+
+    if (!schema->stylesheet_xsp)
+       resDoc = doc;
+    else
+    {
+       resDoc = xsltApplyStylesheet(schema->stylesheet_xsp,
+                                    doc, params);
+       xmlFreeDoc(doc);
+    }
+    if (!resDoc)
+    {
+       p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
+    }
+    else if (!p->input_format 
+             || !oid_oidcmp(p->input_format, yaz_oid_recsyn_xml))
+    {
+       xmlChar *buf_out;
+       int len_out;
+
+        if (schema->stylesheet_xsp)
+            xsltSaveResultToString(&buf_out, &len_out, resDoc,
+                                   schema->stylesheet_xsp);    
+        else
+           xmlDocDumpMemory(resDoc, &buf_out, &len_out);            
+
+       p->output_format = yaz_oid_recsyn_xml;
+       p->rec_len = len_out;
+       p->rec_buf = odr_malloc(p->odr, p->rec_len);
+       memcpy(p->rec_buf, buf_out, p->rec_len);
+       xmlFree(buf_out);
+    }
+    else if (!oid_oidcmp(p->output_format, yaz_oid_recsyn_sutrs))
+    {
+       xmlChar *buf_out;
+       int len_out;
+
+        if (schema->stylesheet_xsp)
+            xsltSaveResultToString(&buf_out, &len_out, resDoc,
+                                   schema->stylesheet_xsp);
+        else
+           xmlDocDumpMemory(resDoc, &buf_out, &len_out);            
+
+       p->output_format = yaz_oid_recsyn_sutrs;
+       p->rec_len = len_out;
+       p->rec_buf = odr_malloc(p->odr, p->rec_len);
+       memcpy(p->rec_buf, buf_out, p->rec_len);
+       
+       xmlFree(buf_out);
+    }
+    else
+    {
+       p->diagnostic = YAZ_BIB1_RECORD_SYNTAX_UNSUPP;
+    }
+    xmlFreeDoc(resDoc);
+    return 0;
+}
+
+static struct recType filter_type = {
+    0,
+    "alvis",
+    filter_init,
+    filter_config,
+    filter_destroy,
+    filter_extract,
+    filter_retrieve
+};
+
+RecType
+#ifdef IDZEBRA_STATIC_ALVIS
+idzebra_filter_alvis
+#else
+idzebra_filter
+#endif
+
+[] = {
+    &filter_type,
+    0,
+};
+/*
+ * Local variables:
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ * vim: shiftwidth=4 tabstop=8 expandtab
+ */
+
diff --git a/index/mod_grs_marc.c b/index/mod_grs_marc.c
new file mode 100644 (file)
index 0000000..52b59e8
--- /dev/null
@@ -0,0 +1,914 @@
+/* This file is part of the Zebra server.
+   Copyright (C) 1995-2008 Index Data
+
+Zebra is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2, or (at your option) any later
+version.
+
+Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+#include <stdio.h>
+#include <ctype.h>
+#include <assert.h>
+
+#include <yaz/yaz-util.h>
+#include <yaz/marcdisp.h>
+#include <idzebra/recgrs.h>
+#include "marcomp.h"
+#include "inline.h"
+
+#define MARC_DEBUG 0
+#define MARCOMP_DEBUG 0
+
+struct marc_info {
+    char type[256];
+};
+
+static data1_node *grs_read_iso2709 (struct grs_read_info *p, int marc_xml)
+{
+    struct marc_info *mi = (struct marc_info*) p->clientData;
+    char buf[100000];
+    int entry_p;
+    int record_length;
+    int indicator_length;
+    int identifier_length;
+    int base_address;
+    int end_of_directory;
+    int length_data_entry;
+    int length_starting;
+    int length_implementation;
+    int read_bytes;
+#if MARC_DEBUG
+    FILE *outf = stdout;
+#endif
+    data1_node *res_root, *res_top;
+    char *absynName;
+    data1_marctab *marctab;
+
+    if (p->stream->readf(p->stream, buf, 5) != 5)
+        return NULL;
+    while (*buf < '0' || *buf > '9')
+    {
+       int i;
+
+       yaz_log(YLOG_WARN, "MARC: Skipping bad byte %d (0x%02X)",
+               *buf & 0xff, *buf & 0xff);
+       for (i = 0; i<4; i++)
+           buf[i] = buf[i+1];
+
+       if (p->stream->readf(p->stream, buf+4, 1) != 1)
+           return NULL;
+    }
+    record_length = atoi_n (buf, 5);
+    if (record_length < 25)
+    {
+        yaz_log (YLOG_WARN, "MARC record length < 25, is %d", record_length);
+        return NULL;
+    }
+
+    read_bytes = p->stream->readf(p->stream, buf+5, record_length-5);
+    if (read_bytes < record_length-5)
+    {
+        yaz_log (YLOG_WARN, "Couldn't read whole MARC record");
+        return NULL;
+    }
+    /* skip until we meet a record separator */
+    while (buf[record_length-1] != ISO2709_RS)
+    {
+        if (record_length > sizeof(buf)-2)
+            break;
+        read_bytes = p->stream->readf(p->stream, buf+record_length, 1);
+        if (read_bytes != 1)
+            break;
+        record_length++;
+    }
+    /* read one byte ahead to see if there is more ... */
+    read_bytes = p->stream->readf(p->stream, buf+record_length, 1);
+    if (read_bytes == 1)
+    {
+        off_t cur_offset = p->stream->tellf(p->stream);
+       if (p->stream->endf)
+        {
+            off_t end_offset = cur_offset - 1;
+           p->stream->endf(p->stream, &end_offset);
+        }
+    }
+
+    absynName = mi->type;
+    res_root = data1_mk_root (p->dh, p->mem, absynName);
+    if (!res_root)
+    {
+        yaz_log (YLOG_WARN, "cannot read MARC without an abstract syntax");
+        return 0;
+    }
+    if (marc_xml)
+    {
+       data1_node *lead;
+       const char *attr[] = { "xmlns", "http://www.loc.gov/MARC21/slim", 0};
+                        
+       res_top = data1_mk_tag (p->dh, p->mem, "record", attr, res_root);
+
+       lead = data1_mk_tag(p->dh, p->mem, "leader", 0, res_top);
+       data1_mk_text_n(p->dh, p->mem, buf, 24, lead);
+    }
+    else
+       res_top = data1_mk_tag (p->dh, p->mem, absynName, 0, res_root);
+
+    if ((marctab = data1_absyn_getmarctab(p->dh, res_root)))
+    {
+       memcpy(marctab->leader, buf, 24);
+        memcpy(marctab->implementation_codes, buf+6, 4);
+        marctab->implementation_codes[4] = '\0';
+        memcpy(marctab->user_systems, buf+17, 3);
+        marctab->user_systems[3] = '\0';
+    }
+
+    if (marctab && marctab->force_indicator_length >= 0)
+       indicator_length = marctab->force_indicator_length;
+    else
+       indicator_length = atoi_n (buf+10, 1);
+    if (marctab && marctab->force_identifier_length >= 0)
+       identifier_length = marctab->force_identifier_length;
+    else
+       identifier_length = atoi_n (buf+11, 1);
+    base_address = atoi_n (buf+12, 5);
+
+    length_data_entry = atoi_n (buf+20, 1);
+    length_starting = atoi_n (buf+21, 1);
+    length_implementation = atoi_n (buf+22, 1);
+
+    for (entry_p = 24; buf[entry_p] != ISO2709_FS; )
+    {
+        int l = 3 + length_data_entry + length_starting;
+        if (entry_p + l >= record_length)
+        {
+           yaz_log(YLOG_WARN, "MARC: Directory offset %d: end of record.",
+                   entry_p);
+           return 0;
+        }
+        /* check for digits in length info */
+        while (--l >= 3)
+            if (!isdigit(*(const unsigned char *) (buf + entry_p+l)))
+                break;
+        if (l >= 3)
+        {
+            /* not all digits, so stop directory scan */
+           yaz_log(YLOG_LOG, "MARC: Bad directory");
+            break;
+        }
+        entry_p += 3 + length_data_entry + length_starting;
+    }
+    end_of_directory = entry_p;
+    if (base_address != entry_p+1)
+    {
+       yaz_log(YLOG_WARN, "MARC: Base address does not follow directory");
+    }
+    for (entry_p = 24; entry_p != end_of_directory; )
+    {
+        int data_length;
+        int data_offset;
+        int end_offset;
+        int i, i0;
+        char tag[4];
+        data1_node *res;
+        data1_node *parent = res_top;
+
+        memcpy (tag, buf+entry_p, 3);
+        entry_p += 3;
+        tag[3] = '\0';
+
+       if (marc_xml)
+           res = parent;
+       else
+           res = data1_mk_tag_n (p->dh, p->mem, tag, 3, 0 /* attr */, parent);
+       
+#if MARC_DEBUG
+        fprintf (outf, "%s ", tag);
+#endif
+        data_length = atoi_n (buf+entry_p, length_data_entry);
+        entry_p += length_data_entry;
+        data_offset = atoi_n (buf+entry_p, length_starting);
+        entry_p += length_starting;
+        i = data_offset + base_address;
+        end_offset = i+data_length-1;
+
+       if (data_length <= 0 || data_offset < 0 || end_offset >= record_length)
+       {
+           yaz_log(YLOG_WARN, "MARC: Bad offsets in data. Skipping rest");
+           break;
+       }
+       
+        if (memcmp (tag, "00", 2) && indicator_length)
+        {
+            /* generate indicator node */
+           if (marc_xml)
+           {
+               const char *attr[10];
+               int j;
+
+               attr[0] = "tag";
+               attr[1] = tag;
+               attr[2] = 0;
+
+               res = data1_mk_tag(p->dh, p->mem, "datafield", attr, res);
+
+               for (j = 0; j<indicator_length; j++)
+               {
+                   char str1[18], str2[2];
+                   sprintf (str1, "ind%d", j+1);
+                   str2[0] = buf[i+j];
+                   str2[1] = '\0';
+
+                   attr[0] = str1;
+                   attr[1] = str2;
+                   
+                   data1_tag_add_attr (p->dh, p->mem, res, attr);
+               }
+           }
+           else
+           {
+#if MARC_DEBUG
+               int j;
+#endif
+               res = data1_mk_tag_n (p->dh, p->mem, 
+                                     buf+i, indicator_length, 0 /* attr */, res);
+#if MARC_DEBUG
+               for (j = 0; j<indicator_length; j++)
+                   fprintf (outf, "%c", buf[j+i]);
+#endif
+           }
+           i += indicator_length;
+        } 
+       else
+       {
+           if (marc_xml)
+           {
+               const char *attr[10];
+               
+               attr[0] = "tag";
+               attr[1] = tag;
+               attr[2] = 0;
+               
+               res = data1_mk_tag(p->dh, p->mem, "controlfield", attr, res);
+           }
+       }
+        parent = res;
+        /* traverse sub fields */
+        i0 = i;
+        while (buf[i] != ISO2709_RS && buf[i] != ISO2709_FS && i < end_offset)
+        {
+           if (memcmp (tag, "00", 2) && identifier_length)
+            {
+               data1_node *res;
+               if (marc_xml)
+               {
+                   int j;
+                   const char *attr[3];
+                   char code[10];
+                   
+                   for (j = 1; j<identifier_length && j < 9; j++)
+                       code[j-1] = buf[i+j];
+                   code[j-1] = 0;
+                   attr[0] = "code";
+                   attr[1] = code;
+                   attr[2] = 0;
+                   res = data1_mk_tag(p->dh, p->mem, "subfield",
+                                      attr, parent);
+               }
+               else
+               {
+                   res = data1_mk_tag_n (p->dh, p->mem,
+                                          buf+i+1, identifier_length-1, 
+                                          0 /* attr */, parent);
+               }
+#if MARC_DEBUG
+                fprintf (outf, " $"); 
+                for (j = 1; j<identifier_length; j++)
+                    fprintf (outf, "%c", buf[j+i]);
+                fprintf (outf, " ");
+#endif
+                i += identifier_length;
+                i0 = i;
+               while (buf[i] != ISO2709_RS && buf[i] != ISO2709_IDFS &&
+                    buf[i] != ISO2709_FS && i < end_offset)
+               {
+#if MARC_DEBUG
+                   fprintf (outf, "%c", buf[i]);
+#endif
+                   i++;
+               }
+               data1_mk_text_n (p->dh, p->mem, buf + i0, i - i0, res);
+               i0 = i;
+            }
+            else
+            {
+#if MARC_DEBUG
+                fprintf (outf, "%c", buf[i]);
+#endif
+                i++;
+            }
+        }
+        if (i > i0)
+       {
+            data1_mk_text_n (p->dh, p->mem, buf + i0, i - i0, parent);
+       }
+#if MARC_DEBUG
+        fprintf (outf, "\n");
+        if (i < end_offset)
+            fprintf (outf, "-- separator but not at end of field\n");
+        if (buf[i] != ISO2709_RS && buf[i] != ISO2709_FS)
+            fprintf (outf, "-- no separator at end of field\n");
+#endif
+    }
+    return res_root;
+}
+
+/*
+ * Locate some data under this node. This routine should handle variants
+ * prettily.
+ */
+static char *get_data(data1_node *n, int *len)
+{
+    char *r;
+
+    while (n)
+    {
+        if (n->which == DATA1N_data)
+        {
+            int i;
+            *len = n->u.data.len;
+
+            for (i = 0; i<*len; i++)
+                if (!d1_isspace(n->u.data.data[i]))
+                    break;
+            while (*len && d1_isspace(n->u.data.data[*len - 1]))
+                (*len)--;
+            *len = *len - i;
+            if (*len > 0)
+                return n->u.data.data + i;
+        }
+        if (n->which == DATA1N_tag)
+            n = n->child;
+       else if (n->which == DATA1N_data)
+            n = n->next;
+       else
+            break;     
+    }
+    r = "";
+    *len = strlen(r);
+    return r;
+}
+
+static data1_node *lookup_subfield(data1_node *node, const char *name)
+{
+    data1_node *p;
+    
+    for (p=node; p; p=p->next)
+    {
+       if (!yaz_matchstr(p->u.tag.tag, name))
+           return p;
+    }
+    return 0;
+}
+
+static inline_subfield *lookup_inline_subfield(inline_subfield *pisf,
+                                              const char *name)
+{
+    inline_subfield *p;
+    
+    for (p=pisf; p; p=p->next)
+    {
+       if (!yaz_matchstr(p->name, name))
+           return p;
+    }
+    return 0;
+}
+
+static inline_subfield *cat_inline_subfield(mc_subfield *psf, WRBUF buf,
+                                           inline_subfield *pisf)
+{
+    mc_subfield *p;
+    
+    for (p = psf; p && pisf; p = p->next)
+    {
+       if (p->which == MC_SF)
+       {
+           inline_subfield *found = lookup_inline_subfield(pisf, p->name);
+           
+           if (found)
+           {
+               if (strcmp(p->prefix, "_"))
+               {
+                   wrbuf_puts(buf, " ");
+                   wrbuf_puts(buf, p->prefix);
+               }
+               if (p->interval.start == -1)
+               {
+                   wrbuf_puts(buf, found->data);
+               }
+               else
+               {
+                   wrbuf_write(buf, found->data+p->interval.start,
+                               p->interval.end-p->interval.start);
+                   wrbuf_puts(buf, "");
+               }
+               if (strcmp(p->suffix, "_"))
+               {
+                   wrbuf_puts(buf, p->suffix);
+                   wrbuf_puts(buf, " ");
+               }
+#if MARCOMP_DEBUG
+               yaz_log(YLOG_LOG, "cat_inline_subfield(): add subfield $%s", found->name);
+#endif         
+               pisf = found->next;
+           }
+       }
+       else if (p->which == MC_SFVARIANT)
+       {
+           inline_subfield *next;
+           
+           do {
+               next = cat_inline_subfield(p->u.child, buf, pisf);
+               if (next == pisf)
+                   break;
+               pisf = next;
+           } while (pisf);
+       }
+       else if (p->which == MC_SFGROUP)
+       {
+           mc_subfield *pp;
+           int found;
+           
+           for (pp = p->u.child, found = 0; pp; pp = pp->next)
+           {
+               if (!yaz_matchstr(pisf->name, p->name))
+               {
+                   found = 1;
+                   break;
+               }
+           }
+           if (found)
+           {
+               wrbuf_puts(buf, " (");
+               pisf = cat_inline_subfield(p->u.child, buf, pisf);
+               wrbuf_puts(buf, ") ");
+           }
+       }
+    }
+    return pisf; 
+}
+
+static void cat_inline_field(mc_field *pf, WRBUF buf, data1_node *subfield)
+{    
+    if (!pf || !subfield)
+       return;
+
+    for (;subfield;)
+    {
+       int len;
+       inline_field *pif=NULL;
+       data1_node *psubf;
+       
+       if (yaz_matchstr(subfield->u.tag.tag, "1"))
+       {
+           subfield = subfield->next;
+           continue;
+       }
+       
+       psubf = subfield;
+       pif = inline_mk_field();
+       do
+       {
+           int i;
+           if ((i=inline_parse(pif, psubf->u.tag.tag, get_data(psubf, &len)))<0)
+           {
+               yaz_log(YLOG_WARN, "inline subfield ($%s): parse error",
+                   psubf->u.tag.tag);
+               inline_destroy_field(pif);
+               return; 
+           }
+           psubf = psubf->next;
+       } while (psubf && yaz_matchstr(psubf->u.tag.tag, "1"));
+       
+       subfield = psubf;
+       
+       if (pif && !yaz_matchstr(pif->name, pf->name))
+       {
+           if (!pf->list && pif->list)
+           {
+               wrbuf_puts(buf, pif->list->data);
+           }
+           else
+           {
+               int ind1, ind2;
+
+               /*
+                   check indicators
+               */
+
+               ind1 = (pif->ind1[0] == ' ') ? '_':pif->ind1[0];
+               ind2 = (pif->ind2[0] == ' ') ? '_':pif->ind2[0];
+    
+               if (((pf->ind1[0] == '.') || (ind1 == pf->ind1[0])) &&
+                   ((pf->ind2[0] == '.') || (ind2 == pf->ind2[0])))
+               {
+                   cat_inline_subfield(pf->list, buf, pif->list);
+                   
+                   /*
+                       add separator for inline fields
+                   */
+                   if (wrbuf_len(buf))
+                   {
+                       wrbuf_puts(buf, "\n");
+                   }
+               }
+               else
+               {
+                   yaz_log(YLOG_WARN, "In-line field %s missed -- indicators do not match", pif->name);
+               }
+           }
+       }
+       inline_destroy_field(pif);
+    }
+#if MARCOMP_DEBUG    
+    yaz_log(YLOG_LOG, "cat_inline_field(): got buffer {%s}", buf);
+#endif
+}
+
+static data1_node *cat_subfield(mc_subfield *psf, WRBUF buf,
+                               data1_node *subfield)
+{
+    mc_subfield *p;
+    
+    for (p = psf; p && subfield; p = p->next)
+    {
+       if (p->which == MC_SF)
+       {
+           data1_node *found = lookup_subfield(subfield, p->name);
+           
+           if (found)
+           {
+               int len;
+               
+               if (strcmp(p->prefix, "_"))
+               {
+                   wrbuf_puts(buf, " ");
+                   wrbuf_puts(buf, p->prefix);
+               }
+               
+               if (p->u.in_line)
+               {
+                   cat_inline_field(p->u.in_line, buf, found);
+               }
+               else if (p->interval.start == -1)
+               {
+                   wrbuf_puts(buf, get_data(found, &len));
+               }
+               else
+               {
+                   wrbuf_write(buf, get_data(found, &len)+p->interval.start,
+                       p->interval.end-p->interval.start);
+                   wrbuf_puts(buf, "");
+               }
+               if (strcmp(p->suffix, "_"))
+               {
+                   wrbuf_puts(buf, p->suffix);
+                   wrbuf_puts(buf, " ");
+               }
+#if MARCOMP_DEBUG              
+               yaz_log(YLOG_LOG, "cat_subfield(): add subfield $%s", found->u.tag.tag);
+#endif         
+               subfield = found->next;
+           }
+       }
+       else if (p->which == MC_SFVARIANT)
+       {
+           data1_node *next;
+           do {
+               next = cat_subfield(p->u.child, buf, subfield);
+               if (next == subfield)
+                   break;
+               subfield = next;
+           } while (subfield);
+       }
+       else if (p->which == MC_SFGROUP)
+       {
+           mc_subfield *pp;
+           int found;
+           
+           for (pp = p->u.child, found = 0; pp; pp = pp->next)
+           {
+               if (!yaz_matchstr(subfield->u.tag.tag, pp->name))
+               {
+                   found = 1;
+                   break;
+               }
+           }
+           if (found)
+           {
+               wrbuf_puts(buf, " (");
+               subfield = cat_subfield(p->u.child, buf, subfield);
+               wrbuf_puts(buf, ") ");
+           }
+       }
+    }
+    return subfield;
+}
+
+static data1_node *cat_field(struct grs_read_info *p, mc_field *pf,
+                            WRBUF buf, data1_node *field)
+{
+    data1_node *subfield;
+    int ind1, ind2;
+    
+    if (!pf || !field)
+       return 0;
+
+    
+    if (yaz_matchstr(field->u.tag.tag, pf->name))
+       return field->next;
+
+    subfield = field->child;
+    
+    if (!subfield)
+       return field->next;
+
+    /*
+       check subfield without indicators
+    */
+    
+    if (!pf->list && subfield->which == DATA1N_data)
+    {
+       int len;
+       
+       if (pf->interval.start == -1)
+       {
+           wrbuf_puts(buf, get_data(field, &len));
+       }
+       else
+       {
+           wrbuf_write(buf, get_data(field, &len)+pf->interval.start,
+                       pf->interval.end-pf->interval.start);
+           wrbuf_puts(buf, "");
+       }
+#if MARCOMP_DEBUG
+        yaz_log(YLOG_LOG, "cat_field(): got buffer {%s}", buf);
+#endif
+       return field->next;
+    }
+    
+    /*
+       check indicators
+    */
+
+    ind1 = (subfield->u.tag.tag[0] == ' ') ? '_':subfield->u.tag.tag[0];
+    ind2 = (subfield->u.tag.tag[1] == ' ') ? '_':subfield->u.tag.tag[1];
+    
+    if (!(
+       ((pf->ind1[0] == '.') || (ind1 == pf->ind1[0])) &&
+       ((pf->ind2[0] == '.') || (ind2 == pf->ind2[0]))
+       ))
+    {
+#if MARCOMP_DEBUG
+       yaz_log(YLOG_WARN, "Field %s missed -- does not match indicators", field->u.tag.tag);
+#endif
+       return field->next;
+    }
+    
+    subfield = subfield->child;
+    
+    if (!subfield)
+       return field->next;
+
+    cat_subfield(pf->list, buf, subfield);
+
+#if MARCOMP_DEBUG    
+    yaz_log(YLOG_LOG, "cat_field(): got buffer {%s}", buf);
+#endif
+    
+    return field->next;    
+}
+
+static int is_empty(char *s)
+{
+    char *p = s;
+    
+    for (p = s; *p; p++)
+    {
+       if (!isspace(*(unsigned char *)p))
+           return 0;
+    }
+    return 1;
+}
+
+static void parse_data1_tree(struct grs_read_info *p, const char *mc_stmnt,
+                            data1_node *root)
+{
+    data1_marctab *marctab = data1_absyn_getmarctab(p->dh, root);
+    data1_node *top = root->child;
+    data1_node *field;
+    mc_context *c;
+    mc_field *pf;
+    WRBUF buf;
+    
+    c = mc_mk_context(mc_stmnt+3);
+    
+    if (!c)
+       return;
+       
+    pf = mc_getfield(c);
+    
+    if (!pf)
+    {
+       mc_destroy_context(c);
+       return;
+    }
+    buf = wrbuf_alloc();
+#if MARCOMP_DEBUG    
+    yaz_log(YLOG_LOG, "parse_data1_tree(): statement -{%s}", mc_stmnt);
+#endif
+    if (!yaz_matchstr(pf->name, "ldr"))
+    {
+       data1_node *new;
+#if MARCOMP_DEBUG
+       yaz_log(YLOG_LOG,"parse_data1_tree(): try LEADER from {%d} to {%d} positions",
+           pf->interval.start, pf->interval.end);
+#endif 
+        if (marctab)
+        {
+            new = data1_mk_tag_n(p->dh, p->mem, mc_stmnt, strlen(mc_stmnt), 0, top);
+            data1_mk_text_n(p->dh, p->mem, marctab->leader+pf->interval.start,
+                            pf->interval.end-pf->interval.start+1, new);
+        }
+    }
+    else
+    {
+       field=top->child;
+       
+       while(field)
+       {
+           if (!yaz_matchstr(field->u.tag.tag, pf->name))
+           {
+               data1_node *new;
+               char *pb;
+#if MARCOMP_DEBUG              
+               yaz_log(YLOG_LOG, "parse_data1_tree(): try field {%s}", field->u.tag.tag);
+#endif         
+               wrbuf_rewind(buf);
+               wrbuf_puts(buf, "");
+
+               field = cat_field(p, pf, buf, field);
+               
+               wrbuf_cstr(buf);
+                pb = wrbuf_buf(buf);
+               for (pb = strtok(pb, "\n"); pb; pb = strtok(NULL, "\n"))
+               {
+                    if (!is_empty(pb))
+                    {
+                        new = data1_mk_tag_n(p->dh, p->mem, mc_stmnt, strlen(mc_stmnt), 0, top);
+                        data1_mk_text_n(p->dh, p->mem, pb, strlen(pb), new);
+                    }
+               }
+           }
+           else
+           {
+               field = field->next;
+           }
+       }
+    }
+    mc_destroy_field(pf);
+    mc_destroy_context(c);
+    wrbuf_destroy(buf);
+}
+
+data1_node *grs_read_marcxml(struct grs_read_info *p)
+{
+    data1_node *root = grs_read_iso2709(p, 1);
+    data1_element *e;
+
+    if (!root)
+       return 0;
+       
+    for (e = data1_absyn_getelements(p->dh, root); e; e=e->next)
+    {
+       data1_tag *tag = e->tag;
+       
+       if (tag && tag->which == DATA1T_string &&
+           !yaz_matchstr(tag->value.string, "mc?"))
+               parse_data1_tree(p, tag->value.string, root);
+    }
+    return root;
+}
+
+data1_node *grs_read_marc(struct grs_read_info *p)
+{
+    data1_node *root = grs_read_iso2709(p, 0);
+    data1_element *e;
+
+    if (!root)
+       return 0;
+       
+    for (e = data1_absyn_getelements(p->dh, root); e; e=e->next)
+    {
+       data1_tag *tag = e->tag;
+       
+       if (tag && tag->which == DATA1T_string &&
+           !yaz_matchstr(tag->value.string, "mc?"))
+               parse_data1_tree(p, tag->value.string, root);
+    }
+    return root;
+}
+
+static void *init_marc(Res res, RecType rt)
+{
+    struct marc_info *p = xmalloc(sizeof(*p));
+    strcpy(p->type, "");
+    return p;
+}
+
+static ZEBRA_RES config_marc(void *clientData, Res res, const char *args)
+{
+    struct marc_info *p = (struct marc_info*) clientData;
+    if (strlen(args) < sizeof(p->type))
+       strcpy(p->type, args);
+    return ZEBRA_OK;
+}
+
+static void destroy_marc(void *clientData)
+{
+    struct marc_info *p = (struct marc_info*) clientData;
+    xfree (p);
+}
+
+
+static int extract_marc(void *clientData, struct recExtractCtrl *ctrl)
+{
+    return zebra_grs_extract(clientData, ctrl, grs_read_marc);
+}
+
+static int retrieve_marc(void *clientData, struct recRetrieveCtrl *ctrl)
+{
+    return zebra_grs_retrieve(clientData, ctrl, grs_read_marc);
+}
+
+static struct recType marc_type = {
+    0,
+    "grs.marc",
+    init_marc,
+    config_marc,
+    destroy_marc,
+    extract_marc,
+    retrieve_marc,
+};
+
+static int extract_marcxml(void *clientData, struct recExtractCtrl *ctrl)
+{
+    return zebra_grs_extract(clientData, ctrl, grs_read_marcxml);
+}
+
+static int retrieve_marcxml(void *clientData, struct recRetrieveCtrl *ctrl)
+{
+    return zebra_grs_retrieve(clientData, ctrl, grs_read_marcxml);
+}
+
+static struct recType marcxml_type = {
+    0,
+    "grs.marcxml",
+    init_marc,
+    config_marc,
+    destroy_marc,
+    extract_marcxml,
+    retrieve_marcxml,
+};
+
+RecType
+#ifdef IDZEBRA_STATIC_GRS_MARC
+idzebra_filter_grs_marc
+#else
+idzebra_filter
+#endif
+
+[] = {
+    &marc_type,
+    &marcxml_type,
+    0,
+};
+    
+/*
+ * Local variables:
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ * vim: shiftwidth=4 tabstop=8 expandtab
+ */
+
diff --git a/index/mod_grs_regx.c b/index/mod_grs_regx.c
new file mode 100644 (file)
index 0000000..ecf8f95
--- /dev/null
@@ -0,0 +1,2037 @@
+/* This file is part of the Zebra server.
+   Copyright (C) 1995-2008 Index Data
+
+Zebra is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2, or (at your option) any later
+version.
+
+Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <yaz/tpath.h>
+#include <idzebra/util.h>
+#include <dfa.h>
+#include <idzebra/recgrs.h>
+
+#if HAVE_TCL_H
+#include <tcl.h>
+
+#if MAJOR_VERSION >= 8
+#define HAVE_TCL_OBJECTS
+#endif
+#endif
+
+#define REGX_DEBUG 0
+
+#define F_WIN_EOF 2000000000
+#define F_WIN_READ 1
+
+#define REGX_EOF     0
+#define REGX_PATTERN 1
+#define REGX_BODY    2
+#define REGX_BEGIN   3
+#define REGX_END     4
+#define REGX_CODE    5
+#define REGX_CONTEXT 6
+#define REGX_INIT    7
+
+struct regxCode {
+    char *str;
+#if HAVE_TCL_OBJECTS
+    Tcl_Obj *tcl_obj;
+#endif
+};
+
+struct lexRuleAction {
+    int which; 
+    union {
+        struct {
+            struct DFA *dfa;    /* REGX_PATTERN */
+            int body;
+        } pattern;
+        struct regxCode *code;  /* REGX_CODE */
+    } u;
+    struct lexRuleAction *next;
+};
+
+struct lexRuleInfo {
+    int no;
+    struct lexRuleAction *actionList;
+};
+
+struct lexRule {
+    struct lexRuleInfo info;
+    struct lexRule *next;
+};
+
+struct lexContext {
+    char *name;
+    struct DFA *dfa;
+    struct lexRule *rules;
+    struct lexRuleInfo **fastRule;
+    int ruleNo;
+    int initFlag;
+
+    struct lexRuleAction *beginActionList;
+    struct lexRuleAction *endActionList;
+    struct lexRuleAction *initActionList;
+    struct lexContext *next;
+};
+
+struct lexConcatBuf {
+    int max;
+    char *buf;
+};
+
+struct lexSpec {
+    char *name;
+    struct lexContext *context;
+
+    struct lexContext **context_stack;
+    int context_stack_size;
+    int context_stack_top;
+
+    int lineNo;
+    NMEM m;
+    data1_handle dh;
+#if HAVE_TCL_H
+    Tcl_Interp *tcl_interp;
+#endif
+    struct ZebraRecStream *stream;
+    off_t (*f_win_ef)(struct ZebraRecStream *s, off_t *);
+
+    int f_win_start;      /* first byte of buffer is this file offset */
+    int f_win_end;        /* last byte of buffer is this offset - 1 */
+    int f_win_size;       /* size of buffer */
+    char *f_win_buf;      /* buffer itself */
+    int (*f_win_rf)(struct ZebraRecStream *, char *, size_t);
+    off_t (*f_win_sf)(struct ZebraRecStream *, off_t);
+
+    struct lexConcatBuf *concatBuf;
+    int maxLevel;
+    data1_node **d1_stack;
+    int d1_level;
+    int stop_flag;
+    
+    int *arg_start;
+    int *arg_end;
+    int arg_no;
+    int ptr;
+};
+
+struct lexSpecs {
+    struct lexSpec *spec;
+    char type[256];
+};
+
+static char *f_win_get (struct lexSpec *spec, off_t start_pos, off_t end_pos,
+                        int *size)
+{
+    int i, r, off = start_pos - spec->f_win_start;
+
+    if (off >= 0 && end_pos <= spec->f_win_end)
+    {
+        *size = end_pos - start_pos;
+        return spec->f_win_buf + off;
+    }
+    if (off < 0 || start_pos >= spec->f_win_end)
+    {
+        (*spec->f_win_sf)(spec->stream, start_pos);
+        spec->f_win_start = start_pos;
+
+        if (!spec->f_win_buf)
+            spec->f_win_buf = (char *) xmalloc (spec->f_win_size);
+        *size = (*spec->f_win_rf)(spec->stream, spec->f_win_buf,
+                                  spec->f_win_size);
+        spec->f_win_end = spec->f_win_start + *size;
+
+        if (*size > end_pos - start_pos)
+            *size = end_pos - start_pos;
+        return spec->f_win_buf;
+    }
+    for (i = 0; i<spec->f_win_end - start_pos; i++)
+        spec->f_win_buf[i] = spec->f_win_buf[i + off];
+    r = (*spec->f_win_rf)(spec->stream,
+                          spec->f_win_buf + i,
+                          spec->f_win_size - i);
+    spec->f_win_start = start_pos;
+    spec->f_win_end += r;
+    *size = i + r;
+    if (*size > end_pos - start_pos)
+        *size = end_pos - start_pos;
+    return spec->f_win_buf;
+}
+
+static int f_win_advance (struct lexSpec *spec, int *pos)
+{
+    int size;
+    char *buf;
+    
+    if (*pos >= spec->f_win_start && *pos < spec->f_win_end)
+        return spec->f_win_buf[(*pos)++ - spec->f_win_start];
+    if (*pos == F_WIN_EOF)
+        return 0;
+    buf = f_win_get (spec, *pos, *pos+1, &size);
+    if (size == 1)
+    {
+        (*pos)++;
+        return *buf;
+    }
+    *pos = F_WIN_EOF;
+    return 0;
+}
+
+static void regxCodeDel (struct regxCode **pp)
+{
+    struct regxCode *p = *pp;
+    if (p)
+    {
+#if HAVE_TCL_OBJECTS
+       if (p->tcl_obj)
+           Tcl_DecrRefCount (p->tcl_obj);
+#endif
+        xfree (p->str); 
+        xfree (p);
+        *pp = NULL;
+    }
+}
+
+static void regxCodeMk (struct regxCode **pp, const char *buf, int len)
+{
+    struct regxCode *p;
+
+    p = (struct regxCode *) xmalloc (sizeof(*p));
+    p->str = (char *) xmalloc (len+1);
+    memcpy (p->str, buf, len);
+    p->str[len] = '\0';
+#if HAVE_TCL_OBJECTS
+    p->tcl_obj = Tcl_NewStringObj ((char *) buf, len);
+    if (p->tcl_obj)
+       Tcl_IncrRefCount (p->tcl_obj);
+#endif
+    *pp = p;
+}
+
+static struct DFA *lexSpecDFA (void)
+{
+    struct DFA *dfa;
+
+    dfa = dfa_init ();
+    dfa_parse_cmap_del (dfa, ' ');
+    dfa_parse_cmap_del (dfa, '\t');
+    dfa_parse_cmap_add (dfa, '/', 0);
+    return dfa;
+}
+
+static void actionListDel (struct lexRuleAction **rap)
+{
+    struct lexRuleAction *ra1, *ra;
+
+    for (ra = *rap; ra; ra = ra1)
+    {
+        ra1 = ra->next;
+        switch (ra->which)
+        {
+        case REGX_PATTERN:
+            dfa_delete (&ra->u.pattern.dfa);
+            break;
+        case REGX_CODE:
+            regxCodeDel (&ra->u.code);
+            break;
+        }
+        xfree (ra);
+    }
+    *rap = NULL;
+}
+
+static struct lexContext *lexContextCreate (const char *name)
+{
+    struct lexContext *p = (struct lexContext *) xmalloc (sizeof(*p));
+
+    p->name = xstrdup (name);
+    p->ruleNo = 1;
+    p->initFlag = 0;
+    p->dfa = lexSpecDFA ();
+    p->rules = NULL;
+    p->fastRule = NULL;
+    p->beginActionList = NULL;
+    p->endActionList = NULL;
+    p->initActionList = NULL;
+    p->next = NULL;
+    return p;
+}
+
+static void lexContextDestroy (struct lexContext *p)
+{
+    struct lexRule *rp, *rp1;
+
+    dfa_delete (&p->dfa);
+    xfree (p->fastRule);
+    for (rp = p->rules; rp; rp = rp1)
+    {
+       rp1 = rp->next;
+        actionListDel (&rp->info.actionList);
+        xfree (rp);
+    }
+    actionListDel (&p->beginActionList);
+    actionListDel (&p->endActionList);
+    actionListDel (&p->initActionList);
+    xfree (p->name);
+    xfree (p);
+}
+
+static struct lexSpec *lexSpecCreate (const char *name, data1_handle dh)
+{
+    struct lexSpec *p;
+    int i;
+    
+    p = (struct lexSpec *) xmalloc (sizeof(*p));
+    p->name = (char *) xmalloc (strlen(name)+1);
+    strcpy (p->name, name);
+
+#if HAVE_TCL_H
+    p->tcl_interp = 0;
+#endif
+    p->dh = dh;
+    p->context = NULL;
+    p->context_stack_size = 100;
+    p->context_stack = (struct lexContext **)
+       xmalloc (sizeof(*p->context_stack) * p->context_stack_size);
+    p->f_win_buf = NULL;
+
+    p->maxLevel = 128;
+    p->concatBuf = (struct lexConcatBuf *)
+       xmalloc (sizeof(*p->concatBuf) * p->maxLevel);
+    for (i = 0; i < p->maxLevel; i++)
+    {
+       p->concatBuf[i].max = 0;
+       p->concatBuf[i].buf = 0;
+    }
+    p->d1_stack = (data1_node **) xmalloc (sizeof(*p->d1_stack) * p->maxLevel);
+    p->d1_level = 0;
+    return p;
+}
+
+static void lexSpecDestroy (struct lexSpec **pp)
+{
+    struct lexSpec *p;
+    struct lexContext *lt;
+    int i;
+
+    assert (pp);
+    p = *pp;
+    if (!p)
+        return ;
+
+    for (i = 0; i < p->maxLevel; i++)
+       xfree (p->concatBuf[i].buf);
+    xfree (p->concatBuf);
+
+    lt = p->context;
+    while (lt)
+    {
+       struct lexContext *lt_next = lt->next;
+       lexContextDestroy (lt);
+       lt = lt_next;
+    }
+#if HAVE_TCL_OBJECTS
+    if (p->tcl_interp)
+       Tcl_DeleteInterp (p->tcl_interp);
+#endif
+    xfree (p->name);
+    xfree (p->f_win_buf);
+    xfree (p->context_stack);
+    xfree (p->d1_stack);
+    xfree (p);
+    *pp = NULL;
+}
+
+static int readParseToken (const char **cpp, int *len)
+{
+    const char *cp = *cpp;
+    char cmd[32];
+    int i, level;
+
+    while (*cp == ' ' || *cp == '\t' || *cp == '\n' || *cp == '\r')
+        cp++;
+    switch (*cp)
+    {
+    case '\0':
+        return 0;
+    case '/':
+        *cpp = cp+1;
+        return REGX_PATTERN;
+    case '{':
+        *cpp = cp+1;
+        level = 1;
+        while (*++cp)
+        {
+            if (*cp == '{')
+                level++;
+            else if (*cp == '}')
+            {
+                level--;
+                if (level == 0)
+                    break;
+            }
+        }
+        *len = cp - *cpp;
+        return REGX_CODE;
+    default:
+        i = 0;
+        while (1)
+        {
+            if (*cp >= 'a' && *cp <= 'z')
+                cmd[i] = *cp;
+            else if (*cp >= 'A' && *cp <= 'Z')
+                cmd[i] = *cp + 'a' - 'A';
+            else
+                break;
+            if (i < (int) sizeof(cmd)-2)
+               i++;
+            cp++;
+        }
+        cmd[i] = '\0';
+        if (i == 0)
+        {
+            yaz_log (YLOG_WARN, "bad character %d %c", *cp, *cp);
+            cp++;
+            while (*cp && *cp != ' ' && *cp != '\t' &&
+                   *cp != '\n' && *cp != '\r')
+                cp++;
+            *cpp = cp;
+            return 0;
+        }
+        *cpp = cp;
+        if (!strcmp (cmd, "begin"))
+            return REGX_BEGIN;
+        else if (!strcmp (cmd, "end"))
+            return REGX_END;
+        else if (!strcmp (cmd, "body"))
+            return REGX_BODY;
+       else if (!strcmp (cmd, "context"))
+           return REGX_CONTEXT;
+       else if (!strcmp (cmd, "init"))
+           return REGX_INIT;
+        else
+        {
+            yaz_log (YLOG_WARN, "bad command %s", cmd);
+            return 0;
+        }
+    }
+}
+
+static int actionListMk (struct lexSpec *spec, const char *s,
+                         struct lexRuleAction **ap)
+{
+    int r, tok, len;
+    int bodyMark = 0;
+    const char *s0;
+
+    while ((tok = readParseToken (&s, &len)))
+    {
+        switch (tok)
+        {
+        case REGX_BODY:
+            bodyMark = 1;
+            continue;
+        case REGX_CODE:
+            *ap = (struct lexRuleAction *) xmalloc (sizeof(**ap));
+            (*ap)->which = tok;
+            regxCodeMk (&(*ap)->u.code, s, len);
+            s += len+1;
+            break;
+        case REGX_PATTERN:
+            *ap = (struct lexRuleAction *) xmalloc (sizeof(**ap));
+            (*ap)->which = tok;
+            (*ap)->u.pattern.body = bodyMark;
+            bodyMark = 0;
+            (*ap)->u.pattern.dfa = lexSpecDFA ();
+           s0 = s;
+            r = dfa_parse ((*ap)->u.pattern.dfa, &s);
+            if (r || *s != '/')
+            {
+                int pos = s - s0;
+                xfree (*ap);
+                *ap = NULL;
+                yaz_log(YLOG_WARN, "regular expression error '%.*s'", pos, s0);
+                return -1;
+            }
+            else
+            {
+                int pos = s - s0;
+                if (debug_dfa_tran)
+                    printf("pattern: %.*s\n", pos, s0);
+                dfa_mkstate((*ap)->u.pattern.dfa);
+                s++;
+            }
+            break;
+        case REGX_BEGIN:
+            yaz_log (YLOG_WARN, "cannot use BEGIN here");
+            continue;
+        case REGX_INIT:
+            yaz_log (YLOG_WARN, "cannot use INIT here");
+            continue;
+        case REGX_END:
+            *ap = (struct lexRuleAction *) xmalloc (sizeof(**ap));
+            (*ap)->which = tok;
+            break;
+        }
+        ap = &(*ap)->next;
+    }
+    *ap = NULL;
+    return 0;
+}
+
+int readOneSpec (struct lexSpec *spec, const char *s)
+{
+    int len, r, tok;
+    struct lexRule *rp;
+    struct lexContext *lc;
+
+    tok = readParseToken (&s, &len);
+    if (tok == REGX_CONTEXT)
+    {
+       char context_name[32];
+       tok = readParseToken (&s, &len);
+       if (tok != REGX_CODE)
+       {
+           yaz_log (YLOG_WARN, "missing name after CONTEXT keyword");
+           return 0;
+       }
+       if (len > 31)
+           len = 31;
+       memcpy (context_name, s, len);
+       context_name[len] = '\0';
+       lc = lexContextCreate (context_name);
+       lc->next = spec->context;
+       spec->context = lc;
+       return 0;
+    }
+    if (!spec->context)
+       spec->context = lexContextCreate ("main");
+       
+    switch (tok)
+    {
+    case REGX_BEGIN:
+        actionListDel (&spec->context->beginActionList);
+        actionListMk (spec, s, &spec->context->beginActionList);
+       break;
+    case REGX_END:
+        actionListDel (&spec->context->endActionList);
+        actionListMk (spec, s, &spec->context->endActionList);
+       break;
+    case REGX_INIT:
+        actionListDel (&spec->context->initActionList);
+        actionListMk (spec, s, &spec->context->initActionList);
+       break;
+    case REGX_PATTERN:
+#if REGX_DEBUG
+       yaz_log (YLOG_LOG, "rule %d %s", spec->context->ruleNo, s);
+#endif
+        r = dfa_parse (spec->context->dfa, &s);
+        if (r)
+        {
+            yaz_log (YLOG_WARN, "regular expression error. r=%d", r);
+            return -1;
+        }
+        if (*s != '/')
+        {
+            yaz_log (YLOG_WARN, "expects / at end of pattern. got %c", *s);
+            return -1;
+        }
+        s++;
+        rp = (struct lexRule *) xmalloc (sizeof(*rp));
+        rp->info.no = spec->context->ruleNo++;
+        rp->next = spec->context->rules;
+        spec->context->rules = rp;
+        actionListMk (spec, s, &rp->info.actionList);
+    }
+    return 0;
+}
+
+int readFileSpec (struct lexSpec *spec)
+{
+    struct lexContext *lc;
+    int c, i, errors = 0;
+    FILE *spec_inf = 0;
+    WRBUF lineBuf;
+    char fname[256];
+
+#if HAVE_TCL_H
+    if (spec->tcl_interp)
+    {
+       sprintf (fname, "%s.tflt", spec->name);
+       spec_inf = data1_path_fopen (spec->dh, fname, "r");
+    }
+#endif
+    if (!spec_inf)
+    {
+       sprintf (fname, "%s.flt", spec->name);
+       spec_inf = data1_path_fopen (spec->dh, fname, "r");
+    }
+    if (!spec_inf)
+    {
+        yaz_log (YLOG_ERRNO|YLOG_WARN, "cannot read spec file %s", spec->name);
+        return -1;
+    }
+    yaz_log (YLOG_LOG, "reading regx filter %s", fname);
+#if HAVE_TCL_H
+    if (spec->tcl_interp)
+       yaz_log (YLOG_LOG, "Tcl enabled");
+#endif
+
+#if 0
+    debug_dfa_trav = 0;
+    debug_dfa_tran = 1;
+    debug_dfa_followpos = 0;
+    dfa_verbose = 1;
+#endif
+
+    lineBuf = wrbuf_alloc();
+    spec->lineNo = 0;
+    c = getc (spec_inf);
+    while (c != EOF)
+    {
+       wrbuf_rewind (lineBuf);
+        if (c == '#' || c == '\n' || c == ' ' || c == '\t' || c == '\r')
+        {
+            while (c != '\n' && c != EOF)
+                c = getc (spec_inf);
+            spec->lineNo++;
+            if (c == '\n')
+                c = getc (spec_inf);
+        }
+        else
+        {
+            int addLine = 0;
+           
+            while (1)
+            {
+                int c1 = c;
+               wrbuf_putc(lineBuf, c);
+                c = getc (spec_inf);
+               while (c == '\r')
+                   c = getc (spec_inf);
+                if (c == EOF)
+                    break;
+                if (c1 == '\n')
+                {
+                    if (c != ' ' && c != '\t')
+                        break;
+                    addLine++;
+                }
+            }
+           wrbuf_putc(lineBuf, '\0');
+            readOneSpec (spec, wrbuf_buf(lineBuf));
+            spec->lineNo += addLine;
+        }
+    }
+    fclose (spec_inf);
+    wrbuf_destroy(lineBuf);
+
+    for (lc = spec->context; lc; lc = lc->next)
+    {
+       struct lexRule *rp;
+       lc->fastRule = (struct lexRuleInfo **)
+           xmalloc (sizeof(*lc->fastRule) * lc->ruleNo);
+       for (i = 0; i < lc->ruleNo; i++)
+           lc->fastRule[i] = NULL;
+       for (rp = lc->rules; rp; rp = rp->next)
+           lc->fastRule[rp->info.no] = &rp->info;
+       dfa_mkstate (lc->dfa);
+    }
+    if (errors)
+        return -1;
+    
+    return 0;
+}
+
+#if 0
+static struct lexSpec *curLexSpec = NULL;
+#endif
+
+static void execData (struct lexSpec *spec,
+                      const char *ebuf, int elen, int formatted_text,
+                     const char *attribute_str, int attribute_len)
+{
+    struct data1_node *res, *parent;
+    int org_len;
+
+    if (elen == 0) /* shouldn't happen, but it does! */
+       return ;
+#if REGX_DEBUG
+    if (elen > 80)
+        yaz_log (YLOG_LOG, "data(%d bytes) %.40s ... %.*s", elen,
+             ebuf, 40, ebuf + elen-40);
+    else if (elen == 1 && ebuf[0] == '\n')
+    {
+        yaz_log (YLOG_LOG, "data(new line)");
+    }
+    else if (elen > 0)
+        yaz_log (YLOG_LOG, "data(%d bytes) %.*s", elen, elen, ebuf);
+    else 
+        yaz_log (YLOG_LOG, "data(%d bytes)", elen);
+#endif
+        
+    if (spec->d1_level <= 1)
+        return;
+
+    parent = spec->d1_stack[spec->d1_level -1];
+    assert (parent);
+
+    if (attribute_str)
+    {
+       data1_xattr **ap;
+       res = parent;
+       if (res->which != DATA1N_tag)
+           return;
+       /* sweep through exising attributes.. */
+       for (ap = &res->u.tag.attributes; *ap; ap = &(*ap)->next)
+           if (strlen((*ap)->name) == attribute_len &&
+               !memcmp((*ap)->name, attribute_str, attribute_len))
+               break;
+       if (!*ap)
+       {
+           /* new attribute. Create it with name + value */
+           *ap = nmem_malloc(spec->m, sizeof(**ap));
+
+           (*ap)->name = nmem_malloc(spec->m, attribute_len+1);
+           memcpy((*ap)->name, attribute_str, attribute_len);
+           (*ap)->name[attribute_len] = '\0';
+
+           (*ap)->value = nmem_malloc(spec->m, elen+1);
+           memcpy((*ap)->value, ebuf, elen);
+           (*ap)->value[elen] = '\0';
+           (*ap)->next = 0;
+       }
+       else
+       {
+           /* append to value if attribute already exists */
+           char *nv = nmem_malloc(spec->m, elen + 1 + strlen((*ap)->value));
+           strcpy(nv, (*ap)->value);
+           memcpy (nv + strlen(nv), ebuf, elen);
+           nv[strlen(nv)+elen] = '\0';
+           (*ap)->value = nv;
+       }
+    } 
+    else 
+    {
+       if ((res = spec->d1_stack[spec->d1_level]) && 
+           res->which == DATA1N_data)
+           org_len = res->u.data.len;
+       else
+       {
+           org_len = 0;
+           
+           res = data1_mk_node2 (spec->dh, spec->m, DATA1N_data, parent);
+           res->u.data.what = DATA1I_text;
+           res->u.data.len = 0;
+           res->u.data.formatted_text = formatted_text;
+           res->u.data.data = 0;
+           
+           if (spec->d1_stack[spec->d1_level])
+               spec->d1_stack[spec->d1_level]->next = res;
+           spec->d1_stack[spec->d1_level] = res;
+       }
+       if (org_len + elen >= spec->concatBuf[spec->d1_level].max)
+       {
+           char *old_buf, *new_buf;
+           
+           spec->concatBuf[spec->d1_level].max = org_len + elen + 256;
+           new_buf = (char *) xmalloc (spec->concatBuf[spec->d1_level].max);
+           if ((old_buf = spec->concatBuf[spec->d1_level].buf))
+           {
+               memcpy (new_buf, old_buf, org_len);
+               xfree (old_buf);
+           }
+           spec->concatBuf[spec->d1_level].buf = new_buf;
+       }
+       memcpy (spec->concatBuf[spec->d1_level].buf + org_len, ebuf, elen);
+       res->u.data.len += elen;
+    }
+}
+
+static void execDataP (struct lexSpec *spec,
+                       const char *ebuf, int elen, int formatted_text)
+{
+    execData (spec, ebuf, elen, formatted_text, 0, 0);
+}
+
+static void tagDataRelease (struct lexSpec *spec)
+{
+    data1_node *res;
+    
+    if ((res = spec->d1_stack[spec->d1_level]) &&
+       res->which == DATA1N_data && 
+       res->u.data.what == DATA1I_text)
+    {
+       assert (!res->u.data.data);
+       assert (res->u.data.len > 0);
+       if (res->u.data.len > DATA1_LOCALDATA)
+           res->u.data.data = (char *) nmem_malloc (spec->m, res->u.data.len);
+       else
+           res->u.data.data = res->lbuf;
+       memcpy (res->u.data.data, spec->concatBuf[spec->d1_level].buf,
+               res->u.data.len);
+    }
+}
+
+static void variantBegin (struct lexSpec *spec, 
+                         const char *class_str, int class_len,
+                         const char *type_str, int type_len,
+                         const char *value_str, int value_len)
+{
+    struct data1_node *parent = spec->d1_stack[spec->d1_level -1];
+    char tclass[DATA1_MAX_SYMBOL], ttype[DATA1_MAX_SYMBOL];
+    data1_vartype *tp;
+    int i;
+    data1_node *res;
+
+    if (spec->d1_level == 0)
+    {
+        yaz_log (YLOG_WARN, "in variant begin. No record type defined");
+        return ;
+    }
+    if (class_len >= DATA1_MAX_SYMBOL)
+       class_len = DATA1_MAX_SYMBOL-1;
+    memcpy (tclass, class_str, class_len);
+    tclass[class_len] = '\0';
+
+    if (type_len >= DATA1_MAX_SYMBOL)
+       type_len = DATA1_MAX_SYMBOL-1;
+    memcpy (ttype, type_str, type_len);
+    ttype[type_len] = '\0';
+
+#if REGX_DEBUG 
+    yaz_log (YLOG_LOG, "variant begin(%s,%s,%d)", tclass, ttype,
+         spec->d1_level);
+#endif
+
+    if (!(tp =
+         data1_getvartypeby_absyn(spec->dh, parent->root->u.root.absyn,
+                                  tclass, ttype)))
+       return;
+    
+    if (parent->which != DATA1N_variant)
+    {
+       res = data1_mk_node2 (spec->dh, spec->m, DATA1N_variant, parent);
+       if (spec->d1_stack[spec->d1_level])
+           tagDataRelease (spec);
+       spec->d1_stack[spec->d1_level] = res;
+       spec->d1_stack[++(spec->d1_level)] = NULL;
+    }
+    for (i = spec->d1_level-1; spec->d1_stack[i]->which == DATA1N_variant; i--)
+       if (spec->d1_stack[i]->u.variant.type == tp)
+       {
+           spec->d1_level = i;
+           break;
+       }
+
+#if REGX_DEBUG 
+    yaz_log (YLOG_LOG, "variant node(%d)", spec->d1_level);
+#endif
+    parent = spec->d1_stack[spec->d1_level-1];
+    res = data1_mk_node2 (spec->dh, spec->m, DATA1N_variant, parent);
+    res->u.variant.type = tp;
+
+    if (value_len >= DATA1_LOCALDATA)
+       value_len =DATA1_LOCALDATA-1;
+    memcpy (res->lbuf, value_str, value_len);
+    res->lbuf[value_len] = '\0';
+
+    res->u.variant.value = res->lbuf;
+    
+    if (spec->d1_stack[spec->d1_level])
+       tagDataRelease (spec);
+    spec->d1_stack[spec->d1_level] = res;
+    spec->d1_stack[++(spec->d1_level)] = NULL;
+}
+
+static void tagStrip (const char **tag, int *len)
+{
+    int i;
+
+    for (i = *len; i > 0 && isspace((*tag)[i-1]); --i)
+        ;
+    *len = i;
+    for (i = 0; i < *len && isspace((*tag)[i]); i++)
+        ;
+    *tag += i;
+    *len -= i;
+}
+
+static void tagBegin (struct lexSpec *spec, 
+                      const char *tag, int len)
+{
+    if (spec->d1_level == 0)
+    {
+        yaz_log (YLOG_WARN, "in element begin. No record type defined");
+        return ;
+    }
+    tagStrip (&tag, &len);
+    if (spec->d1_stack[spec->d1_level])
+       tagDataRelease (spec);
+
+#if REGX_DEBUG 
+    yaz_log (YLOG_LOG, "begin tag(%.*s, %d)", len, tag, spec->d1_level);
+#endif
+
+    spec->d1_stack[spec->d1_level] = data1_mk_tag_n (
+        spec->dh, spec->m, tag, len, 0, spec->d1_stack[spec->d1_level -1]);
+    spec->d1_stack[++(spec->d1_level)] = NULL;
+}
+
+static void tagEnd (struct lexSpec *spec, int min_level,
+                    const char *tag, int len)
+{
+    tagStrip (&tag, &len);
+    while (spec->d1_level > min_level)
+    {
+       tagDataRelease (spec);
+        (spec->d1_level)--;
+        if (spec->d1_level == 0)
+           break;
+        if ((spec->d1_stack[spec->d1_level]->which == DATA1N_tag) &&
+           (!tag ||
+            (strlen(spec->d1_stack[spec->d1_level]->u.tag.tag) ==
+             (size_t) len &&
+             !memcmp (spec->d1_stack[spec->d1_level]->u.tag.tag, tag, len))))
+            break;
+    }
+#if REGX_DEBUG
+    yaz_log (YLOG_LOG, "end tag(%d)", spec->d1_level);
+#endif
+}
+
+
+static int tryMatch (struct lexSpec *spec, int *pptr, int *mptr,
+                     struct DFA *dfa, int greedy)
+{
+    struct DFA_state *state = dfa->states[0];
+    struct DFA_tran *t;
+    unsigned char c = 0;
+    unsigned char c_prev = 0;
+    int ptr = *pptr;          /* current pointer */
+    int start_ptr = *pptr;    /* first char of match */
+    int last_ptr = 0;         /* last char of match */
+    int last_rule = 0;        /* rule number of current match */
+    int restore_ptr = 0;
+    int i;
+
+    if (ptr)
+    {
+       --ptr;
+        c = f_win_advance (spec, &ptr);
+    }
+    while (1)
+    {
+       if (dfa->states[0] == state)
+       {
+           c_prev = c;
+           restore_ptr = ptr;
+       }
+        c = f_win_advance (spec, &ptr);
+
+        if (ptr == F_WIN_EOF)
+        {
+            if (last_rule)
+            {
+                *mptr = start_ptr;
+                *pptr = last_ptr;
+                return 1;
+            }
+            break;
+        }
+
+        t = state->trans;
+        i = state->tran_no;
+        while (1)
+            if (--i < 0)    /* no transition for character c */
+            {
+                if (last_rule)
+                {
+                    *mptr = start_ptr;     /* match starts here */
+                    *pptr = last_ptr;      /* match end here (+1) */
+                    return 1;
+                }
+                state = dfa->states[0];
+
+               ptr = restore_ptr;
+               c = f_win_advance (spec, &ptr);
+
+                start_ptr = ptr;
+
+                break;
+            }
+            else if (c >= t->ch[0] && c <= t->ch[1])
+            {
+                state = dfa->states[t->to];
+                if (state->rule_no && c_prev == '\n')
+               {
+                   last_rule = state->rule_no;
+                   last_ptr = ptr;
+               }
+               else if (state->rule_nno)
+               {
+                   last_rule = state->rule_nno;
+                   last_ptr = ptr;
+               }
+               break;
+            }
+            else
+                t++;
+    }
+    return 0;
+}
+
+static int execTok (struct lexSpec *spec, const char **src,
+                    const char **tokBuf, int *tokLen)
+{
+    const char *s = *src;
+
+    while (*s == ' ' || *s == '\t')
+        s++;
+    if (!*s)
+        return 0;
+    if (*s == '$' && s[1] >= '0' && s[1] <= '9')
+    {
+        int n = 0;
+        s++;
+        while (*s >= '0' && *s <= '9')
+            n = n*10 + (*s++ -'0');
+        if (spec->arg_no == 0)
+        {
+            *tokBuf = "";
+            *tokLen = 0;
+        }
+        else
+        {
+            if (n >= spec->arg_no)
+                n = spec->arg_no-1;
+            *tokBuf = f_win_get (spec, spec->arg_start[n], spec->arg_end[n],
+                                tokLen);
+        }
+    }
+    else if (*s == '\"')
+    {
+        *tokBuf = ++s;
+        while (*s && *s != '\"')
+            s++;
+        *tokLen = s - *tokBuf;
+        if (*s)
+            s++;
+        *src = s;
+    }
+    else if (*s == '\n' || *s == ';')
+    {
+        *src = s+1;
+        return 1;
+    }
+    else if (*s == '-')
+    {
+        *tokBuf = s++;
+        while (*s && *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r' &&
+               *s != ';')
+            s++;
+        *tokLen = s - *tokBuf;
+        *src = s;
+        return 3;
+    }
+    else
+    {
+        *tokBuf = s++;
+        while (*s && *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r' &&
+               *s != ';')
+            s++;
+        *tokLen = s - *tokBuf;
+    }
+    *src = s;
+    return 2;
+}
+
+static char *regxStrz (const char *src, int len, char *str)
+{
+    if (len > 63)
+        len = 63;
+    memcpy (str, src, len);
+    str[len] = '\0';
+    return str;
+}
+
+#if HAVE_TCL_H
+static int cmd_tcl_begin (ClientData clientData, Tcl_Interp *interp,
+                         int argc, const char **argv)
+{
+    struct lexSpec *spec = (struct lexSpec *) clientData;
+    if (argc < 2)
+       return TCL_ERROR;
+    if (!strcmp(argv[1], "record") && argc == 3)
+    {
+       const char *absynName = argv[2];
+        data1_node *res;
+
+#if REGX_DEBUG
+       yaz_log (YLOG_LOG, "begin record %s", absynName);
+#endif
+        res = data1_mk_root (spec->dh, spec->m, absynName);
+        
+       spec->d1_level = 0;
+
+        spec->d1_stack[spec->d1_level++] = res;
+
+        res = data1_mk_tag (spec->dh, spec->m, absynName, 0, res);
+
+        spec->d1_stack[spec->d1_level++] = res;
+
+        spec->d1_stack[spec->d1_level] = NULL;
+    }
+    else if (!strcmp(argv[1], "element") && argc == 3)
+    {
+       tagBegin (spec, argv[2], strlen(argv[2]));
+    }
+    else if (!strcmp (argv[1], "variant") && argc == 5)
+    {
+       variantBegin (spec, argv[2], strlen(argv[2]),
+                     argv[3], strlen(argv[3]),
+                     argv[4], strlen(argv[4]));
+    }
+    else if (!strcmp (argv[1], "context") && argc == 3)
+    {
+       struct lexContext *lc = spec->context;
+#if REGX_DEBUG
+       yaz_log (YLOG_LOG, "begin context %s",argv[2]);
+#endif
+       while (lc && strcmp (argv[2], lc->name))
+           lc = lc->next;
+       if (lc)
+       {
+           spec->context_stack[++(spec->context_stack_top)] = lc;
+       }
+       else
+           yaz_log (YLOG_WARN, "unknown context %s", argv[2]);
+    }
+    else
+       return TCL_ERROR;
+    return TCL_OK;
+}
+
+static int cmd_tcl_end (ClientData clientData, Tcl_Interp *interp,
+                       int argc, const char **argv)
+{
+    struct lexSpec *spec = (struct lexSpec *) clientData;
+    if (argc < 2)
+       return TCL_ERROR;
+    
+    if (!strcmp (argv[1], "record"))
+    {
+       while (spec->d1_level)
+       {
+           tagDataRelease (spec);
+           (spec->d1_level)--;
+       }
+#if REGX_DEBUG
+       yaz_log (YLOG_LOG, "end record");
+#endif
+       spec->stop_flag = 1;
+    }
+    else if (!strcmp (argv[1], "element"))
+    {
+       int min_level = 2;
+       const char *element = 0;
+       if (argc >= 3 && !strcmp(argv[2], "-record"))
+       {
+           min_level = 0;
+           if (argc == 4)
+               element = argv[3];
+       }
+       else
+           if (argc == 3)
+               element = argv[2];
+       tagEnd (spec, min_level, element, (element ? strlen(element) : 0));
+       if (spec->d1_level <= 1)
+       {
+#if REGX_DEBUG
+           yaz_log (YLOG_LOG, "end element end records");
+#endif
+           spec->stop_flag = 1;
+       }
+    }
+    else if (!strcmp (argv[1], "context"))
+    {
+#if REGX_DEBUG
+       yaz_log (YLOG_LOG, "end context");
+#endif
+       if (spec->context_stack_top)
+           (spec->context_stack_top)--;
+    }
+    else
+       return TCL_ERROR;
+    return TCL_OK;
+}
+
+static int cmd_tcl_data (ClientData clientData, Tcl_Interp *interp,
+                        int argc, const char **argv)
+{
+    int argi = 1;
+    int textFlag = 0;
+    const char *element = 0;
+    const char *attribute = 0;
+    struct lexSpec *spec = (struct lexSpec *) clientData;
+    
+    while (argi < argc)
+    {
+       if (!strcmp("-text", argv[argi]))
+       {
+           textFlag = 1;
+           argi++;
+       }
+       else if (!strcmp("-element", argv[argi]))
+       {
+           argi++;
+           if (argi < argc)
+               element = argv[argi++];
+       }
+       else if (!strcmp("-attribute", argv[argi]))
+       {
+           argi++;
+           if (argi < argc)
+               attribute = argv[argi++];
+       }
+       else
+           break;
+    }
+    if (element)
+       tagBegin (spec, element, strlen(element));
+
+    while (argi < argc)
+    {
+#if TCL_MAJOR_VERSION > 8 || (TCL_MAJOR_VERSION == 8 && TCL_MINOR_VERSION > 0)
+       Tcl_DString ds;
+       char *native = Tcl_UtfToExternalDString(0, argv[argi], -1, &ds);
+       execData (spec, native, strlen(native), textFlag, attribute, 
+                 attribute ? strlen(attribute) : 0);
+       Tcl_DStringFree (&ds);
+#else
+       execData (spec, argv[argi], strlen(argv[argi]), textFlag, attribute,
+                 attribute ? strlen(attribute) : 0);
+#endif
+       argi++;
+    }
+    if (element)
+       tagEnd (spec, 2, NULL, 0);
+    return TCL_OK;
+}
+
+static int cmd_tcl_unread (ClientData clientData, Tcl_Interp *interp,
+                          int argc, const char **argv)
+{
+    struct lexSpec *spec = (struct lexSpec *) clientData;
+    int argi = 1;
+    int offset = 0;
+    int no;
+    
+    while (argi < argc)
+    {
+       if (!strcmp("-offset", argv[argi]))
+       {
+           argi++;
+           if (argi < argc)
+           {
+               offset = atoi(argv[argi]);
+               argi++;
+           }
+       }
+       else
+           break;
+    }
+    if (argi != argc-1)
+       return TCL_ERROR;
+    no = atoi(argv[argi]);
+    if (no >= spec->arg_no)
+       no = spec->arg_no - 1;
+    spec->ptr = spec->arg_start[no] + offset;
+    return TCL_OK;
+}
+
+static void execTcl (struct lexSpec *spec, struct regxCode *code)
+{   
+    int i;
+    int ret;
+    for (i = 0; i < spec->arg_no; i++)
+    {
+       char var_name[10], *var_buf;
+       int var_len, ch;
+       
+       sprintf (var_name, "%d", i);
+       var_buf = f_win_get (spec, spec->arg_start[i], spec->arg_end[i],
+                            &var_len); 
+       if (var_buf)
+       {
+           ch = var_buf[var_len];
+           var_buf[var_len] = '\0';
+           Tcl_SetVar (spec->tcl_interp, var_name, var_buf, 0);
+           var_buf[var_len] = ch;
+       }
+    }
+#if HAVE_TCL_OBJECTS
+    ret = Tcl_GlobalEvalObj(spec->tcl_interp, code->tcl_obj);
+#else
+    ret = Tcl_GlobalEval (spec->tcl_interp, code->str);
+#endif
+    if (ret != TCL_OK)
+    {
+       const char *err = Tcl_GetVar(spec->tcl_interp, "errorInfo", 0);
+       yaz_log(YLOG_FATAL, "Tcl error, line=%d, \"%s\"\n%s", 
+           spec->tcl_interp->errorLine,
+           spec->tcl_interp->result,
+           err ? err : "[NO ERRORINFO]");
+    }
+}
+/* HAVE_TCL_H */
+#endif
+
+static void execCode (struct lexSpec *spec, struct regxCode *code)
+{
+    const char *s = code->str;
+    int cmd_len, r;
+    const char *cmd_str;
+    
+    r = execTok (spec, &s, &cmd_str, &cmd_len);
+    while (r)
+    {
+        char *p, ptmp[64];
+        
+        if (r == 1)
+        {
+            r = execTok (spec, &s, &cmd_str, &cmd_len);
+            continue;
+        }
+        p = regxStrz (cmd_str, cmd_len, ptmp);
+        if (!strcmp (p, "begin"))
+        {
+            r = execTok (spec, &s, &cmd_str, &cmd_len);
+            if (r < 2)
+           {
+               yaz_log (YLOG_WARN, "missing keyword after 'begin'");
+                continue;
+           }
+            p = regxStrz (cmd_str, cmd_len, ptmp);
+            if (!strcmp (p, "record"))
+            {
+                r = execTok (spec, &s, &cmd_str, &cmd_len);
+                if (r < 2)
+                    continue;
+                if (spec->d1_level <= 1)
+                {
+                    static char absynName[64];
+                    data1_node *res;
+
+                    if (cmd_len > 63)
+                        cmd_len = 63;
+                    memcpy (absynName, cmd_str, cmd_len);
+                    absynName[cmd_len] = '\0';
+#if REGX_DEBUG
+                    yaz_log (YLOG_LOG, "begin record %s", absynName);
+#endif
+                    res = data1_mk_root (spec->dh, spec->m, absynName);
+                    
+                   spec->d1_level = 0;
+
+                    spec->d1_stack[spec->d1_level++] = res;
+
+                    res = data1_mk_tag (spec->dh, spec->m, absynName, 0, res);
+
+                    spec->d1_stack[spec->d1_level++] = res;
+
+                    spec->d1_stack[spec->d1_level] = NULL;
+                }
+                r = execTok (spec, &s, &cmd_str, &cmd_len);
+            }
+            else if (!strcmp (p, "element"))
+            {
+                r = execTok (spec, &s, &cmd_str, &cmd_len);
+                if (r < 2)
+                    continue;
+                tagBegin (spec, cmd_str, cmd_len);
+                r = execTok (spec, &s, &cmd_str, &cmd_len);
+            } 
+           else if (!strcmp (p, "variant"))
+           {
+               int class_len;
+               const char *class_str = NULL;
+               int type_len;
+               const char *type_str = NULL;
+               int value_len;
+               const char *value_str = NULL;
+               r = execTok (spec, &s, &cmd_str, &cmd_len);
+               if (r < 2)
+                   continue;
+               class_str = cmd_str;
+               class_len = cmd_len;
+               r = execTok (spec, &s, &cmd_str, &cmd_len);
+               if (r < 2)
+                   continue;
+               type_str = cmd_str;
+               type_len = cmd_len;
+
+               r = execTok (spec, &s, &cmd_str, &cmd_len);
+               if (r < 2)
+                   continue;
+               value_str = cmd_str;
+               value_len = cmd_len;
+
+                variantBegin (spec, class_str, class_len,
+                             type_str, type_len, value_str, value_len);
+               
+               
+               r = execTok (spec, &s, &cmd_str, &cmd_len);
+           }
+           else if (!strcmp (p, "context"))
+           {
+               if (r > 1)
+               {
+                   struct lexContext *lc = spec->context;
+                   r = execTok (spec, &s, &cmd_str, &cmd_len);
+                   p = regxStrz (cmd_str, cmd_len, ptmp);
+#if REGX_DEBUG
+                   yaz_log (YLOG_LOG, "begin context %s", p);
+#endif
+                   while (lc && strcmp (p, lc->name))
+                       lc = lc->next;
+                   if (lc)
+                       spec->context_stack[++(spec->context_stack_top)] = lc;
+                   else
+                       yaz_log (YLOG_WARN, "unknown context %s", p);
+                   
+               }
+               r = execTok (spec, &s, &cmd_str, &cmd_len);
+           }
+           else
+           {
+               yaz_log (YLOG_WARN, "bad keyword '%s' after begin", p);
+           }
+        }
+        else if (!strcmp (p, "end"))
+        {
+            r = execTok (spec, &s, &cmd_str, &cmd_len);
+            if (r < 2)
+           {
+               yaz_log (YLOG_WARN, "missing keyword after 'end'");
+               continue;
+           }
+           p = regxStrz (cmd_str, cmd_len, ptmp);
+           if (!strcmp (p, "record"))
+           {
+               while (spec->d1_level)
+               {
+                   tagDataRelease (spec);
+                   (spec->d1_level)--;
+               }
+               r = execTok (spec, &s, &cmd_str, &cmd_len);
+#if REGX_DEBUG
+               yaz_log (YLOG_LOG, "end record");
+#endif
+               spec->stop_flag = 1;
+           }
+           else if (!strcmp (p, "element"))
+           {
+                int min_level = 2;
+                while ((r = execTok (spec, &s, &cmd_str, &cmd_len)) == 3)
+                {
+                    if (cmd_len==7 && !memcmp ("-record", cmd_str, cmd_len))
+                        min_level = 0;
+                }
+               if (r > 2)
+               {
+                   tagEnd (spec, min_level, cmd_str, cmd_len);
+                   r = execTok (spec, &s, &cmd_str, &cmd_len);
+               }
+               else
+                   tagEnd (spec, min_level, NULL, 0);
+                if (spec->d1_level <= 1)
+                {
+#if REGX_DEBUG
+                   yaz_log (YLOG_LOG, "end element end records");
+#endif
+                   spec->stop_flag = 1;
+                }
+
+           }
+           else if (!strcmp (p, "context"))
+           {
+#if REGX_DEBUG
+               yaz_log (YLOG_LOG, "end context");
+#endif
+               if (spec->context_stack_top)
+                   (spec->context_stack_top)--;
+               r = execTok (spec, &s, &cmd_str, &cmd_len);
+           }       
+           else
+               yaz_log (YLOG_WARN, "bad keyword '%s' after end", p);
+       }
+        else if (!strcmp (p, "data"))
+        {
+            int textFlag = 0;
+            int element_len;
+            const char *element_str = NULL;
+           int attribute_len;
+           const char *attribute_str = NULL;
+            
+            while ((r = execTok (spec, &s, &cmd_str, &cmd_len)) == 3)
+            {
+                if (cmd_len==5 && !memcmp ("-text", cmd_str, cmd_len))
+                    textFlag = 1;
+                else if (cmd_len==8 && !memcmp ("-element", cmd_str, cmd_len))
+                {
+                    r = execTok (spec, &s, &element_str, &element_len);
+                    if (r < 2)
+                        break;
+                }
+                else if (cmd_len==10 && !memcmp ("-attribute", cmd_str, 
+                                                cmd_len))
+                {
+                    r = execTok (spec, &s, &attribute_str, &attribute_len);
+                    if (r < 2)
+                        break;
+                }
+                else 
+                    yaz_log (YLOG_WARN, "bad data option: %.*s",
+                          cmd_len, cmd_str);
+            }
+            if (r != 2)
+            {
+                yaz_log (YLOG_WARN, "missing data item after data");
+                continue;
+            }
+            if (element_str)
+                tagBegin (spec, element_str, element_len);
+            do
+            {
+                execData (spec, cmd_str, cmd_len, textFlag,
+                         attribute_str, attribute_len);
+                r = execTok (spec, &s, &cmd_str, &cmd_len);
+            } while (r > 1);
+            if (element_str)
+                tagEnd (spec, 2, NULL, 0);
+        }
+        else if (!strcmp (p, "unread"))
+        {
+            int no, offset;
+            r = execTok (spec, &s, &cmd_str, &cmd_len);
+            if (r==3 && cmd_len == 7 && !memcmp ("-offset", cmd_str, cmd_len))
+            {
+                r = execTok (spec, &s, &cmd_str, &cmd_len);
+                if (r < 2)
+                {
+                    yaz_log (YLOG_WARN, "missing number after -offset");
+                    continue;
+                }
+                p = regxStrz (cmd_str, cmd_len, ptmp);
+                offset = atoi (p);
+                r = execTok (spec, &s, &cmd_str, &cmd_len);
+            }
+            else
+                offset = 0;
+            if (r < 2)
+            {
+                yaz_log (YLOG_WARN, "missing index after unread command");
+                continue;
+            }
+            if (cmd_len != 1 || *cmd_str < '0' || *cmd_str > '9')
+            {
+                yaz_log (YLOG_WARN, "bad index after unread command");
+                continue;
+            }
+            else
+            {
+                no = *cmd_str - '0';
+                if (no >= spec->arg_no)
+                    no = spec->arg_no - 1;
+                spec->ptr = spec->arg_start[no] + offset;
+            }
+            r = execTok (spec, &s, &cmd_str, &cmd_len);
+        }
+       else if (!strcmp (p, "context"))
+       {
+            if (r > 1)
+           {
+               struct lexContext *lc = spec->context;
+               r = execTok (spec, &s, &cmd_str, &cmd_len);
+               p = regxStrz (cmd_str, cmd_len, ptmp);
+               
+               while (lc && strcmp (p, lc->name))
+                   lc = lc->next;
+               if (lc)
+                   spec->context_stack[spec->context_stack_top] = lc;
+               else
+                   yaz_log (YLOG_WARN, "unknown context %s", p);
+
+           }
+           r = execTok (spec, &s, &cmd_str, &cmd_len);
+       }
+        else
+        {
+            yaz_log (YLOG_WARN, "unknown code command '%.*s'", cmd_len, cmd_str);
+            r = execTok (spec, &s, &cmd_str, &cmd_len);
+            continue;
+        }
+        if (r > 1)
+        {
+            yaz_log (YLOG_WARN, "ignoring token %.*s", cmd_len, cmd_str);
+            do {
+                r = execTok (spec, &s, &cmd_str, &cmd_len);
+            } while (r > 1);
+        }
+    }
+}
+
+
+static int execAction (struct lexSpec *spec, struct lexRuleAction *ap,
+                       int start_ptr, int *pptr)
+{
+    int sptr;
+    int arg_start[20];
+    int arg_end[20];
+    int arg_no = 1;
+
+    if (!ap)
+       return 1;
+    arg_start[0] = start_ptr;
+    arg_end[0] = *pptr;
+    spec->arg_start = arg_start;
+    spec->arg_end = arg_end;
+
+    while (ap)
+    {
+        switch (ap->which)
+        {
+        case REGX_PATTERN:
+            if (ap->u.pattern.body)
+            {
+                arg_start[arg_no] = *pptr;
+                if (!tryMatch (spec, pptr, &sptr, ap->u.pattern.dfa, 0))
+                {
+                    arg_end[arg_no] = F_WIN_EOF;
+                    arg_no++;
+                    arg_start[arg_no] = F_WIN_EOF;
+                    arg_end[arg_no] = F_WIN_EOF;
+                   yaz_log(YLOG_DEBUG, "Pattern match rest of record");
+                   *pptr = F_WIN_EOF;
+                }
+                else
+                {
+                    arg_end[arg_no] = sptr;
+                    arg_no++;
+                    arg_start[arg_no] = sptr;
+                    arg_end[arg_no] = *pptr;
+                }
+            }
+            else
+            {
+                arg_start[arg_no] = *pptr;
+                if (!tryMatch (spec, pptr, &sptr, ap->u.pattern.dfa, 1))
+                    return 1;
+                if (sptr != arg_start[arg_no])
+                    return 1;
+                arg_end[arg_no] = *pptr;
+            }
+            arg_no++;
+            break;
+        case REGX_CODE:
+           spec->arg_no = arg_no;
+           spec->ptr = *pptr;
+#if HAVE_TCL_H
+           if (spec->tcl_interp)
+               execTcl(spec, ap->u.code);
+           else
+               execCode (spec, ap->u.code);
+#else
+           execCode (spec, ap->u.code);
+#endif
+           *pptr = spec->ptr;
+           if (spec->stop_flag)
+               return 0;
+            break;
+        case REGX_END:
+            arg_start[arg_no] = *pptr;
+            arg_end[arg_no] = F_WIN_EOF;
+            arg_no++;
+            *pptr = F_WIN_EOF;
+        }
+        ap = ap->next;
+    }
+    return 1;
+}
+
+static int execRule (struct lexSpec *spec, struct lexContext *context,
+                     int ruleNo, int start_ptr, int *pptr)
+{
+#if REGX_DEBUG
+    yaz_log (YLOG_LOG, "exec rule %d", ruleNo);
+#endif
+    return execAction (spec, context->fastRule[ruleNo]->actionList,
+                       start_ptr, pptr);
+}
+
+int lexNode (struct lexSpec *spec, int *ptr)
+{
+    struct lexContext *context = spec->context_stack[spec->context_stack_top];
+    struct DFA_state *state = context->dfa->states[0];
+    struct DFA_tran *t;
+    unsigned char c;
+    unsigned char c_prev = '\n';
+    int i;
+    int last_rule = 0;        /* rule number of current match */
+    int last_ptr = *ptr;      /* last char of match */
+    int start_ptr = *ptr;     /* first char of match */
+    int skip_ptr = *ptr;      /* first char of run */
+    int more = 0;
+
+    while (1)
+    {
+        c = f_win_advance (spec, ptr);
+        if (*ptr == F_WIN_EOF)
+        {
+           /* end of file met */
+            if (last_rule)
+            {
+               /* there was a match */
+                if (skip_ptr < start_ptr)
+                {
+                   /* deal with chars that didn't match */
+                    int size;
+                    char *buf;
+                    buf = f_win_get (spec, skip_ptr, start_ptr, &size);
+                    execDataP (spec, buf, size, 0);
+                }
+               /* restore pointer */
+                *ptr = last_ptr;
+               /* execute rule */
+                if (!execRule (spec, context, last_rule, start_ptr, ptr))
+                    return more;
+               /* restore skip pointer */
+                skip_ptr = *ptr;
+                last_rule = 0;
+            }
+            else if (skip_ptr < *ptr)
+            {
+               /* deal with chars that didn't match */
+                int size;
+                char *buf;
+                buf = f_win_get (spec, skip_ptr, *ptr, &size);
+                execDataP (spec, buf, size, 0);
+            }
+            state = context->dfa->states[0];
+            if (*ptr == F_WIN_EOF)
+                return more;
+        }
+        t = state->trans;
+        i = state->tran_no;
+        while (1)
+            if (--i < 0)
+            {   /* no transition for character c ... */
+                if (last_rule)
+                {
+                    if (skip_ptr < start_ptr)
+                    {
+                       /* deal with chars that didn't match */
+                        int size;
+                        char *buf;
+                        buf = f_win_get (spec, skip_ptr, start_ptr, &size);
+                        execDataP (spec, buf, size, 0);
+                    }
+                   /* restore pointer */
+                    *ptr = last_ptr;
+                    if (!execRule (spec, context, last_rule, start_ptr, ptr))
+                    {
+                        if (spec->f_win_ef && *ptr != F_WIN_EOF)
+                       {
+                            off_t end_offset = *ptr;
+#if REGX_DEBUG
+                           yaz_log (YLOG_LOG, "regx: endf ptr=%d", *ptr);
+#endif
+                            (*spec->f_win_ef)(spec->stream, &end_offset);
+                       }
+                        return more;
+                    }
+                   context = spec->context_stack[spec->context_stack_top];
+                    skip_ptr = *ptr;
+                    last_rule = 0;
+                    last_ptr = start_ptr = *ptr;
+                    if (start_ptr > 0)
+                    {
+                        --start_ptr;
+                        c_prev = f_win_advance (spec, &start_ptr);
+                    }
+                }
+                else
+                {
+                    c_prev = f_win_advance (spec, &start_ptr);
+                    *ptr = start_ptr;
+                }
+                state = context->dfa->states[0];
+                break;
+            }
+            else if (c >= t->ch[0] && c <= t->ch[1])
+            {   /* transition ... */
+                state = context->dfa->states[t->to];
+                if (state->rule_no)
+                {
+                    if (c_prev == '\n')
+                    {
+                        last_rule = state->rule_no;
+                        last_ptr = *ptr;
+                    } 
+                    else if (state->rule_nno)
+                    {
+                        last_rule = state->rule_nno;
+                        last_ptr = *ptr;
+                    }
+                    more = 1;
+                }
+                break;
+            }
+            else
+                t++;
+    }
+    return more;
+}
+
+static data1_node *lexRoot (struct lexSpec *spec, off_t offset,
+                           const char *context_name)
+{
+    struct lexContext *lt = spec->context;
+    int ptr = offset;
+    int ret;
+
+    spec->stop_flag = 0;
+    spec->d1_level = 0;
+    spec->context_stack_top = 0;    
+    while (lt)
+    {
+       if (!strcmp (lt->name, context_name))
+           break;
+       lt = lt->next;
+    }
+    if (!lt)
+    {
+       yaz_log (YLOG_WARN, "cannot find context %s", context_name);
+       return NULL;
+    }
+    spec->context_stack[spec->context_stack_top] = lt;
+    spec->d1_stack[spec->d1_level] = NULL;
+#if 1
+    if (!lt->initFlag)
+    {
+       lt->initFlag = 1;
+       execAction (spec, lt->initActionList, ptr, &ptr);
+    }
+#endif
+    execAction (spec, lt->beginActionList, ptr, &ptr);
+
+    ret = lexNode (spec, &ptr);
+    while (spec->d1_level)
+    {
+       tagDataRelease (spec);
+       (spec->d1_level)--;
+    }
+    if (!ret)
+        return 0;
+    execAction (spec, lt->endActionList, ptr, &ptr);
+    return spec->d1_stack[0];
+}
+
+void grs_destroy(void *clientData)
+{
+    struct lexSpecs *specs = (struct lexSpecs *) clientData;
+    if (specs->spec)
+    {
+       lexSpecDestroy(&specs->spec);
+    }
+    xfree (specs);
+}
+
+void *grs_init(Res res, RecType recType)
+{
+    struct lexSpecs *specs = (struct lexSpecs *) xmalloc (sizeof(*specs));
+    specs->spec = 0;
+    strcpy(specs->type, "");
+    return specs;
+}
+
+
+ZEBRA_RES grs_config(void *clientData, Res res, const char *args)
+{
+    struct lexSpecs *specs = (struct lexSpecs *) clientData;
+    if (strlen(args) < sizeof(specs->type))
+       strcpy(specs->type, args);
+    return ZEBRA_OK;
+}
+
+data1_node *grs_read_regx (struct grs_read_info *p)
+{
+    int res;
+    struct lexSpecs *specs = (struct lexSpecs *) p->clientData;
+    struct lexSpec **curLexSpec = &specs->spec;
+    off_t start_offset;
+
+#if REGX_DEBUG
+    yaz_log (YLOG_LOG, "grs_read_regx");
+#endif
+    if (!*curLexSpec || strcmp ((*curLexSpec)->name, specs->type))
+    {
+        if (*curLexSpec)
+            lexSpecDestroy (curLexSpec);
+        *curLexSpec = lexSpecCreate (specs->type, p->dh);
+        res = readFileSpec (*curLexSpec);
+        if (res)
+        {
+            lexSpecDestroy (curLexSpec);
+            return NULL;
+        }
+    }
+    (*curLexSpec)->dh = p->dh;
+    start_offset = p->stream->tellf(p->stream);
+    if (start_offset == 0)
+    {
+        (*curLexSpec)->f_win_start = 0;
+        (*curLexSpec)->f_win_end = 0;
+        (*curLexSpec)->f_win_rf = p->stream->readf;
+        (*curLexSpec)->f_win_sf = p->stream->seekf;
+        (*curLexSpec)->stream = p->stream;
+        (*curLexSpec)->f_win_ef = p->stream->endf;
+        (*curLexSpec)->f_win_size = 500000;
+    }
+    (*curLexSpec)->m = p->mem;
+    return lexRoot (*curLexSpec, start_offset, "main");
+}
+
+static int extract_regx(void *clientData, struct recExtractCtrl *ctrl)
+{
+    return zebra_grs_extract(clientData, ctrl, grs_read_regx);
+}
+
+static int retrieve_regx(void *clientData, struct recRetrieveCtrl *ctrl)
+{
+    return zebra_grs_retrieve(clientData, ctrl, grs_read_regx);
+}
+
+static struct recType regx_type = {
+    0,
+    "grs.regx",
+    grs_init,
+    grs_config,
+    grs_destroy,
+    extract_regx,
+    retrieve_regx,
+};
+
+
+#if HAVE_TCL_H
+data1_node *grs_read_tcl (struct grs_read_info *p)
+{
+    int res;
+    struct lexSpecs *specs = (struct lexSpecs *) p->clientData;
+    struct lexSpec **curLexSpec = &specs->spec;
+    off_t start_offset;
+
+#if REGX_DEBUG
+    yaz_log (YLOG_LOG, "grs_read_tcl");
+#endif
+    if (!*curLexSpec || strcmp ((*curLexSpec)->name, specs->type))
+    {
+       Tcl_Interp *tcl_interp;
+        if (*curLexSpec)
+            lexSpecDestroy (curLexSpec);
+        *curLexSpec = lexSpecCreate (specs->type, p->dh);
+       Tcl_FindExecutable("");
+       tcl_interp = (*curLexSpec)->tcl_interp = Tcl_CreateInterp();
+       Tcl_Init(tcl_interp);
+       Tcl_CreateCommand (tcl_interp, "begin", cmd_tcl_begin, *curLexSpec, 0);
+       Tcl_CreateCommand (tcl_interp, "end", cmd_tcl_end, *curLexSpec, 0);
+       Tcl_CreateCommand (tcl_interp, "data", cmd_tcl_data, *curLexSpec, 0);
+       Tcl_CreateCommand (tcl_interp, "unread", cmd_tcl_unread,
+                          *curLexSpec, 0);
+        res = readFileSpec (*curLexSpec);
+        if (res)
+        {
+            lexSpecDestroy (curLexSpec);
+            return NULL;
+        }
+    }
+    (*curLexSpec)->dh = p->dh;
+    start_offset = p->stream->tellf(p->stream);
+    if (start_offset == 0)
+    {
+        (*curLexSpec)->f_win_start = 0;
+        (*curLexSpec)->f_win_end = 0;
+        (*curLexSpec)->f_win_rf = p->stream->readf;
+        (*curLexSpec)->f_win_sf = p->stream->seekf;
+        (*curLexSpec)->stream = p->stream;
+        (*curLexSpec)->f_win_ef = p->stream->endf;
+        (*curLexSpec)->f_win_size = 500000;
+    }
+    (*curLexSpec)->m = p->mem;
+    return lexRoot (*curLexSpec, start_offset, "main");
+}
+
+static int extract_tcl(void *clientData, struct recExtractCtrl *ctrl)
+{
+    return zebra_grs_extract(clientData, ctrl, grs_read_tcl);
+}
+
+static int retrieve_tcl(void *clientData, struct recRetrieveCtrl *ctrl)
+{
+    return zebra_grs_retrieve(clientData, ctrl, grs_read_tcl);
+}
+
+static struct recType tcl_type = {
+    0,
+    "grs.tcl",
+    grs_init,
+    grs_config,
+    grs_destroy,
+    extract_tcl,
+    retrieve_tcl,
+};
+
+#endif
+
+RecType
+#ifdef IDZEBRA_STATIC_GRS_REGX
+idzebra_filter_grs_regx
+#else
+idzebra_filter
+#endif
+
+[] = {
+    &regx_type,
+#if HAVE_TCL_H
+    &tcl_type,
+#endif
+    0,
+};
+/*
+ * Local variables:
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ * vim: shiftwidth=4 tabstop=8 expandtab
+ */
+
diff --git a/index/mod_grs_sgml.c b/index/mod_grs_sgml.c
new file mode 100644 (file)
index 0000000..38fcbe3
--- /dev/null
@@ -0,0 +1,143 @@
+/* This file is part of the Zebra server.
+   Copyright (C) 1995-2008 Index Data
+
+Zebra is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2, or (at your option) any later
+version.
+
+Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+
+#include <assert.h>
+#include <yaz/log.h>
+
+#include <idzebra/recgrs.h>
+
+struct sgml_getc_info {
+    char *buf;
+    int buf_size;
+    int size;
+    int off;
+    struct ZebraRecStream *stream;
+    off_t moffset;
+    WRBUF wrbuf;
+};
+
+int sgml_getc (void *clientData)
+{
+    struct sgml_getc_info *p = (struct sgml_getc_info *) clientData;
+    int res;
+    
+    if (p->off < p->size)
+       return p->buf[(p->off)++];
+    if (p->size < p->buf_size)
+       return 0;
+    p->moffset += p->off;
+    p->off = 0;
+    p->size = 0;
+    res = p->stream->readf(p->stream, p->buf, p->buf_size);
+    if (res > 0)
+    {
+       p->size += res;
+       return p->buf[(p->off)++];
+    }
+    return 0;
+}
+
+static data1_node *grs_read_sgml (struct grs_read_info *p)
+{
+    struct sgml_getc_info *sgi = (struct sgml_getc_info *) p->clientData;
+    data1_node *node;
+    int res;
+    
+    sgi->moffset = p->stream->tellf(p->stream);
+    sgi->stream = p->stream;
+    sgi->off = 0;
+    sgi->size = 0;
+    res = sgi->stream->readf(sgi->stream, sgi->buf, sgi->buf_size);
+    if (res > 0)
+       sgi->size += res;
+    else
+       return 0;
+    node = data1_read_nodex(p->dh, p->mem, sgml_getc, sgi, sgi->wrbuf);
+    if (node && p->stream->endf)
+    {
+        off_t end_offset = sgi->moffset + sgi->off;
+       p->stream->endf(sgi->stream, &end_offset);
+    }
+    return node;
+}
+
+static void *grs_init_sgml(Res res, RecType recType)
+{
+    struct sgml_getc_info *p = (struct sgml_getc_info *) xmalloc (sizeof(*p));
+    p->buf_size = 512;
+    p->buf = xmalloc (p->buf_size);
+    p->wrbuf = wrbuf_alloc();
+    return p;
+}
+
+static ZEBRA_RES grs_config_sgml(void *clientData, Res res, const char *args)
+{
+    return ZEBRA_OK;
+}
+
+static void grs_destroy_sgml(void *clientData)
+{
+    struct sgml_getc_info *p = (struct sgml_getc_info *) clientData;
+
+    wrbuf_destroy(p->wrbuf);
+    xfree(p->buf);
+    xfree(p);
+}
+
+static int grs_extract_sgml(void *clientData, struct recExtractCtrl *ctrl)
+{
+    return zebra_grs_extract(clientData, ctrl, grs_read_sgml);
+}
+
+static int grs_retrieve_sgml(void *clientData, struct recRetrieveCtrl *ctrl)
+{
+    return zebra_grs_retrieve(clientData, ctrl, grs_read_sgml);
+}
+
+static struct recType grs_type_sgml =
+{
+    0,
+    "grs.sgml",
+    grs_init_sgml,
+    grs_config_sgml,
+    grs_destroy_sgml,
+    grs_extract_sgml,
+    grs_retrieve_sgml
+};
+
+RecType
+#ifdef IDZEBRA_STATIC_GRS_SGML
+idzebra_filter_grs_sgml
+#else
+idzebra_filter
+#endif
+
+[] = {
+    &grs_type_sgml,
+    0,
+};
+/*
+ * Local variables:
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ * vim: shiftwidth=4 tabstop=8 expandtab
+ */
+
diff --git a/index/mod_grs_xml.c b/index/mod_grs_xml.c
new file mode 100644 (file)
index 0000000..248342b
--- /dev/null
@@ -0,0 +1,547 @@
+/* This file is part of the Zebra server.
+   Copyright (C) 1995-2008 Index Data
+
+Zebra is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2, or (at your option) any later
+version.
+
+Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+#if HAVE_EXPAT_H
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#if HAVE_ICONV_H
+#include <errno.h>
+#include <iconv.h>
+#endif
+
+#include <yaz/log.h>
+
+#include <idzebra/recgrs.h>
+
+#include <yaz/log.h>
+#include <yaz/xmalloc.h>
+
+#include <expat.h>
+
+#define XML_CHUNK 1024
+
+struct user_info {
+    data1_node *d1_stack[256];
+    int level;
+    data1_handle dh;
+    NMEM nmem;
+    int loglevel;
+};
+
+static void cb_start (void *user, const char *el, const char **attr)
+{
+    struct user_info *ui = (struct user_info*) user;
+    if (ui->level == 1)
+        data1_set_root (ui->dh, ui->d1_stack[0], ui->nmem, el);
+    ui->d1_stack[ui->level] = data1_mk_tag (ui->dh, ui->nmem, el, attr,
+                                                ui->d1_stack[ui->level-1]);
+    ui->level++;
+    yaz_log (ui->loglevel, "cb_start %s", el);
+}
+
+static void cb_end (void *user, const char *el)
+{
+    struct user_info *ui = (struct user_info*) user;
+
+    ui->level--;
+    yaz_log (ui->loglevel, "cb_end %s", el);
+}
+
+static void cb_chardata (void *user, const char *s, int len)
+{
+    struct user_info *ui = (struct user_info*) user;
+#if 0
+    yaz_log (ui->loglevel, "cb_chardata %.*s", len, s);
+#endif
+    ui->d1_stack[ui->level] = data1_mk_text_n (ui->dh, ui->nmem, s, len,
+                                                   ui->d1_stack[ui->level -1]);
+}
+
+static void cb_decl (void *user, const char *version, const char *encoding,
+                     int standalone)
+{
+    struct user_info *ui = (struct user_info*) user;
+    const char *attr_list[7];
+
+    attr_list[0] = "version";
+    attr_list[1] = version;
+
+    attr_list[2] = "encoding";
+    attr_list[3] = "UTF-8"; /* internally it's always UTF-8 */
+
+    attr_list[4] = "standalone";
+    attr_list[5] = standalone  ? "yes" : "no";
+
+    attr_list[6] = 0;
+    
+    data1_mk_preprocess (ui->dh, ui->nmem, "xml", attr_list,
+                             ui->d1_stack[ui->level-1]);
+#if 0
+    yaz_log (YLOG_LOG, "decl version=%s encoding=%s",
+             version ? version : "null",
+             encoding ? encoding : "null");
+#endif
+}
+    
+static void cb_processing (void *user, const char *target,
+                           const char *data)
+{
+    struct user_info *ui = (struct user_info*) user;
+    data1_node *res =
+        data1_mk_preprocess (ui->dh, ui->nmem, target, 0,
+                             ui->d1_stack[ui->level-1]);
+    data1_mk_text_nf (ui->dh, ui->nmem, data, strlen(data), res);
+    
+    yaz_log (ui->loglevel, "decl processing target=%s data=%s",
+             target ? target : "null",
+             data ? data : "null");
+}
+
+static void cb_comment (void *user, const char *data)
+{
+    struct user_info *ui = (struct user_info*) user;
+    yaz_log (ui->loglevel, "decl comment data=%s", data ? data : "null");
+    data1_mk_comment (ui->dh, ui->nmem, data, ui->d1_stack[ui->level-1]);
+}
+
+static void cb_doctype_start (void *userData, const char *doctypeName,
+                              const char *sysid, const char *pubid,
+                              int has_internal_subset)
+{
+    struct user_info *ui = (struct user_info*) userData;
+    yaz_log (ui->loglevel, "doctype start doctype=%s sysid=%s pubid=%s",
+             doctypeName, sysid, pubid);
+}
+
+static void cb_doctype_end (void *userData)
+{
+    struct user_info *ui = (struct user_info*) userData;
+    yaz_log (ui->loglevel, "doctype end");
+}
+
+
+static void cb_entity_decl (void *userData, const char *entityName,
+                            int is_parameter_entity,
+                            const char *value, int value_length,
+                            const char *base, const char *systemId,
+                            const char *publicId, const char *notationName)
+{
+    struct user_info *ui = (struct user_info*) userData;
+    yaz_log (ui->loglevel,
+             "entity decl %s is_para_entry=%d value=%.*s base=%s systemId=%s"
+             " publicId=%s notationName=%s",
+             entityName, is_parameter_entity, value_length, value,
+             base, systemId, publicId, notationName);
+    
+}
+
+static int cb_external_entity(XML_Parser pparser,
+                              const char *context,
+                              const char *base,
+                              const char *systemId,
+                              const char *publicId)
+{
+    struct user_info *ui = (struct user_info*) XML_GetUserData(pparser);
+    FILE *inf;
+    int done = 0;
+    XML_Parser parser;
+
+    yaz_log (ui->loglevel,
+             "external entity context=%s base=%s systemid=%s publicid=%s",
+             context, base, systemId, publicId);
+    if (!systemId)
+        return 1;
+
+    if (!(inf = fopen (systemId, "rb")))
+    {
+        yaz_log (YLOG_WARN|YLOG_ERRNO, "fopen %s", systemId);
+        return 0;
+    }
+
+    parser = XML_ExternalEntityParserCreate (pparser, "", 0);
+    while (!done)
+    {
+        int r;
+        void *buf = XML_GetBuffer (parser, XML_CHUNK);
+        if (!buf)
+        {
+            yaz_log (YLOG_WARN, "XML_GetBuffer fail");
+            break;
+        }
+        r = fread (buf, 1, XML_CHUNK, inf);
+        if (r == 0)
+        {
+            if (ferror(inf))
+            {
+                yaz_log (YLOG_WARN|YLOG_ERRNO, "fread %s", systemId);
+                break;
+            }
+            done = 1;
+        }
+        if (!XML_ParseBuffer (parser, r, done))
+        {
+           done = 1;
+           yaz_log (YLOG_WARN, "%s:%d:%d:XML error: %s",
+                    systemId,
+                    XML_GetCurrentLineNumber(parser),
+                    XML_GetCurrentColumnNumber(parser),
+                    XML_ErrorString(XML_GetErrorCode(parser)));
+       }
+    }
+    fclose (inf);
+    XML_ParserFree (parser);
+    return done;
+}
+
+
+#if HAVE_ICONV_H
+static int cb_encoding_convert (void *data, const char *s)
+{
+    iconv_t t = (iconv_t) data;
+    size_t ret;
+    size_t outleft = 2;
+    char outbuf_[2], *outbuf = outbuf_;
+    size_t inleft = 4;
+    char *inbuf = (char *) s;
+    unsigned short code;
+
+#if 1
+    yaz_log(YLOG_LOG, "------------------------- cb_encoding_convert --- ");
+#endif
+    ret = iconv (t, &inbuf, &inleft, &outbuf, &outleft);
+    if (ret == (size_t) (-1) && errno != E2BIG)
+    {
+        iconv (t, 0, 0, 0, 0);
+        return -1;
+    }
+    if (outleft != 0)
+        return -1;
+    memcpy (&code, outbuf_, sizeof(short));
+    return code;
+}
+
+static void cb_encoding_release (void *data)
+{
+    iconv_t t = (iconv_t) data;
+    iconv_close (t);
+}
+
+static int cb_encoding_handler (void *userData, const char *name,
+                                XML_Encoding *info)
+{
+    int i = 0;
+    int no_ok = 0;
+    struct user_info *ui = (struct user_info*) userData;
+
+    iconv_t t = iconv_open ("UNICODE", name);
+    if (t == (iconv_t) (-1))
+        return 0;
+   
+    info->data = 0;  /* signal that multibyte is not in use */
+    yaz_log (ui->loglevel, "Encoding handler of %s", name);
+    for (i = 0; i<256; i++)
+    {
+        size_t ret;
+        char outbuf_[5];
+        char inbuf_[5];
+        char *inbuf = inbuf_;
+        char *outbuf = outbuf_;
+        size_t inleft = 1;
+        size_t outleft = 2;
+        inbuf_[0] = i;
+
+        iconv (t, 0, 0, 0, 0);  /* reset iconv */
+
+        ret = iconv(t, &inbuf, &inleft, &outbuf, &outleft);
+        if (ret == (size_t) (-1))
+        {
+            if (errno == EILSEQ)
+            {
+                yaz_log (ui->loglevel, "Encoding %d: invalid sequence", i);
+                info->map[i] = -1;  /* invalid sequence */
+            }
+            if (errno == EINVAL)
+            {                       /* multi byte input */
+                int len = 2;
+                int j = 0;
+                info->map[i] = -1;
+                
+                while (len <= 4)
+                {
+                    char sbuf[80];
+                    int k;
+                    inbuf = inbuf_;
+                    inleft = len;
+                    outbuf = outbuf_;
+                    outleft = 2;
+
+                    inbuf_[len-1] = j;
+                    iconv (t, 0,0,0,0);
+
+                    assert (i >= 0 && i<255);
+
+                    *sbuf = 0;
+                    for (k = 0; k<len; k++)
+                    {
+                        sprintf (sbuf+strlen(sbuf), "%d ", inbuf_[k]&255);
+                    }
+                    ret = iconv (t, &inbuf, &inleft, &outbuf, &outleft);
+                    if (ret == (size_t) (-1))
+                    {
+                        if (errno == EILSEQ || errno == E2BIG)
+                        {
+                            j++;
+                            if (j > 255)
+                                break;
+                        }
+                        else if (errno == EINVAL)
+                        {
+                            len++;
+                            j = 7;
+                        }
+                    }
+                    else if (outleft == 0)
+                    {
+                        info->map[i] = -len;
+                        info->data = t;  /* signal that multibyte is in use */
+                        break;
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+                if (info->map[i] < -1)
+                    yaz_log (ui->loglevel, "Encoding %d: multibyte input %d",
+                             i, -info->map[i]);
+                else
+                    yaz_log (ui->loglevel, "Encoding %d: multibyte input failed",
+                             i);
+            }
+            if (errno == E2BIG)
+            {
+                info->map[i] = -1;  /* no room for output */
+                if (i != 0)
+                    yaz_log (YLOG_WARN, "Encoding %d: no room for output",
+                             i);
+            }
+        }
+        else if (outleft == 0)
+        {
+            unsigned short code;
+            memcpy (&code, outbuf_, sizeof(short));
+            info->map[i] = code;
+            no_ok++;
+        }
+        else
+        {   /* should never happen */
+            info->map[i] = -1;
+            yaz_log (YLOG_DEBUG, "Encoding %d: bad state", i);
+        }
+    }
+    if (info->data)
+    {   /* at least one multi byte */
+        info->convert = cb_encoding_convert;
+        info->release = cb_encoding_release;
+    }
+    else
+    {
+        /* no multi byte - we no longer need iconv handler */
+        iconv_close(t);
+        info->convert = 0;
+        info->release = 0;
+    }
+    if (!no_ok)
+        return 0;
+    return 1;
+}
+/* HAVE_ICONV_H */
+#endif
+
+static void cb_ns_start(void *userData, const char *prefix, const char *uri)
+{
+    struct user_info *ui = (struct user_info*) userData;
+    if (prefix && uri)
+       yaz_log(ui->loglevel, "cb_ns_start %s %s", prefix, uri);
+}
+
+static void cb_ns_end(void *userData, const char *prefix)
+{
+    struct user_info *ui = (struct user_info*) userData;
+    if (prefix)
+       yaz_log(ui->loglevel, "cb_ns_end %s", prefix);
+}
+
+data1_node *zebra_read_xml(data1_handle dh,
+                           struct ZebraRecStream *stream,
+                           NMEM m)
+{
+    XML_Parser parser;
+    struct user_info uinfo;
+    int done = 0;
+    data1_node *first_node;
+    int no_read = 0;
+
+    uinfo.loglevel = YLOG_DEBUG;
+    uinfo.level = 1;
+    uinfo.dh = dh;
+    uinfo.nmem = m;
+    uinfo.d1_stack[0] = data1_mk_node2 (dh, m, DATA1N_root, 0);
+    uinfo.d1_stack[1] = 0; /* indicate no children (see end of routine) */
+    
+    parser = XML_ParserCreate (0 /* encoding */);
+    
+    XML_SetElementHandler (parser, cb_start, cb_end);
+    XML_SetCharacterDataHandler (parser, cb_chardata);
+    XML_SetXmlDeclHandler (parser, cb_decl);
+    XML_SetProcessingInstructionHandler (parser, cb_processing);
+    XML_SetUserData (parser, &uinfo);
+    XML_SetCommentHandler (parser, cb_comment);
+    XML_SetDoctypeDeclHandler (parser, cb_doctype_start, cb_doctype_end);
+    XML_SetEntityDeclHandler (parser, cb_entity_decl);
+    XML_SetExternalEntityRefHandler (parser, cb_external_entity);
+    XML_SetNamespaceDeclHandler(parser, cb_ns_start, cb_ns_end);
+#if HAVE_ICONV_H
+    XML_SetUnknownEncodingHandler (parser, cb_encoding_handler, &uinfo);
+#endif
+    while (!done)
+    {
+        int r;
+        void *buf = XML_GetBuffer (parser, XML_CHUNK);
+        if (!buf)
+        {
+            /* error */
+            yaz_log (YLOG_WARN, "XML_GetBuffer fail");
+            break;
+        }
+        r = stream->readf(stream, buf, XML_CHUNK);
+        if (r < 0)
+        {
+            /* error */
+            yaz_log (YLOG_WARN, "XML read fail");
+            break;
+        }
+        else if (r == 0)
+            done = 1;
+       else
+           no_read += r;
+        if (no_read && !XML_ParseBuffer (parser, r, done))
+        {
+           done = 1;
+           yaz_log (YLOG_WARN, "%d:%d:XML error: %s",
+                    XML_GetCurrentLineNumber(parser),
+                    XML_GetCurrentColumnNumber(parser),
+                    XML_ErrorString(XML_GetErrorCode(parser)));
+       }
+    }
+    XML_ParserFree (parser);
+    if (no_read == 0)
+       return 0;
+    if (!uinfo.d1_stack[1] || !done)
+        return 0;
+    /* insert XML header if not present .. */
+    first_node = uinfo.d1_stack[0]->child;
+    if (first_node->which != DATA1N_preprocess ||
+       strcmp(first_node->u.preprocess.target, "xml"))
+    {
+        const char *attr_list[5];
+
+        attr_list[0] = "version";
+        attr_list[1] = "1.0";
+
+        attr_list[2] = "encoding";
+        attr_list[3] = "UTF-8"; /* encoding */
+
+        attr_list[4] = 0;
+    
+        data1_insert_preprocess (uinfo.dh, uinfo.nmem, "xml", attr_list,
+                                uinfo.d1_stack[0]);
+    }
+    return uinfo.d1_stack[0];
+}
+
+struct xml_info {
+    XML_Expat_Version expat_version;
+};
+
+static data1_node *grs_read_xml(struct grs_read_info *p)
+{
+    return zebra_read_xml(p->dh, p->stream, p->mem);
+}
+
+static void *filter_init(Res res, RecType recType)
+{
+    struct xml_info *p = (struct xml_info *) xmalloc (sizeof(*p));
+
+    p->expat_version = XML_ExpatVersionInfo();
+
+    return p;
+}
+
+static void filter_destroy(void *clientData)
+{
+    struct xml_info *p = (struct xml_info *) clientData;
+
+    xfree (p);
+}
+
+static int filter_extract(void *clientData, struct recExtractCtrl *ctrl)
+{
+    return zebra_grs_extract(clientData, ctrl, grs_read_xml);
+}
+
+static int filter_retrieve(void *clientData, struct recRetrieveCtrl *ctrl)
+{
+    return zebra_grs_retrieve(clientData, ctrl, grs_read_xml);
+}
+
+static struct recType filter_type = {
+    0,
+    "grs.xml",
+    filter_init,
+    0,
+    filter_destroy,
+    filter_extract,
+    filter_retrieve,
+};
+
+RecType
+#ifdef IDZEBRA_STATIC_GRS_XML
+idzebra_filter_grs_xml
+#else
+idzebra_filter
+#endif
+
+[] = {
+    &filter_type,
+    0,
+};
+    
+#endif
+
+/*
+ * Local variables:
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ * vim: shiftwidth=4 tabstop=8 expandtab
+ */
+
diff --git a/index/mod_safari.c b/index/mod_safari.c
new file mode 100644 (file)
index 0000000..be8c567
--- /dev/null
@@ -0,0 +1,320 @@
+/* This file is part of the Zebra server.
+   Copyright (C) 1995-2008 Index Data
+
+Zebra is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2, or (at your option) any later
+version.
+
+Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+#include <yaz/oid_db.h>
+#include <stdio.h>
+#include <assert.h>
+#include <ctype.h>
+
+#include <idzebra/util.h>
+#include <idzebra/recctrl.h>
+
+struct filter_info {
+    int segments;
+};
+
+static void *filter_init(Res res, RecType recType)
+{
+    struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo));
+    tinfo->segments = 0;
+    return tinfo;
+}
+
+static void *filter_init2(Res res, RecType recType)
+{
+    struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo));
+    tinfo->segments = 1;
+    return tinfo;
+}
+
+static ZEBRA_RES filter_config(void *clientData, Res res, const char *args)
+{
+    return ZEBRA_OK;
+}
+
+static void filter_destroy(void *clientData)
+{
+    struct filter_info *tinfo = clientData;
+    xfree (tinfo);
+}
+
+struct fi_info {
+    struct recExtractCtrl *p;
+    char *buf;
+    int offset;
+    int max;
+};
+
+static struct fi_info *fi_open(struct recExtractCtrl *p)
+{
+    struct fi_info *fi = (struct fi_info *) xmalloc (sizeof(*fi));
+
+    fi->p = p;
+    fi->buf = (char *) xmalloc (4096);
+    fi->offset = 1;
+    fi->max = 1;
+    return fi;
+}
+
+static int fi_getchar(struct fi_info *fi, char *dst)
+{
+    if (fi->offset >= fi->max)
+    {
+        if (fi->max <= 0)
+            return 0;
+        fi->max = fi->p->stream->readf(fi->p->stream, fi->buf, 4096);
+        fi->offset = 0;
+        if (fi->max <= 0)
+            return 0;
+    }
+    *dst = fi->buf[(fi->offset)++];
+    return 1;
+}
+
+static int fi_gets(struct fi_info *fi, char *dst, int max)
+{
+    int l = 0;
+    while(1)
+    {
+       char dstbyte;
+       if (!fi_getchar(fi, &dstbyte))
+           return 0;
+       if (dstbyte == '\n')
+           break;
+       if (l < max)
+           dst[l++] = dstbyte;
+    }
+    dst[l] = '\0';
+    return 1;
+}
+
+static void fi_close (struct fi_info *fi)
+{
+    xfree (fi->buf);
+    xfree (fi);
+}
+
+static int filter_extract(void *clientData, struct recExtractCtrl *p)
+{
+    struct filter_info *tinfo = clientData;
+    char line[512];
+    RecWord recWord;
+    int ret = RECCTRL_EXTRACT_OK;
+    struct fi_info *fi = fi_open(p);
+
+#if 0
+    yaz_log(YLOG_LOG, "filter_extract off=%ld",
+           (long) (*fi->p->tellf)(fi->p->fh));
+#endif
+    (*p->init)(p, &recWord);
+
+    if (!fi_gets(fi, line, sizeof(line)-1))
+        ret = RECCTRL_EXTRACT_EOF;
+    else
+    {
+        sscanf(line, "%255s", p->match_criteria);
+        while (fi_gets(fi, line, sizeof(line)-1))
+        {
+            int nor = 0;
+            char field[40];
+            const char *cp = line;
+            char type_cstr[2];
+#if 0
+            yaz_log(YLOG_LOG, "safari line: %s", line);
+#endif
+            type_cstr[1] = '\0';
+            if (*cp >= '0' && *cp <= '9')
+                type_cstr[0] = '0'; /* the default is 0 (raw) */
+            else
+                type_cstr[0] = *cp++; /* type given */
+            type_cstr[1] = '\0';
+
+            recWord.index_type = type_cstr;
+            if (tinfo->segments)
+            {
+                if (sscanf(cp, ZINT_FORMAT " " ZINT_FORMAT " " ZINT_FORMAT 
+                           ZINT_FORMAT " %39s %n",
+                           &recWord.record_id, &recWord.section_id, 
+                           &recWord.segment,
+                           &recWord.seqno,
+                           field, &nor) < 5)
+                {
+                    yaz_log(YLOG_WARN, "Bad safari record line: %s", line);
+                    ret = RECCTRL_EXTRACT_ERROR_GENERIC;
+                    break;
+                }
+            }
+            else
+            {
+                if (sscanf(cp, ZINT_FORMAT " " ZINT_FORMAT " " ZINT_FORMAT " %39s %n",
+                           &recWord.record_id, &recWord.section_id, &recWord.seqno,
+                           field, &nor) < 4)
+                {
+                    yaz_log(YLOG_WARN, "Bad safari record line: %s", line);
+                    ret = RECCTRL_EXTRACT_ERROR_GENERIC;
+                    break;
+                }
+            }
+            for (cp = cp + nor; *cp == ' '; cp++)
+                ;
+            recWord.index_name = field;
+            recWord.term_buf = cp;
+            recWord.term_len = strlen(cp);
+            (*p->tokenAdd)(&recWord);
+        }
+    }
+    fi_close(fi);
+    return ret;
+}
+
+static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p)
+{
+    int r, filter_ptr = 0;
+    static char *filter_buf = NULL;
+    static int filter_size = 0;
+    int make_header = 1;
+    int make_body = 1;
+    const char *elementSetName = NULL;
+    int no_lines = 0;
+
+    if (p->comp && p->comp->which == Z_RecordComp_simple &&
+        p->comp->u.simple->which == Z_ElementSetNames_generic)
+        elementSetName = p->comp->u.simple->u.generic;
+
+    if (elementSetName)
+    {
+       /* don't make header for the R(aw) element set name */
+       if (!strcmp(elementSetName, "R"))
+       {
+           make_header = 0;
+           make_body = 1;
+       }
+       /* only make header for the H(eader) element set name */
+       else if (!strcmp(elementSetName, "H"))
+       {
+           make_header = 1;
+           make_body = 0;
+       }
+    }
+    while (1)
+    {
+        if (filter_ptr + 4096 >= filter_size)
+        {
+            char *nb;
+
+            filter_size = 2*filter_size + 8192;
+            nb = (char *) xmalloc (filter_size);
+            if (filter_buf)
+            {
+                memcpy (nb, filter_buf, filter_ptr);
+                xfree (filter_buf);
+            }
+            filter_buf = nb;
+        }
+        if (make_header && filter_ptr == 0)
+        {
+            if (p->score >= 0)
+            {
+                sprintf (filter_buf, "Rank: %d\n", p->score);
+                filter_ptr = strlen(filter_buf);
+            }
+            sprintf (filter_buf + filter_ptr, "Local Number: " ZINT_FORMAT "\n",
+                    p->localno);
+            filter_ptr = strlen(filter_buf);
+           if (p->fname)
+           {
+               sprintf (filter_buf + filter_ptr, "Filename: %s\n", p->fname);
+               filter_ptr = strlen(filter_buf);
+           }
+           strcpy(filter_buf+filter_ptr++, "\n");
+        }
+       if (!make_body)
+           break;
+        r = p->stream->readf(p->stream, filter_buf + filter_ptr, 4096);
+        if (r <= 0)
+            break;
+        filter_ptr += r;
+    }
+    filter_buf[filter_ptr] = '\0';
+    if (elementSetName)
+    {
+        if (!strcmp (elementSetName, "B"))
+            no_lines = 4;
+        if (!strcmp (elementSetName, "M"))
+            no_lines = 20;
+    }
+    if (no_lines)
+    {
+        char *p = filter_buf;
+        int i = 0;
+
+        while (++i <= no_lines && (p = strchr (p, '\n')))
+            p++;
+        if (p)
+        {
+            p[1] = '\0';
+            filter_ptr = p-filter_buf;
+        }
+    }
+    p->output_format = yaz_oid_recsyn_sutrs;
+    p->rec_buf = filter_buf;
+    p->rec_len = filter_ptr; 
+    return 0;
+}
+
+static struct recType filter_type = {
+    0,
+    "safari",
+    filter_init,
+    filter_config,
+    filter_destroy,
+    filter_extract,
+    filter_retrieve
+};
+
+static struct recType filter_type2 = {
+    0,
+    "safari2",
+    filter_init2,
+    filter_config,
+    filter_destroy,
+    filter_extract,
+    filter_retrieve
+};
+
+RecType
+#ifdef IDZEBRA_STATIC_SAFARI
+idzebra_filter_safari
+#else
+idzebra_filter
+#endif
+
+[] = {
+    &filter_type,
+    &filter_type2,
+    0,
+};
+/*
+ * Local variables:
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ * vim: shiftwidth=4 tabstop=8 expandtab
+ */
+
diff --git a/index/mod_text.c b/index/mod_text.c
new file mode 100644 (file)
index 0000000..cfa97c0
--- /dev/null
@@ -0,0 +1,264 @@
+/* This file is part of the Zebra server.
+   Copyright (C) 1995-2008 Index Data
+
+Zebra is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2, or (at your option) any later
+version.
+
+Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+*/
+
+
+#include <stdio.h>
+#include <assert.h>
+#include <ctype.h>
+
+#include <idzebra/util.h>
+#include <idzebra/recctrl.h>
+#include <yaz/oid_db.h>
+
+struct filter_info {
+    char *sep;
+};
+
+static void *filter_init(Res res, RecType recType)
+{
+    struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo));
+    tinfo->sep = 0;
+    return tinfo;
+}
+
+static ZEBRA_RES filter_config(void *clientData, Res res, const char *args)
+{
+    struct filter_info *tinfo = (struct filter_info*) clientData;
+    xfree(tinfo->sep);
+    tinfo->sep = 0;
+    if (args && *args)
+       tinfo->sep = xstrdup(args);
+    return ZEBRA_OK;
+}
+
+static void filter_destroy(void *clientData)
+{
+    struct filter_info *tinfo = clientData;
+    xfree(tinfo->sep);
+    xfree(tinfo);
+}
+
+struct buf_info {
+    struct recExtractCtrl *p;
+    char *buf;
+    int offset;
+    int max;
+};
+
+static struct buf_info *buf_open(struct recExtractCtrl *p)
+{
+    struct buf_info *fi = (struct buf_info *) xmalloc(sizeof(*fi));
+
+    fi->p = p;
+    fi->buf = (char *) xmalloc(4096);
+    fi->offset = 1;
+    fi->max = 1;
+    return fi;
+}
+
+static int buf_getchar(struct filter_info *tinfo, struct buf_info *fi, char *dst)
+{
+    if (fi->offset >= fi->max)
+    {
+        if (fi->max <= 0)
+            return 0;
+        fi->max = fi->p->stream->readf(fi->p->stream, fi->buf, 4096);
+        fi->offset = 0;
+        if (fi->max <= 0)
+            return 0;
+    }
+    *dst = fi->buf[(fi->offset)++];
+    if (tinfo->sep && *dst == *tinfo->sep)
+    {
+       off_t off = fi->p->stream->tellf(fi->p->stream);
+        off_t end_offset = off - (fi->max - fi->offset);
+       fi->p->stream->endf(fi->p->stream, &end_offset);
+       return 0;
+    }
+    return 1;
+}
+
+static void buf_close(struct buf_info *fi)
+{
+    xfree(fi->buf);
+    xfree(fi);
+}
+
+static int filter_extract(void *clientData, struct recExtractCtrl *p)
+{
+    struct filter_info *tinfo = clientData;
+    char w[512];
+    RecWord recWord;
+    int r;
+    struct buf_info *fi = buf_open(p);
+    int no_read = 0;
+
+#if 0
+    yaz_log(YLOG_LOG, "filter_extract off=%ld",
+           (long) (*fi->p->tellf)(fi->p->fh));
+#endif
+    (*p->init)(p, &recWord);
+    do
+    {
+        int i = 0;
+            
+        r = buf_getchar(tinfo, fi, w);
+        while (r > 0 && i < 511 && w[i] != '\n' && w[i] != '\r')
+        {
+            i++;
+            r = buf_getchar(tinfo, fi, w + i); 
+       }
+        if (i)
+        {
+           no_read += i;
+            recWord.term_buf = w;
+           recWord.term_len = i;
+            (*p->tokenAdd)(&recWord);
+        }
+    } while (r > 0);
+    buf_close(fi);
+    if (no_read == 0)
+       return RECCTRL_EXTRACT_EOF;
+    return RECCTRL_EXTRACT_OK;
+}
+
+static int filter_retrieve(void *clientData, struct recRetrieveCtrl *p)
+{
+    int r, filter_ptr = 0;
+    static char *filter_buf = NULL;
+    static int filter_size = 0;
+    int make_header = 1;
+    int make_body = 1;
+    const char *elementSetName = NULL;
+    int no_lines = 0;
+
+    if (p->comp && p->comp->which == Z_RecordComp_simple &&
+        p->comp->u.simple->which == Z_ElementSetNames_generic)
+        elementSetName = p->comp->u.simple->u.generic;
+
+    if (elementSetName)
+    {
+       /* don't make header for the R(aw) element set name */
+       if (!strcmp(elementSetName, "R"))
+       {
+           make_header = 0;
+           make_body = 1;
+       }
+       /* only make header for the H(eader) element set name */
+       else if (!strcmp(elementSetName, "H"))
+       {
+           make_header = 1;
+           make_body = 0;
+       }
+    }
+    while (1)
+    {
+        if (filter_ptr + 4096 >= filter_size)
+        {
+            char *nb;
+
+            filter_size = 2*filter_size + 8192;
+            nb = (char *) xmalloc(filter_size);
+            if (filter_buf)
+            {
+                memcpy(nb, filter_buf, filter_ptr);
+                xfree(filter_buf);
+            }
+            filter_buf = nb;
+        }
+        if (make_header && filter_ptr == 0)
+        {
+            if (p->score >= 0)
+            {
+                sprintf(filter_buf, "Rank: %d\n", p->score);
+                filter_ptr = strlen(filter_buf);
+            }
+            sprintf(filter_buf + filter_ptr, "Local Number: " ZINT_FORMAT "\n",
+                    p->localno);
+            filter_ptr = strlen(filter_buf);
+           if (p->fname)
+           {
+               sprintf(filter_buf + filter_ptr, "Filename: %s\n", p->fname);
+               filter_ptr = strlen(filter_buf);
+           }
+           strcpy(filter_buf+filter_ptr++, "\n");
+        }
+       if (!make_body)
+           break;
+        r = p->stream->readf(p->stream, filter_buf + filter_ptr, 4096);
+        if (r <= 0)
+            break;
+        filter_ptr += r;
+    }
+    filter_buf[filter_ptr] = '\0';
+    if (elementSetName)
+    {
+        if (!strcmp(elementSetName, "B"))
+            no_lines = 4;
+        if (!strcmp(elementSetName, "M"))
+            no_lines = 20;
+    }
+    if (no_lines)
+    {
+        char *p = filter_buf;
+        int i = 0;
+
+        while (++i <= no_lines && (p = strchr(p, '\n')))
+            p++;
+        if (p)
+        {
+            p[1] = '\0';
+            filter_ptr = p-filter_buf;
+        }
+    }
+    p->output_format = yaz_oid_recsyn_sutrs;
+    p->rec_buf = filter_buf;
+    p->rec_len = filter_ptr; 
+    return 0;
+}
+
+static struct recType filter_type = {
+    0,
+    "text",
+    filter_init,
+    filter_config,
+    filter_destroy,
+    filter_extract,
+    filter_retrieve
+};
+
+RecType
+#ifdef IDZEBRA_STATIC_TEXT
+idzebra_filter_text
+#else
+idzebra_filter
+#endif
+
+[] = {
+    &filter_type,
+    0,
+};
+/*
+ * Local variables:
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ * vim: shiftwidth=4 tabstop=8 expandtab
+ */
+
diff --git a/index/rectext.c b/index/rectext.c
deleted file mode 100644 (file)
index cfa97c0..0000000
+++ /dev/null
@@ -1,264 +0,0 @@
-/* This file is part of the Zebra server.
-   Copyright (C) 1995-2008 Index Data
-
-Zebra is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 2, or (at your option) any later
-version.
-
-Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-
-*/
-
-
-#include <stdio.h>
-#include <assert.h>
-#include <ctype.h>
-
-#include <idzebra/util.h>
-#include <idzebra/recctrl.h>
-#include <yaz/oid_db.h>
-
-struct filter_info {
-    char *sep;
-};
-
-static void *filter_init(Res res, RecType recType)
-{
-    struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo));
-    tinfo->sep = 0;
-    return tinfo;
-}
-
-static ZEBRA_RES filter_config(void *clientData, Res res, const char *args)
-{
-    struct filter_info *tinfo = (struct filter_info*) clientData;
-    xfree(tinfo->sep);
-    tinfo->sep = 0;
-    if (args && *args)
-       tinfo->sep = xstrdup(args);
-    return ZEBRA_OK;
-}
-
-static void filter_destroy(void *clientData)
-{
-    struct filter_info *tinfo = clientData;
-    xfree(tinfo->sep);
-    xfree(tinfo);
-}
-
-struct buf_info {
-    struct recExtractCtrl *p;
-    char *buf;
-    int offset;
-    int max;
-};
-
-static struct buf_info *buf_open(struct recExtractCtrl *p)
-{
-    struct buf_info *fi = (struct buf_info *) xmalloc(sizeof(*fi));
-
-    fi->p = p;
-    fi->buf = (char *) xmalloc(4096);
-    fi->offset = 1;
-    fi->max = 1;
-    return fi;
-}
-
-static int buf_getchar(struct filter_info *tinfo, struct buf_info *fi, char *dst)
-{
-    if (fi->offset >= fi->max)
-    {
-        if (fi->max <= 0)
-            return 0;
-        fi->max = fi->p->stream->readf(fi->p->stream, fi->buf, 4096);
-        fi->offset = 0;
-        if (fi->max <= 0)
-            return 0;
-    }
-    *dst = fi->buf[(fi->offset)++];
-    if (tinfo->sep && *dst == *tinfo->sep)
-    {
-       off_t off = fi->p->stream->tellf(fi->p->stream);
-        off_t end_offset = off - (fi->max - fi->offset);
-       fi->p->stream->endf(fi->p->stream, &end_offset);
-       return 0;
-    }
-    return 1;
-}
-
-static void buf_close(struct buf_info *fi)
-{
-    xfree(fi->buf);
-    xfree(fi);
-}
-
-static int filter_extract(void *clientData, struct recExtractCtrl *p)
-{
-    struct filter_info *tinfo = clientData;
-    char w[512];
-    RecWord recWord;
-    int r;
-    struct buf_info *fi = buf_open(p);
-    int no_read = 0;
-
-#if 0
-    yaz_log(YLOG_LOG, "filter_extract off=%ld",
-           (long) (*fi->p->tellf)(fi->p->fh));
-#endif
-    (*p->init)(p, &recWord);
-    do
-    {
-        int i = 0;
-            
-        r = buf_getchar(tinfo, fi, w);
-        while (r > 0 && i < 511 && w[i] != '\n' && w[i] != '\r')
-        {
-            i++;
-            r = buf_getchar(tinfo, fi, w + i); 
-       }
-        if (i)
-        {
-           no_read += i;
-            recWord.term_buf = w;
-           recWord.term_len = i;
-            (*p->tokenAdd)(&recWord);
-        }
-    } while (r > 0);
-    buf_close(fi);
-    if (no_read == 0)
-       return RECCTRL_EXTRACT_EOF;
-    return RECCTRL_EXTRACT_OK;
-}
-
-static int filter_retrieve(void *clientData, struct recRetrieveCtrl *p)
-{
-    int r, filter_ptr = 0;
-    static char *filter_buf = NULL;
-    static int filter_size = 0;
-    int make_header = 1;
-    int make_body = 1;
-    const char *elementSetName = NULL;
-    int no_lines = 0;
-
-    if (p->comp && p->comp->which == Z_RecordComp_simple &&
-        p->comp->u.simple->which == Z_ElementSetNames_generic)
-        elementSetName = p->comp->u.simple->u.generic;
-
-    if (elementSetName)
-    {
-       /* don't make header for the R(aw) element set name */
-       if (!strcmp(elementSetName, "R"))
-       {
-           make_header = 0;
-           make_body = 1;
-       }
-       /* only make header for the H(eader) element set name */
-       else if (!strcmp(elementSetName, "H"))
-       {
-           make_header = 1;
-           make_body = 0;
-       }
-    }
-    while (1)
-    {
-        if (filter_ptr + 4096 >= filter_size)
-        {
-            char *nb;
-
-            filter_size = 2*filter_size + 8192;
-            nb = (char *) xmalloc(filter_size);
-            if (filter_buf)
-            {
-                memcpy(nb, filter_buf, filter_ptr);
-                xfree(filter_buf);
-            }
-            filter_buf = nb;
-        }
-        if (make_header && filter_ptr == 0)
-        {
-            if (p->score >= 0)
-            {
-                sprintf(filter_buf, "Rank: %d\n", p->score);
-                filter_ptr = strlen(filter_buf);
-            }
-            sprintf(filter_buf + filter_ptr, "Local Number: " ZINT_FORMAT "\n",
-                    p->localno);
-            filter_ptr = strlen(filter_buf);
-           if (p->fname)
-           {
-               sprintf(filter_buf + filter_ptr, "Filename: %s\n", p->fname);
-               filter_ptr = strlen(filter_buf);
-           }
-           strcpy(filter_buf+filter_ptr++, "\n");
-        }
-       if (!make_body)
-           break;
-        r = p->stream->readf(p->stream, filter_buf + filter_ptr, 4096);
-        if (r <= 0)
-            break;
-        filter_ptr += r;
-    }
-    filter_buf[filter_ptr] = '\0';
-    if (elementSetName)
-    {
-        if (!strcmp(elementSetName, "B"))
-            no_lines = 4;
-        if (!strcmp(elementSetName, "M"))
-            no_lines = 20;
-    }
-    if (no_lines)
-    {
-        char *p = filter_buf;
-        int i = 0;
-
-        while (++i <= no_lines && (p = strchr(p, '\n')))
-            p++;
-        if (p)
-        {
-            p[1] = '\0';
-            filter_ptr = p-filter_buf;
-        }
-    }
-    p->output_format = yaz_oid_recsyn_sutrs;
-    p->rec_buf = filter_buf;
-    p->rec_len = filter_ptr; 
-    return 0;
-}
-
-static struct recType filter_type = {
-    0,
-    "text",
-    filter_init,
-    filter_config,
-    filter_destroy,
-    filter_extract,
-    filter_retrieve
-};
-
-RecType
-#ifdef IDZEBRA_STATIC_TEXT
-idzebra_filter_text
-#else
-idzebra_filter
-#endif
-
-[] = {
-    &filter_type,
-    0,
-};
-/*
- * Local variables:
- * c-basic-offset: 4
- * indent-tabs-mode: nil
- * End:
- * vim: shiftwidth=4 tabstop=8 expandtab
- */
-
diff --git a/index/regxread.c b/index/regxread.c
deleted file mode 100644 (file)
index ecf8f95..0000000
+++ /dev/null
@@ -1,2037 +0,0 @@
-/* This file is part of the Zebra server.
-   Copyright (C) 1995-2008 Index Data
-
-Zebra is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 2, or (at your option) any later
-version.
-
-Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include <ctype.h>
-
-#include <yaz/tpath.h>
-#include <idzebra/util.h>
-#include <dfa.h>
-#include <idzebra/recgrs.h>
-
-#if HAVE_TCL_H
-#include <tcl.h>
-
-#if MAJOR_VERSION >= 8
-#define HAVE_TCL_OBJECTS
-#endif
-#endif
-
-#define REGX_DEBUG 0
-
-#define F_WIN_EOF 2000000000
-#define F_WIN_READ 1
-
-#define REGX_EOF     0
-#define REGX_PATTERN 1
-#define REGX_BODY    2
-#define REGX_BEGIN   3
-#define REGX_END     4
-#define REGX_CODE    5
-#define REGX_CONTEXT 6
-#define REGX_INIT    7
-
-struct regxCode {
-    char *str;
-#if HAVE_TCL_OBJECTS
-    Tcl_Obj *tcl_obj;
-#endif
-};
-
-struct lexRuleAction {
-    int which; 
-    union {
-        struct {
-            struct DFA *dfa;    /* REGX_PATTERN */
-            int body;
-        } pattern;
-        struct regxCode *code;  /* REGX_CODE */
-    } u;
-    struct lexRuleAction *next;
-};
-
-struct lexRuleInfo {
-    int no;
-    struct lexRuleAction *actionList;
-};
-
-struct lexRule {
-    struct lexRuleInfo info;
-    struct lexRule *next;
-};
-
-struct lexContext {
-    char *name;
-    struct DFA *dfa;
-    struct lexRule *rules;
-    struct lexRuleInfo **fastRule;
-    int ruleNo;
-    int initFlag;
-
-    struct lexRuleAction *beginActionList;
-    struct lexRuleAction *endActionList;
-    struct lexRuleAction *initActionList;
-    struct lexContext *next;
-};
-
-struct lexConcatBuf {
-    int max;
-    char *buf;
-};
-
-struct lexSpec {
-    char *name;
-    struct lexContext *context;
-
-    struct lexContext **context_stack;
-    int context_stack_size;
-    int context_stack_top;
-
-    int lineNo;
-    NMEM m;
-    data1_handle dh;
-#if HAVE_TCL_H
-    Tcl_Interp *tcl_interp;
-#endif
-    struct ZebraRecStream *stream;
-    off_t (*f_win_ef)(struct ZebraRecStream *s, off_t *);
-
-    int f_win_start;      /* first byte of buffer is this file offset */
-    int f_win_end;        /* last byte of buffer is this offset - 1 */
-    int f_win_size;       /* size of buffer */
-    char *f_win_buf;      /* buffer itself */
-    int (*f_win_rf)(struct ZebraRecStream *, char *, size_t);
-    off_t (*f_win_sf)(struct ZebraRecStream *, off_t);
-
-    struct lexConcatBuf *concatBuf;
-    int maxLevel;
-    data1_node **d1_stack;
-    int d1_level;
-    int stop_flag;
-    
-    int *arg_start;
-    int *arg_end;
-    int arg_no;
-    int ptr;
-};
-
-struct lexSpecs {
-    struct lexSpec *spec;
-    char type[256];
-};
-
-static char *f_win_get (struct lexSpec *spec, off_t start_pos, off_t end_pos,
-                        int *size)
-{
-    int i, r, off = start_pos - spec->f_win_start;
-
-    if (off >= 0 && end_pos <= spec->f_win_end)
-    {
-        *size = end_pos - start_pos;
-        return spec->f_win_buf + off;
-    }
-    if (off < 0 || start_pos >= spec->f_win_end)
-    {
-        (*spec->f_win_sf)(spec->stream, start_pos);
-        spec->f_win_start = start_pos;
-
-        if (!spec->f_win_buf)
-            spec->f_win_buf = (char *) xmalloc (spec->f_win_size);
-        *size = (*spec->f_win_rf)(spec->stream, spec->f_win_buf,
-                                  spec->f_win_size);
-        spec->f_win_end = spec->f_win_start + *size;
-
-        if (*size > end_pos - start_pos)
-            *size = end_pos - start_pos;
-        return spec->f_win_buf;
-    }
-    for (i = 0; i<spec->f_win_end - start_pos; i++)
-        spec->f_win_buf[i] = spec->f_win_buf[i + off];
-    r = (*spec->f_win_rf)(spec->stream,
-                          spec->f_win_buf + i,
-                          spec->f_win_size - i);
-    spec->f_win_start = start_pos;
-    spec->f_win_end += r;
-    *size = i + r;
-    if (*size > end_pos - start_pos)
-        *size = end_pos - start_pos;
-    return spec->f_win_buf;
-}
-
-static int f_win_advance (struct lexSpec *spec, int *pos)
-{
-    int size;
-    char *buf;
-    
-    if (*pos >= spec->f_win_start && *pos < spec->f_win_end)
-        return spec->f_win_buf[(*pos)++ - spec->f_win_start];
-    if (*pos == F_WIN_EOF)
-        return 0;
-    buf = f_win_get (spec, *pos, *pos+1, &size);
-    if (size == 1)
-    {
-        (*pos)++;
-        return *buf;
-    }
-    *pos = F_WIN_EOF;
-    return 0;
-}
-
-static void regxCodeDel (struct regxCode **pp)
-{
-    struct regxCode *p = *pp;
-    if (p)
-    {
-#if HAVE_TCL_OBJECTS
-       if (p->tcl_obj)
-           Tcl_DecrRefCount (p->tcl_obj);
-#endif
-        xfree (p->str); 
-        xfree (p);
-        *pp = NULL;
-    }
-}
-
-static void regxCodeMk (struct regxCode **pp, const char *buf, int len)
-{
-    struct regxCode *p;
-
-    p = (struct regxCode *) xmalloc (sizeof(*p));
-    p->str = (char *) xmalloc (len+1);
-    memcpy (p->str, buf, len);
-    p->str[len] = '\0';
-#if HAVE_TCL_OBJECTS
-    p->tcl_obj = Tcl_NewStringObj ((char *) buf, len);
-    if (p->tcl_obj)
-       Tcl_IncrRefCount (p->tcl_obj);
-#endif
-    *pp = p;
-}
-
-static struct DFA *lexSpecDFA (void)
-{
-    struct DFA *dfa;
-
-    dfa = dfa_init ();
-    dfa_parse_cmap_del (dfa, ' ');
-    dfa_parse_cmap_del (dfa, '\t');
-    dfa_parse_cmap_add (dfa, '/', 0);
-    return dfa;
-}
-
-static void actionListDel (struct lexRuleAction **rap)
-{
-    struct lexRuleAction *ra1, *ra;
-
-    for (ra = *rap; ra; ra = ra1)
-    {
-        ra1 = ra->next;
-        switch (ra->which)
-        {
-        case REGX_PATTERN:
-            dfa_delete (&ra->u.pattern.dfa);
-            break;
-        case REGX_CODE:
-            regxCodeDel (&ra->u.code);
-            break;
-        }
-        xfree (ra);
-    }
-    *rap = NULL;
-}
-
-static struct lexContext *lexContextCreate (const char *name)
-{
-    struct lexContext *p = (struct lexContext *) xmalloc (sizeof(*p));
-
-    p->name = xstrdup (name);
-    p->ruleNo = 1;
-    p->initFlag = 0;
-    p->dfa = lexSpecDFA ();
-    p->rules = NULL;
-    p->fastRule = NULL;
-    p->beginActionList = NULL;
-    p->endActionList = NULL;
-    p->initActionList = NULL;
-    p->next = NULL;
-    return p;
-}
-
-static void lexContextDestroy (struct lexContext *p)
-{
-    struct lexRule *rp, *rp1;
-
-    dfa_delete (&p->dfa);
-    xfree (p->fastRule);
-    for (rp = p->rules; rp; rp = rp1)
-    {
-       rp1 = rp->next;
-        actionListDel (&rp->info.actionList);
-        xfree (rp);
-    }
-    actionListDel (&p->beginActionList);
-    actionListDel (&p->endActionList);
-    actionListDel (&p->initActionList);
-    xfree (p->name);
-    xfree (p);
-}
-
-static struct lexSpec *lexSpecCreate (const char *name, data1_handle dh)
-{
-    struct lexSpec *p;
-    int i;
-    
-    p = (struct lexSpec *) xmalloc (sizeof(*p));
-    p->name = (char *) xmalloc (strlen(name)+1);
-    strcpy (p->name, name);
-
-#if HAVE_TCL_H
-    p->tcl_interp = 0;
-#endif
-    p->dh = dh;
-    p->context = NULL;
-    p->context_stack_size = 100;
-    p->context_stack = (struct lexContext **)
-       xmalloc (sizeof(*p->context_stack) * p->context_stack_size);
-    p->f_win_buf = NULL;
-
-    p->maxLevel = 128;
-    p->concatBuf = (struct lexConcatBuf *)
-       xmalloc (sizeof(*p->concatBuf) * p->maxLevel);
-    for (i = 0; i < p->maxLevel; i++)
-    {
-       p->concatBuf[i].max = 0;
-       p->concatBuf[i].buf = 0;
-    }
-    p->d1_stack = (data1_node **) xmalloc (sizeof(*p->d1_stack) * p->maxLevel);
-    p->d1_level = 0;
-    return p;
-}
-
-static void lexSpecDestroy (struct lexSpec **pp)
-{
-    struct lexSpec *p;
-    struct lexContext *lt;
-    int i;
-
-    assert (pp);
-    p = *pp;
-    if (!p)
-        return ;
-
-    for (i = 0; i < p->maxLevel; i++)
-       xfree (p->concatBuf[i].buf);
-    xfree (p->concatBuf);
-
-    lt = p->context;
-    while (lt)
-    {
-       struct lexContext *lt_next = lt->next;
-       lexContextDestroy (lt);
-       lt = lt_next;
-    }
-#if HAVE_TCL_OBJECTS
-    if (p->tcl_interp)
-       Tcl_DeleteInterp (p->tcl_interp);
-#endif
-    xfree (p->name);
-    xfree (p->f_win_buf);
-    xfree (p->context_stack);
-    xfree (p->d1_stack);
-    xfree (p);
-    *pp = NULL;
-}
-
-static int readParseToken (const char **cpp, int *len)
-{
-    const char *cp = *cpp;
-    char cmd[32];
-    int i, level;
-
-    while (*cp == ' ' || *cp == '\t' || *cp == '\n' || *cp == '\r')
-        cp++;
-    switch (*cp)
-    {
-    case '\0':
-        return 0;
-    case '/':
-        *cpp = cp+1;
-        return REGX_PATTERN;
-    case '{':
-        *cpp = cp+1;
-        level = 1;
-        while (*++cp)
-        {
-            if (*cp == '{')
-                level++;
-            else if (*cp == '}')
-            {
-                level--;
-                if (level == 0)
-                    break;
-            }
-        }
-        *len = cp - *cpp;
-        return REGX_CODE;
-    default:
-        i = 0;
-        while (1)
-        {
-            if (*cp >= 'a' && *cp <= 'z')
-                cmd[i] = *cp;
-            else if (*cp >= 'A' && *cp <= 'Z')
-                cmd[i] = *cp + 'a' - 'A';
-            else
-                break;
-            if (i < (int) sizeof(cmd)-2)
-               i++;
-            cp++;
-        }
-        cmd[i] = '\0';
-        if (i == 0)
-        {
-            yaz_log (YLOG_WARN, "bad character %d %c", *cp, *cp);
-            cp++;
-            while (*cp && *cp != ' ' && *cp != '\t' &&
-                   *cp != '\n' && *cp != '\r')
-                cp++;
-            *cpp = cp;
-            return 0;
-        }
-        *cpp = cp;
-        if (!strcmp (cmd, "begin"))
-            return REGX_BEGIN;
-        else if (!strcmp (cmd, "end"))
-            return REGX_END;
-        else if (!strcmp (cmd, "body"))
-            return REGX_BODY;
-       else if (!strcmp (cmd, "context"))
-           return REGX_CONTEXT;
-       else if (!strcmp (cmd, "init"))
-           return REGX_INIT;
-        else
-        {
-            yaz_log (YLOG_WARN, "bad command %s", cmd);
-            return 0;
-        }
-    }
-}
-
-static int actionListMk (struct lexSpec *spec, const char *s,
-                         struct lexRuleAction **ap)
-{
-    int r, tok, len;
-    int bodyMark = 0;
-    const char *s0;
-
-    while ((tok = readParseToken (&s, &len)))
-    {
-        switch (tok)
-        {
-        case REGX_BODY:
-            bodyMark = 1;
-            continue;
-        case REGX_CODE:
-            *ap = (struct lexRuleAction *) xmalloc (sizeof(**ap));
-            (*ap)->which = tok;
-            regxCodeMk (&(*ap)->u.code, s, len);
-            s += len+1;
-            break;
-        case REGX_PATTERN:
-            *ap = (struct lexRuleAction *) xmalloc (sizeof(**ap));
-            (*ap)->which = tok;
-            (*ap)->u.pattern.body = bodyMark;
-            bodyMark = 0;
-            (*ap)->u.pattern.dfa = lexSpecDFA ();
-           s0 = s;
-            r = dfa_parse ((*ap)->u.pattern.dfa, &s);
-            if (r || *s != '/')
-            {
-                int pos = s - s0;
-                xfree (*ap);
-                *ap = NULL;
-                yaz_log(YLOG_WARN, "regular expression error '%.*s'", pos, s0);
-                return -1;
-            }
-            else
-            {
-                int pos = s - s0;
-                if (debug_dfa_tran)
-                    printf("pattern: %.*s\n", pos, s0);
-                dfa_mkstate((*ap)->u.pattern.dfa);
-                s++;
-            }
-            break;
-        case REGX_BEGIN:
-            yaz_log (YLOG_WARN, "cannot use BEGIN here");
-            continue;
-        case REGX_INIT:
-            yaz_log (YLOG_WARN, "cannot use INIT here");
-            continue;
-        case REGX_END:
-            *ap = (struct lexRuleAction *) xmalloc (sizeof(**ap));
-            (*ap)->which = tok;
-            break;
-        }
-        ap = &(*ap)->next;
-    }
-    *ap = NULL;
-    return 0;
-}
-
-int readOneSpec (struct lexSpec *spec, const char *s)
-{
-    int len, r, tok;
-    struct lexRule *rp;
-    struct lexContext *lc;
-
-    tok = readParseToken (&s, &len);
-    if (tok == REGX_CONTEXT)
-    {
-       char context_name[32];
-       tok = readParseToken (&s, &len);
-       if (tok != REGX_CODE)
-       {
-           yaz_log (YLOG_WARN, "missing name after CONTEXT keyword");
-           return 0;
-       }
-       if (len > 31)
-           len = 31;
-       memcpy (context_name, s, len);
-       context_name[len] = '\0';
-       lc = lexContextCreate (context_name);
-       lc->next = spec->context;
-       spec->context = lc;
-       return 0;
-    }
-    if (!spec->context)
-       spec->context = lexContextCreate ("main");
-       
-    switch (tok)
-    {
-    case REGX_BEGIN:
-        actionListDel (&spec->context->beginActionList);
-        actionListMk (spec, s, &spec->context->beginActionList);
-       break;
-    case REGX_END:
-        actionListDel (&spec->context->endActionList);
-        actionListMk (spec, s, &spec->context->endActionList);
-       break;
-    case REGX_INIT:
-        actionListDel (&spec->context->initActionList);
-        actionListMk (spec, s, &spec->context->initActionList);
-       break;
-    case REGX_PATTERN:
-#if REGX_DEBUG
-       yaz_log (YLOG_LOG, "rule %d %s", spec->context->ruleNo, s);
-#endif
-        r = dfa_parse (spec->context->dfa, &s);
-        if (r)
-        {
-            yaz_log (YLOG_WARN, "regular expression error. r=%d", r);
-            return -1;
-        }
-        if (*s != '/')
-        {
-            yaz_log (YLOG_WARN, "expects / at end of pattern. got %c", *s);
-            return -1;
-        }
-        s++;
-        rp = (struct lexRule *) xmalloc (sizeof(*rp));
-        rp->info.no = spec->context->ruleNo++;
-        rp->next = spec->context->rules;
-        spec->context->rules = rp;
-        actionListMk (spec, s, &rp->info.actionList);
-    }
-    return 0;
-}
-
-int readFileSpec (struct lexSpec *spec)
-{
-    struct lexContext *lc;
-    int c, i, errors = 0;
-    FILE *spec_inf = 0;
-    WRBUF lineBuf;
-    char fname[256];
-
-#if HAVE_TCL_H
-    if (spec->tcl_interp)
-    {
-       sprintf (fname, "%s.tflt", spec->name);
-       spec_inf = data1_path_fopen (spec->dh, fname, "r");
-    }
-#endif
-    if (!spec_inf)
-    {
-       sprintf (fname, "%s.flt", spec->name);
-       spec_inf = data1_path_fopen (spec->dh, fname, "r");
-    }
-    if (!spec_inf)
-    {
-        yaz_log (YLOG_ERRNO|YLOG_WARN, "cannot read spec file %s", spec->name);
-        return -1;
-    }
-    yaz_log (YLOG_LOG, "reading regx filter %s", fname);
-#if HAVE_TCL_H
-    if (spec->tcl_interp)
-       yaz_log (YLOG_LOG, "Tcl enabled");
-#endif
-
-#if 0
-    debug_dfa_trav = 0;
-    debug_dfa_tran = 1;
-    debug_dfa_followpos = 0;
-    dfa_verbose = 1;
-#endif
-
-    lineBuf = wrbuf_alloc();
-    spec->lineNo = 0;
-    c = getc (spec_inf);
-    while (c != EOF)
-    {
-       wrbuf_rewind (lineBuf);
-        if (c == '#' || c == '\n' || c == ' ' || c == '\t' || c == '\r')
-        {
-            while (c != '\n' && c != EOF)
-                c = getc (spec_inf);
-            spec->lineNo++;
-            if (c == '\n')
-                c = getc (spec_inf);
-        }
-        else
-        {
-            int addLine = 0;
-           
-            while (1)
-            {
-                int c1 = c;
-               wrbuf_putc(lineBuf, c);
-                c = getc (spec_inf);
-               while (c == '\r')
-                   c = getc (spec_inf);
-                if (c == EOF)
-                    break;
-                if (c1 == '\n')
-                {
-                    if (c != ' ' && c != '\t')
-                        break;
-                    addLine++;
-                }
-            }
-           wrbuf_putc(lineBuf, '\0');
-            readOneSpec (spec, wrbuf_buf(lineBuf));
-            spec->lineNo += addLine;
-        }
-    }
-    fclose (spec_inf);
-    wrbuf_destroy(lineBuf);
-
-    for (lc = spec->context; lc; lc = lc->next)
-    {
-       struct lexRule *rp;
-       lc->fastRule = (struct lexRuleInfo **)
-           xmalloc (sizeof(*lc->fastRule) * lc->ruleNo);
-       for (i = 0; i < lc->ruleNo; i++)
-           lc->fastRule[i] = NULL;
-       for (rp = lc->rules; rp; rp = rp->next)
-           lc->fastRule[rp->info.no] = &rp->info;
-       dfa_mkstate (lc->dfa);
-    }
-    if (errors)
-        return -1;
-    
-    return 0;
-}
-
-#if 0
-static struct lexSpec *curLexSpec = NULL;
-#endif
-
-static void execData (struct lexSpec *spec,
-                      const char *ebuf, int elen, int formatted_text,
-                     const char *attribute_str, int attribute_len)
-{
-    struct data1_node *res, *parent;
-    int org_len;
-
-    if (elen == 0) /* shouldn't happen, but it does! */
-       return ;
-#if REGX_DEBUG
-    if (elen > 80)
-        yaz_log (YLOG_LOG, "data(%d bytes) %.40s ... %.*s", elen,
-             ebuf, 40, ebuf + elen-40);
-    else if (elen == 1 && ebuf[0] == '\n')
-    {
-        yaz_log (YLOG_LOG, "data(new line)");
-    }
-    else if (elen > 0)
-        yaz_log (YLOG_LOG, "data(%d bytes) %.*s", elen, elen, ebuf);
-    else 
-        yaz_log (YLOG_LOG, "data(%d bytes)", elen);
-#endif
-        
-    if (spec->d1_level <= 1)
-        return;
-
-    parent = spec->d1_stack[spec->d1_level -1];
-    assert (parent);
-
-    if (attribute_str)
-    {
-       data1_xattr **ap;
-       res = parent;
-       if (res->which != DATA1N_tag)
-           return;
-       /* sweep through exising attributes.. */
-       for (ap = &res->u.tag.attributes; *ap; ap = &(*ap)->next)
-           if (strlen((*ap)->name) == attribute_len &&
-               !memcmp((*ap)->name, attribute_str, attribute_len))
-               break;
-       if (!*ap)
-       {
-           /* new attribute. Create it with name + value */
-           *ap = nmem_malloc(spec->m, sizeof(**ap));
-
-           (*ap)->name = nmem_malloc(spec->m, attribute_len+1);
-           memcpy((*ap)->name, attribute_str, attribute_len);
-           (*ap)->name[attribute_len] = '\0';
-
-           (*ap)->value = nmem_malloc(spec->m, elen+1);
-           memcpy((*ap)->value, ebuf, elen);
-           (*ap)->value[elen] = '\0';
-           (*ap)->next = 0;
-       }
-       else
-       {
-           /* append to value if attribute already exists */
-           char *nv = nmem_malloc(spec->m, elen + 1 + strlen((*ap)->value));
-           strcpy(nv, (*ap)->value);
-           memcpy (nv + strlen(nv), ebuf, elen);
-           nv[strlen(nv)+elen] = '\0';
-           (*ap)->value = nv;
-       }
-    } 
-    else 
-    {
-       if ((res = spec->d1_stack[spec->d1_level]) && 
-           res->which == DATA1N_data)
-           org_len = res->u.data.len;
-       else
-       {
-           org_len = 0;
-           
-           res = data1_mk_node2 (spec->dh, spec->m, DATA1N_data, parent);
-           res->u.data.what = DATA1I_text;
-           res->u.data.len = 0;
-           res->u.data.formatted_text = formatted_text;
-           res->u.data.data = 0;
-           
-           if (spec->d1_stack[spec->d1_level])
-               spec->d1_stack[spec->d1_level]->next = res;
-           spec->d1_stack[spec->d1_level] = res;
-       }
-       if (org_len + elen >= spec->concatBuf[spec->d1_level].max)
-       {
-           char *old_buf, *new_buf;
-           
-           spec->concatBuf[spec->d1_level].max = org_len + elen + 256;
-           new_buf = (char *) xmalloc (spec->concatBuf[spec->d1_level].max);
-           if ((old_buf = spec->concatBuf[spec->d1_level].buf))
-           {
-               memcpy (new_buf, old_buf, org_len);
-               xfree (old_buf);
-           }
-           spec->concatBuf[spec->d1_level].buf = new_buf;
-       }
-       memcpy (spec->concatBuf[spec->d1_level].buf + org_len, ebuf, elen);
-       res->u.data.len += elen;
-    }
-}
-
-static void execDataP (struct lexSpec *spec,
-                       const char *ebuf, int elen, int formatted_text)
-{
-    execData (spec, ebuf, elen, formatted_text, 0, 0);
-}
-
-static void tagDataRelease (struct lexSpec *spec)
-{
-    data1_node *res;
-    
-    if ((res = spec->d1_stack[spec->d1_level]) &&
-       res->which == DATA1N_data && 
-       res->u.data.what == DATA1I_text)
-    {
-       assert (!res->u.data.data);
-       assert (res->u.data.len > 0);
-       if (res->u.data.len > DATA1_LOCALDATA)
-           res->u.data.data = (char *) nmem_malloc (spec->m, res->u.data.len);
-       else
-           res->u.data.data = res->lbuf;
-       memcpy (res->u.data.data, spec->concatBuf[spec->d1_level].buf,
-               res->u.data.len);
-    }
-}
-
-static void variantBegin (struct lexSpec *spec, 
-                         const char *class_str, int class_len,
-                         const char *type_str, int type_len,
-                         const char *value_str, int value_len)
-{
-    struct data1_node *parent = spec->d1_stack[spec->d1_level -1];
-    char tclass[DATA1_MAX_SYMBOL], ttype[DATA1_MAX_SYMBOL];
-    data1_vartype *tp;
-    int i;
-    data1_node *res;
-
-    if (spec->d1_level == 0)
-    {
-        yaz_log (YLOG_WARN, "in variant begin. No record type defined");
-        return ;
-    }
-    if (class_len >= DATA1_MAX_SYMBOL)
-       class_len = DATA1_MAX_SYMBOL-1;
-    memcpy (tclass, class_str, class_len);
-    tclass[class_len] = '\0';
-
-    if (type_len >= DATA1_MAX_SYMBOL)
-       type_len = DATA1_MAX_SYMBOL-1;
-    memcpy (ttype, type_str, type_len);
-    ttype[type_len] = '\0';
-
-#if REGX_DEBUG 
-    yaz_log (YLOG_LOG, "variant begin(%s,%s,%d)", tclass, ttype,
-         spec->d1_level);
-#endif
-
-    if (!(tp =
-         data1_getvartypeby_absyn(spec->dh, parent->root->u.root.absyn,
-                                  tclass, ttype)))
-       return;
-    
-    if (parent->which != DATA1N_variant)
-    {
-       res = data1_mk_node2 (spec->dh, spec->m, DATA1N_variant, parent);
-       if (spec->d1_stack[spec->d1_level])
-           tagDataRelease (spec);
-       spec->d1_stack[spec->d1_level] = res;
-       spec->d1_stack[++(spec->d1_level)] = NULL;
-    }
-    for (i = spec->d1_level-1; spec->d1_stack[i]->which == DATA1N_variant; i--)
-       if (spec->d1_stack[i]->u.variant.type == tp)
-       {
-           spec->d1_level = i;
-           break;
-       }
-
-#if REGX_DEBUG 
-    yaz_log (YLOG_LOG, "variant node(%d)", spec->d1_level);
-#endif
-    parent = spec->d1_stack[spec->d1_level-1];
-    res = data1_mk_node2 (spec->dh, spec->m, DATA1N_variant, parent);
-    res->u.variant.type = tp;
-
-    if (value_len >= DATA1_LOCALDATA)
-       value_len =DATA1_LOCALDATA-1;
-    memcpy (res->lbuf, value_str, value_len);
-    res->lbuf[value_len] = '\0';
-
-    res->u.variant.value = res->lbuf;
-    
-    if (spec->d1_stack[spec->d1_level])
-       tagDataRelease (spec);
-    spec->d1_stack[spec->d1_level] = res;
-    spec->d1_stack[++(spec->d1_level)] = NULL;
-}
-
-static void tagStrip (const char **tag, int *len)
-{
-    int i;
-
-    for (i = *len; i > 0 && isspace((*tag)[i-1]); --i)
-        ;
-    *len = i;
-    for (i = 0; i < *len && isspace((*tag)[i]); i++)
-        ;
-    *tag += i;
-    *len -= i;
-}
-
-static void tagBegin (struct lexSpec *spec, 
-                      const char *tag, int len)
-{
-    if (spec->d1_level == 0)
-    {
-        yaz_log (YLOG_WARN, "in element begin. No record type defined");
-        return ;
-    }
-    tagStrip (&tag, &len);
-    if (spec->d1_stack[spec->d1_level])
-       tagDataRelease (spec);
-
-#if REGX_DEBUG 
-    yaz_log (YLOG_LOG, "begin tag(%.*s, %d)", len, tag, spec->d1_level);
-#endif
-
-    spec->d1_stack[spec->d1_level] = data1_mk_tag_n (
-        spec->dh, spec->m, tag, len, 0, spec->d1_stack[spec->d1_level -1]);
-    spec->d1_stack[++(spec->d1_level)] = NULL;
-}
-
-static void tagEnd (struct lexSpec *spec, int min_level,
-                    const char *tag, int len)
-{
-    tagStrip (&tag, &len);
-    while (spec->d1_level > min_level)
-    {
-       tagDataRelease (spec);
-        (spec->d1_level)--;
-        if (spec->d1_level == 0)
-           break;
-        if ((spec->d1_stack[spec->d1_level]->which == DATA1N_tag) &&
-           (!tag ||
-            (strlen(spec->d1_stack[spec->d1_level]->u.tag.tag) ==
-             (size_t) len &&
-             !memcmp (spec->d1_stack[spec->d1_level]->u.tag.tag, tag, len))))
-            break;
-    }
-#if REGX_DEBUG
-    yaz_log (YLOG_LOG, "end tag(%d)", spec->d1_level);
-#endif
-}
-
-
-static int tryMatch (struct lexSpec *spec, int *pptr, int *mptr,
-                     struct DFA *dfa, int greedy)
-{
-    struct DFA_state *state = dfa->states[0];
-    struct DFA_tran *t;
-    unsigned char c = 0;
-    unsigned char c_prev = 0;
-    int ptr = *pptr;          /* current pointer */
-    int start_ptr = *pptr;    /* first char of match */
-    int last_ptr = 0;         /* last char of match */
-    int last_rule = 0;        /* rule number of current match */
-    int restore_ptr = 0;
-    int i;
-
-    if (ptr)
-    {
-       --ptr;
-        c = f_win_advance (spec, &ptr);
-    }
-    while (1)
-    {
-       if (dfa->states[0] == state)
-       {
-           c_prev = c;
-           restore_ptr = ptr;
-       }
-        c = f_win_advance (spec, &ptr);
-
-        if (ptr == F_WIN_EOF)
-        {
-            if (last_rule)
-            {
-                *mptr = start_ptr;
-                *pptr = last_ptr;
-                return 1;
-            }
-            break;
-        }
-
-        t = state->trans;
-        i = state->tran_no;
-        while (1)
-            if (--i < 0)    /* no transition for character c */
-            {
-                if (last_rule)
-                {
-                    *mptr = start_ptr;     /* match starts here */
-                    *pptr = last_ptr;      /* match end here (+1) */
-                    return 1;
-                }
-                state = dfa->states[0];
-
-               ptr = restore_ptr;
-               c = f_win_advance (spec, &ptr);
-
-                start_ptr = ptr;
-
-                break;
-            }
-            else if (c >= t->ch[0] && c <= t->ch[1])
-            {
-                state = dfa->states[t->to];
-                if (state->rule_no && c_prev == '\n')
-               {
-                   last_rule = state->rule_no;
-                   last_ptr = ptr;
-               }
-               else if (state->rule_nno)
-               {
-                   last_rule = state->rule_nno;
-                   last_ptr = ptr;
-               }
-               break;
-            }
-            else
-                t++;
-    }
-    return 0;
-}
-
-static int execTok (struct lexSpec *spec, const char **src,
-                    const char **tokBuf, int *tokLen)
-{
-    const char *s = *src;
-
-    while (*s == ' ' || *s == '\t')
-        s++;
-    if (!*s)
-        return 0;
-    if (*s == '$' && s[1] >= '0' && s[1] <= '9')
-    {
-        int n = 0;
-        s++;
-        while (*s >= '0' && *s <= '9')
-            n = n*10 + (*s++ -'0');
-        if (spec->arg_no == 0)
-        {
-            *tokBuf = "";
-            *tokLen = 0;
-        }
-        else
-        {
-            if (n >= spec->arg_no)
-                n = spec->arg_no-1;
-            *tokBuf = f_win_get (spec, spec->arg_start[n], spec->arg_end[n],
-                                tokLen);
-        }
-    }
-    else if (*s == '\"')
-    {
-        *tokBuf = ++s;
-        while (*s && *s != '\"')
-            s++;
-        *tokLen = s - *tokBuf;
-        if (*s)
-            s++;
-        *src = s;
-    }
-    else if (*s == '\n' || *s == ';')
-    {
-        *src = s+1;
-        return 1;
-    }
-    else if (*s == '-')
-    {
-        *tokBuf = s++;
-        while (*s && *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r' &&
-               *s != ';')
-            s++;
-        *tokLen = s - *tokBuf;
-        *src = s;
-        return 3;
-    }
-    else
-    {
-        *tokBuf = s++;
-        while (*s && *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r' &&
-               *s != ';')
-            s++;
-        *tokLen = s - *tokBuf;
-    }
-    *src = s;
-    return 2;
-}
-
-static char *regxStrz (const char *src, int len, char *str)
-{
-    if (len > 63)
-        len = 63;
-    memcpy (str, src, len);
-    str[len] = '\0';
-    return str;
-}
-
-#if HAVE_TCL_H
-static int cmd_tcl_begin (ClientData clientData, Tcl_Interp *interp,
-                         int argc, const char **argv)
-{
-    struct lexSpec *spec = (struct lexSpec *) clientData;
-    if (argc < 2)
-       return TCL_ERROR;
-    if (!strcmp(argv[1], "record") && argc == 3)
-    {
-       const char *absynName = argv[2];
-        data1_node *res;
-
-#if REGX_DEBUG
-       yaz_log (YLOG_LOG, "begin record %s", absynName);
-#endif
-        res = data1_mk_root (spec->dh, spec->m, absynName);
-        
-       spec->d1_level = 0;
-
-        spec->d1_stack[spec->d1_level++] = res;
-
-        res = data1_mk_tag (spec->dh, spec->m, absynName, 0, res);
-
-        spec->d1_stack[spec->d1_level++] = res;
-
-        spec->d1_stack[spec->d1_level] = NULL;
-    }
-    else if (!strcmp(argv[1], "element") && argc == 3)
-    {
-       tagBegin (spec, argv[2], strlen(argv[2]));
-    }
-    else if (!strcmp (argv[1], "variant") && argc == 5)
-    {
-       variantBegin (spec, argv[2], strlen(argv[2]),
-                     argv[3], strlen(argv[3]),
-                     argv[4], strlen(argv[4]));
-    }
-    else if (!strcmp (argv[1], "context") && argc == 3)
-    {
-       struct lexContext *lc = spec->context;
-#if REGX_DEBUG
-       yaz_log (YLOG_LOG, "begin context %s",argv[2]);
-#endif
-       while (lc && strcmp (argv[2], lc->name))
-           lc = lc->next;
-       if (lc)
-       {
-           spec->context_stack[++(spec->context_stack_top)] = lc;
-       }
-       else
-           yaz_log (YLOG_WARN, "unknown context %s", argv[2]);
-    }
-    else
-       return TCL_ERROR;
-    return TCL_OK;
-}
-
-static int cmd_tcl_end (ClientData clientData, Tcl_Interp *interp,
-                       int argc, const char **argv)
-{
-    struct lexSpec *spec = (struct lexSpec *) clientData;
-    if (argc < 2)
-       return TCL_ERROR;
-    
-    if (!strcmp (argv[1], "record"))
-    {
-       while (spec->d1_level)
-       {
-           tagDataRelease (spec);
-           (spec->d1_level)--;
-       }
-#if REGX_DEBUG
-       yaz_log (YLOG_LOG, "end record");
-#endif
-       spec->stop_flag = 1;
-    }
-    else if (!strcmp (argv[1], "element"))
-    {
-       int min_level = 2;
-       const char *element = 0;
-       if (argc >= 3 && !strcmp(argv[2], "-record"))
-       {
-           min_level = 0;
-           if (argc == 4)
-               element = argv[3];
-       }
-       else
-           if (argc == 3)
-               element = argv[2];
-       tagEnd (spec, min_level, element, (element ? strlen(element) : 0));
-       if (spec->d1_level <= 1)
-       {
-#if REGX_DEBUG
-           yaz_log (YLOG_LOG, "end element end records");
-#endif
-           spec->stop_flag = 1;
-       }
-    }
-    else if (!strcmp (argv[1], "context"))
-    {
-#if REGX_DEBUG
-       yaz_log (YLOG_LOG, "end context");
-#endif
-       if (spec->context_stack_top)
-           (spec->context_stack_top)--;
-    }
-    else
-       return TCL_ERROR;
-    return TCL_OK;
-}
-
-static int cmd_tcl_data (ClientData clientData, Tcl_Interp *interp,
-                        int argc, const char **argv)
-{
-    int argi = 1;
-    int textFlag = 0;
-    const char *element = 0;
-    const char *attribute = 0;
-    struct lexSpec *spec = (struct lexSpec *) clientData;
-    
-    while (argi < argc)
-    {
-       if (!strcmp("-text", argv[argi]))
-       {
-           textFlag = 1;
-           argi++;
-       }
-       else if (!strcmp("-element", argv[argi]))
-       {
-           argi++;
-           if (argi < argc)
-               element = argv[argi++];
-       }
-       else if (!strcmp("-attribute", argv[argi]))
-       {
-           argi++;
-           if (argi < argc)
-               attribute = argv[argi++];
-       }
-       else
-           break;
-    }
-    if (element)
-       tagBegin (spec, element, strlen(element));
-
-    while (argi < argc)
-    {
-#if TCL_MAJOR_VERSION > 8 || (TCL_MAJOR_VERSION == 8 && TCL_MINOR_VERSION > 0)
-       Tcl_DString ds;
-       char *native = Tcl_UtfToExternalDString(0, argv[argi], -1, &ds);
-       execData (spec, native, strlen(native), textFlag, attribute, 
-                 attribute ? strlen(attribute) : 0);
-       Tcl_DStringFree (&ds);
-#else
-       execData (spec, argv[argi], strlen(argv[argi]), textFlag, attribute,
-                 attribute ? strlen(attribute) : 0);
-#endif
-       argi++;
-    }
-    if (element)
-       tagEnd (spec, 2, NULL, 0);
-    return TCL_OK;
-}
-
-static int cmd_tcl_unread (ClientData clientData, Tcl_Interp *interp,
-                          int argc, const char **argv)
-{
-    struct lexSpec *spec = (struct lexSpec *) clientData;
-    int argi = 1;
-    int offset = 0;
-    int no;
-    
-    while (argi < argc)
-    {
-       if (!strcmp("-offset", argv[argi]))
-       {
-           argi++;
-           if (argi < argc)
-           {
-               offset = atoi(argv[argi]);
-               argi++;
-           }
-       }
-       else
-           break;
-    }
-    if (argi != argc-1)
-       return TCL_ERROR;
-    no = atoi(argv[argi]);
-    if (no >= spec->arg_no)
-       no = spec->arg_no - 1;
-    spec->ptr = spec->arg_start[no] + offset;
-    return TCL_OK;
-}
-
-static void execTcl (struct lexSpec *spec, struct regxCode *code)
-{   
-    int i;
-    int ret;
-    for (i = 0; i < spec->arg_no; i++)
-    {
-       char var_name[10], *var_buf;
-       int var_len, ch;
-       
-       sprintf (var_name, "%d", i);
-       var_buf = f_win_get (spec, spec->arg_start[i], spec->arg_end[i],
-                            &var_len); 
-       if (var_buf)
-       {
-           ch = var_buf[var_len];
-           var_buf[var_len] = '\0';
-           Tcl_SetVar (spec->tcl_interp, var_name, var_buf, 0);
-           var_buf[var_len] = ch;
-       }
-    }
-#if HAVE_TCL_OBJECTS
-    ret = Tcl_GlobalEvalObj(spec->tcl_interp, code->tcl_obj);
-#else
-    ret = Tcl_GlobalEval (spec->tcl_interp, code->str);
-#endif
-    if (ret != TCL_OK)
-    {
-       const char *err = Tcl_GetVar(spec->tcl_interp, "errorInfo", 0);
-       yaz_log(YLOG_FATAL, "Tcl error, line=%d, \"%s\"\n%s", 
-           spec->tcl_interp->errorLine,
-           spec->tcl_interp->result,
-           err ? err : "[NO ERRORINFO]");
-    }
-}
-/* HAVE_TCL_H */
-#endif
-
-static void execCode (struct lexSpec *spec, struct regxCode *code)
-{
-    const char *s = code->str;
-    int cmd_len, r;
-    const char *cmd_str;
-    
-    r = execTok (spec, &s, &cmd_str, &cmd_len);
-    while (r)
-    {
-        char *p, ptmp[64];
-        
-        if (r == 1)
-        {
-            r = execTok (spec, &s, &cmd_str, &cmd_len);
-            continue;
-        }
-        p = regxStrz (cmd_str, cmd_len, ptmp);
-        if (!strcmp (p, "begin"))
-        {
-            r = execTok (spec, &s, &cmd_str, &cmd_len);
-            if (r < 2)
-           {
-               yaz_log (YLOG_WARN, "missing keyword after 'begin'");
-                continue;
-           }
-            p = regxStrz (cmd_str, cmd_len, ptmp);
-            if (!strcmp (p, "record"))
-            {
-                r = execTok (spec, &s, &cmd_str, &cmd_len);
-                if (r < 2)
-                    continue;
-                if (spec->d1_level <= 1)
-                {
-                    static char absynName[64];
-                    data1_node *res;
-
-                    if (cmd_len > 63)
-                        cmd_len = 63;
-                    memcpy (absynName, cmd_str, cmd_len);
-                    absynName[cmd_len] = '\0';
-#if REGX_DEBUG
-                    yaz_log (YLOG_LOG, "begin record %s", absynName);
-#endif
-                    res = data1_mk_root (spec->dh, spec->m, absynName);
-                    
-                   spec->d1_level = 0;
-
-                    spec->d1_stack[spec->d1_level++] = res;
-
-                    res = data1_mk_tag (spec->dh, spec->m, absynName, 0, res);
-
-                    spec->d1_stack[spec->d1_level++] = res;
-
-                    spec->d1_stack[spec->d1_level] = NULL;
-                }
-                r = execTok (spec, &s, &cmd_str, &cmd_len);
-            }
-            else if (!strcmp (p, "element"))
-            {
-                r = execTok (spec, &s, &cmd_str, &cmd_len);
-                if (r < 2)
-                    continue;
-                tagBegin (spec, cmd_str, cmd_len);
-                r = execTok (spec, &s, &cmd_str, &cmd_len);
-            } 
-           else if (!strcmp (p, "variant"))
-           {
-               int class_len;
-               const char *class_str = NULL;
-               int type_len;
-               const char *type_str = NULL;
-               int value_len;
-               const char *value_str = NULL;
-               r = execTok (spec, &s, &cmd_str, &cmd_len);
-               if (r < 2)
-                   continue;
-               class_str = cmd_str;
-               class_len = cmd_len;
-               r = execTok (spec, &s, &cmd_str, &cmd_len);
-               if (r < 2)
-                   continue;
-               type_str = cmd_str;
-               type_len = cmd_len;
-
-               r = execTok (spec, &s, &cmd_str, &cmd_len);
-               if (r < 2)
-                   continue;
-               value_str = cmd_str;
-               value_len = cmd_len;
-
-                variantBegin (spec, class_str, class_len,
-                             type_str, type_len, value_str, value_len);
-               
-               
-               r = execTok (spec, &s, &cmd_str, &cmd_len);
-           }
-           else if (!strcmp (p, "context"))
-           {
-               if (r > 1)
-               {
-                   struct lexContext *lc = spec->context;
-                   r = execTok (spec, &s, &cmd_str, &cmd_len);
-                   p = regxStrz (cmd_str, cmd_len, ptmp);
-#if REGX_DEBUG
-                   yaz_log (YLOG_LOG, "begin context %s", p);
-#endif
-                   while (lc && strcmp (p, lc->name))
-                       lc = lc->next;
-                   if (lc)
-                       spec->context_stack[++(spec->context_stack_top)] = lc;
-                   else
-                       yaz_log (YLOG_WARN, "unknown context %s", p);
-                   
-               }
-               r = execTok (spec, &s, &cmd_str, &cmd_len);
-           }
-           else
-           {
-               yaz_log (YLOG_WARN, "bad keyword '%s' after begin", p);
-           }
-        }
-        else if (!strcmp (p, "end"))
-        {
-            r = execTok (spec, &s, &cmd_str, &cmd_len);
-            if (r < 2)
-           {
-               yaz_log (YLOG_WARN, "missing keyword after 'end'");
-               continue;
-           }
-           p = regxStrz (cmd_str, cmd_len, ptmp);
-           if (!strcmp (p, "record"))
-           {
-               while (spec->d1_level)
-               {
-                   tagDataRelease (spec);
-                   (spec->d1_level)--;
-               }
-               r = execTok (spec, &s, &cmd_str, &cmd_len);
-#if REGX_DEBUG
-               yaz_log (YLOG_LOG, "end record");
-#endif
-               spec->stop_flag = 1;
-           }
-           else if (!strcmp (p, "element"))
-           {
-                int min_level = 2;
-                while ((r = execTok (spec, &s, &cmd_str, &cmd_len)) == 3)
-                {
-                    if (cmd_len==7 && !memcmp ("-record", cmd_str, cmd_len))
-                        min_level = 0;
-                }
-               if (r > 2)
-               {
-                   tagEnd (spec, min_level, cmd_str, cmd_len);
-                   r = execTok (spec, &s, &cmd_str, &cmd_len);
-               }
-               else
-                   tagEnd (spec, min_level, NULL, 0);
-                if (spec->d1_level <= 1)
-                {
-#if REGX_DEBUG
-                   yaz_log (YLOG_LOG, "end element end records");
-#endif
-                   spec->stop_flag = 1;
-                }
-
-           }
-           else if (!strcmp (p, "context"))
-           {
-#if REGX_DEBUG
-               yaz_log (YLOG_LOG, "end context");
-#endif
-               if (spec->context_stack_top)
-                   (spec->context_stack_top)--;
-               r = execTok (spec, &s, &cmd_str, &cmd_len);
-           }       
-           else
-               yaz_log (YLOG_WARN, "bad keyword '%s' after end", p);
-       }
-        else if (!strcmp (p, "data"))
-        {
-            int textFlag = 0;
-            int element_len;
-            const char *element_str = NULL;
-           int attribute_len;
-           const char *attribute_str = NULL;
-            
-            while ((r = execTok (spec, &s, &cmd_str, &cmd_len)) == 3)
-            {
-                if (cmd_len==5 && !memcmp ("-text", cmd_str, cmd_len))
-                    textFlag = 1;
-                else if (cmd_len==8 && !memcmp ("-element", cmd_str, cmd_len))
-                {
-                    r = execTok (spec, &s, &element_str, &element_len);
-                    if (r < 2)
-                        break;
-                }
-                else if (cmd_len==10 && !memcmp ("-attribute", cmd_str, 
-                                                cmd_len))
-                {
-                    r = execTok (spec, &s, &attribute_str, &attribute_len);
-                    if (r < 2)
-                        break;
-                }
-                else 
-                    yaz_log (YLOG_WARN, "bad data option: %.*s",
-                          cmd_len, cmd_str);
-            }
-            if (r != 2)
-            {
-                yaz_log (YLOG_WARN, "missing data item after data");
-                continue;
-            }
-            if (element_str)
-                tagBegin (spec, element_str, element_len);
-            do
-            {
-                execData (spec, cmd_str, cmd_len, textFlag,
-                         attribute_str, attribute_len);
-                r = execTok (spec, &s, &cmd_str, &cmd_len);
-            } while (r > 1);
-            if (element_str)
-                tagEnd (spec, 2, NULL, 0);
-        }
-        else if (!strcmp (p, "unread"))
-        {
-            int no, offset;
-            r = execTok (spec, &s, &cmd_str, &cmd_len);
-            if (r==3 && cmd_len == 7 && !memcmp ("-offset", cmd_str, cmd_len))
-            {
-                r = execTok (spec, &s, &cmd_str, &cmd_len);
-                if (r < 2)
-                {
-                    yaz_log (YLOG_WARN, "missing number after -offset");
-                    continue;
-                }
-                p = regxStrz (cmd_str, cmd_len, ptmp);
-                offset = atoi (p);
-                r = execTok (spec, &s, &cmd_str, &cmd_len);
-            }
-            else
-                offset = 0;
-            if (r < 2)
-            {
-                yaz_log (YLOG_WARN, "missing index after unread command");
-                continue;
-            }
-            if (cmd_len != 1 || *cmd_str < '0' || *cmd_str > '9')
-            {
-                yaz_log (YLOG_WARN, "bad index after unread command");
-                continue;
-            }
-            else
-            {
-                no = *cmd_str - '0';
-                if (no >= spec->arg_no)
-                    no = spec->arg_no - 1;
-                spec->ptr = spec->arg_start[no] + offset;
-            }
-            r = execTok (spec, &s, &cmd_str, &cmd_len);
-        }
-       else if (!strcmp (p, "context"))
-       {
-            if (r > 1)
-           {
-               struct lexContext *lc = spec->context;
-               r = execTok (spec, &s, &cmd_str, &cmd_len);
-               p = regxStrz (cmd_str, cmd_len, ptmp);
-               
-               while (lc && strcmp (p, lc->name))
-                   lc = lc->next;
-               if (lc)
-                   spec->context_stack[spec->context_stack_top] = lc;
-               else
-                   yaz_log (YLOG_WARN, "unknown context %s", p);
-
-           }
-           r = execTok (spec, &s, &cmd_str, &cmd_len);
-       }
-        else
-        {
-            yaz_log (YLOG_WARN, "unknown code command '%.*s'", cmd_len, cmd_str);
-            r = execTok (spec, &s, &cmd_str, &cmd_len);
-            continue;
-        }
-        if (r > 1)
-        {
-            yaz_log (YLOG_WARN, "ignoring token %.*s", cmd_len, cmd_str);
-            do {
-                r = execTok (spec, &s, &cmd_str, &cmd_len);
-            } while (r > 1);
-        }
-    }
-}
-
-
-static int execAction (struct lexSpec *spec, struct lexRuleAction *ap,
-                       int start_ptr, int *pptr)
-{
-    int sptr;
-    int arg_start[20];
-    int arg_end[20];
-    int arg_no = 1;
-
-    if (!ap)
-       return 1;
-    arg_start[0] = start_ptr;
-    arg_end[0] = *pptr;
-    spec->arg_start = arg_start;
-    spec->arg_end = arg_end;
-
-    while (ap)
-    {
-        switch (ap->which)
-        {
-        case REGX_PATTERN:
-            if (ap->u.pattern.body)
-            {
-                arg_start[arg_no] = *pptr;
-                if (!tryMatch (spec, pptr, &sptr, ap->u.pattern.dfa, 0))
-                {
-                    arg_end[arg_no] = F_WIN_EOF;
-                    arg_no++;
-                    arg_start[arg_no] = F_WIN_EOF;
-                    arg_end[arg_no] = F_WIN_EOF;
-                   yaz_log(YLOG_DEBUG, "Pattern match rest of record");
-                   *pptr = F_WIN_EOF;
-                }
-                else
-                {
-                    arg_end[arg_no] = sptr;
-                    arg_no++;
-                    arg_start[arg_no] = sptr;
-                    arg_end[arg_no] = *pptr;
-                }
-            }
-            else
-            {
-                arg_start[arg_no] = *pptr;
-                if (!tryMatch (spec, pptr, &sptr, ap->u.pattern.dfa, 1))
-                    return 1;
-                if (sptr != arg_start[arg_no])
-                    return 1;
-                arg_end[arg_no] = *pptr;
-            }
-            arg_no++;
-            break;
-        case REGX_CODE:
-           spec->arg_no = arg_no;
-           spec->ptr = *pptr;
-#if HAVE_TCL_H
-           if (spec->tcl_interp)
-               execTcl(spec, ap->u.code);
-           else
-               execCode (spec, ap->u.code);
-#else
-           execCode (spec, ap->u.code);
-#endif
-           *pptr = spec->ptr;
-           if (spec->stop_flag)
-               return 0;
-            break;
-        case REGX_END:
-            arg_start[arg_no] = *pptr;
-            arg_end[arg_no] = F_WIN_EOF;
-            arg_no++;
-            *pptr = F_WIN_EOF;
-        }
-        ap = ap->next;
-    }
-    return 1;
-}
-
-static int execRule (struct lexSpec *spec, struct lexContext *context,
-                     int ruleNo, int start_ptr, int *pptr)
-{
-#if REGX_DEBUG
-    yaz_log (YLOG_LOG, "exec rule %d", ruleNo);
-#endif
-    return execAction (spec, context->fastRule[ruleNo]->actionList,
-                       start_ptr, pptr);
-}
-
-int lexNode (struct lexSpec *spec, int *ptr)
-{
-    struct lexContext *context = spec->context_stack[spec->context_stack_top];
-    struct DFA_state *state = context->dfa->states[0];
-    struct DFA_tran *t;
-    unsigned char c;
-    unsigned char c_prev = '\n';
-    int i;
-    int last_rule = 0;        /* rule number of current match */
-    int last_ptr = *ptr;      /* last char of match */
-    int start_ptr = *ptr;     /* first char of match */
-    int skip_ptr = *ptr;      /* first char of run */
-    int more = 0;
-
-    while (1)
-    {
-        c = f_win_advance (spec, ptr);
-        if (*ptr == F_WIN_EOF)
-        {
-           /* end of file met */
-            if (last_rule)
-            {
-               /* there was a match */
-                if (skip_ptr < start_ptr)
-                {
-                   /* deal with chars that didn't match */
-                    int size;
-                    char *buf;
-                    buf = f_win_get (spec, skip_ptr, start_ptr, &size);
-                    execDataP (spec, buf, size, 0);
-                }
-               /* restore pointer */
-                *ptr = last_ptr;
-               /* execute rule */
-                if (!execRule (spec, context, last_rule, start_ptr, ptr))
-                    return more;
-               /* restore skip pointer */
-                skip_ptr = *ptr;
-                last_rule = 0;
-            }
-            else if (skip_ptr < *ptr)
-            {
-               /* deal with chars that didn't match */
-                int size;
-                char *buf;
-                buf = f_win_get (spec, skip_ptr, *ptr, &size);
-                execDataP (spec, buf, size, 0);
-            }
-            state = context->dfa->states[0];
-            if (*ptr == F_WIN_EOF)
-                return more;
-        }
-        t = state->trans;
-        i = state->tran_no;
-        while (1)
-            if (--i < 0)
-            {   /* no transition for character c ... */
-                if (last_rule)
-                {
-                    if (skip_ptr < start_ptr)
-                    {
-                       /* deal with chars that didn't match */
-                        int size;
-                        char *buf;
-                        buf = f_win_get (spec, skip_ptr, start_ptr, &size);
-                        execDataP (spec, buf, size, 0);
-                    }
-                   /* restore pointer */
-                    *ptr = last_ptr;
-                    if (!execRule (spec, context, last_rule, start_ptr, ptr))
-                    {
-                        if (spec->f_win_ef && *ptr != F_WIN_EOF)
-                       {
-                            off_t end_offset = *ptr;
-#if REGX_DEBUG
-                           yaz_log (YLOG_LOG, "regx: endf ptr=%d", *ptr);
-#endif
-                            (*spec->f_win_ef)(spec->stream, &end_offset);
-                       }
-                        return more;
-                    }
-                   context = spec->context_stack[spec->context_stack_top];
-                    skip_ptr = *ptr;
-                    last_rule = 0;
-                    last_ptr = start_ptr = *ptr;
-                    if (start_ptr > 0)
-                    {
-                        --start_ptr;
-                        c_prev = f_win_advance (spec, &start_ptr);
-                    }
-                }
-                else
-                {
-                    c_prev = f_win_advance (spec, &start_ptr);
-                    *ptr = start_ptr;
-                }
-                state = context->dfa->states[0];
-                break;
-            }
-            else if (c >= t->ch[0] && c <= t->ch[1])
-            {   /* transition ... */
-                state = context->dfa->states[t->to];
-                if (state->rule_no)
-                {
-                    if (c_prev == '\n')
-                    {
-                        last_rule = state->rule_no;
-                        last_ptr = *ptr;
-                    } 
-                    else if (state->rule_nno)
-                    {
-                        last_rule = state->rule_nno;
-                        last_ptr = *ptr;
-                    }
-                    more = 1;
-                }
-                break;
-            }
-            else
-                t++;
-    }
-    return more;
-}
-
-static data1_node *lexRoot (struct lexSpec *spec, off_t offset,
-                           const char *context_name)
-{
-    struct lexContext *lt = spec->context;
-    int ptr = offset;
-    int ret;
-
-    spec->stop_flag = 0;
-    spec->d1_level = 0;
-    spec->context_stack_top = 0;    
-    while (lt)
-    {
-       if (!strcmp (lt->name, context_name))
-           break;
-       lt = lt->next;
-    }
-    if (!lt)
-    {
-       yaz_log (YLOG_WARN, "cannot find context %s", context_name);
-       return NULL;
-    }
-    spec->context_stack[spec->context_stack_top] = lt;
-    spec->d1_stack[spec->d1_level] = NULL;
-#if 1
-    if (!lt->initFlag)
-    {
-       lt->initFlag = 1;
-       execAction (spec, lt->initActionList, ptr, &ptr);
-    }
-#endif
-    execAction (spec, lt->beginActionList, ptr, &ptr);
-
-    ret = lexNode (spec, &ptr);
-    while (spec->d1_level)
-    {
-       tagDataRelease (spec);
-       (spec->d1_level)--;
-    }
-    if (!ret)
-        return 0;
-    execAction (spec, lt->endActionList, ptr, &ptr);
-    return spec->d1_stack[0];
-}
-
-void grs_destroy(void *clientData)
-{
-    struct lexSpecs *specs = (struct lexSpecs *) clientData;
-    if (specs->spec)
-    {
-       lexSpecDestroy(&specs->spec);
-    }
-    xfree (specs);
-}
-
-void *grs_init(Res res, RecType recType)
-{
-    struct lexSpecs *specs = (struct lexSpecs *) xmalloc (sizeof(*specs));
-    specs->spec = 0;
-    strcpy(specs->type, "");
-    return specs;
-}
-
-
-ZEBRA_RES grs_config(void *clientData, Res res, const char *args)
-{
-    struct lexSpecs *specs = (struct lexSpecs *) clientData;
-    if (strlen(args) < sizeof(specs->type))
-       strcpy(specs->type, args);
-    return ZEBRA_OK;
-}
-
-data1_node *grs_read_regx (struct grs_read_info *p)
-{
-    int res;
-    struct lexSpecs *specs = (struct lexSpecs *) p->clientData;
-    struct lexSpec **curLexSpec = &specs->spec;
-    off_t start_offset;
-
-#if REGX_DEBUG
-    yaz_log (YLOG_LOG, "grs_read_regx");
-#endif
-    if (!*curLexSpec || strcmp ((*curLexSpec)->name, specs->type))
-    {
-        if (*curLexSpec)
-            lexSpecDestroy (curLexSpec);
-        *curLexSpec = lexSpecCreate (specs->type, p->dh);
-        res = readFileSpec (*curLexSpec);
-        if (res)
-        {
-            lexSpecDestroy (curLexSpec);
-            return NULL;
-        }
-    }
-    (*curLexSpec)->dh = p->dh;
-    start_offset = p->stream->tellf(p->stream);
-    if (start_offset == 0)
-    {
-        (*curLexSpec)->f_win_start = 0;
-        (*curLexSpec)->f_win_end = 0;
-        (*curLexSpec)->f_win_rf = p->stream->readf;
-        (*curLexSpec)->f_win_sf = p->stream->seekf;
-        (*curLexSpec)->stream = p->stream;
-        (*curLexSpec)->f_win_ef = p->stream->endf;
-        (*curLexSpec)->f_win_size = 500000;
-    }
-    (*curLexSpec)->m = p->mem;
-    return lexRoot (*curLexSpec, start_offset, "main");
-}
-
-static int extract_regx(void *clientData, struct recExtractCtrl *ctrl)
-{
-    return zebra_grs_extract(clientData, ctrl, grs_read_regx);
-}
-
-static int retrieve_regx(void *clientData, struct recRetrieveCtrl *ctrl)
-{
-    return zebra_grs_retrieve(clientData, ctrl, grs_read_regx);
-}
-
-static struct recType regx_type = {
-    0,
-    "grs.regx",
-    grs_init,
-    grs_config,
-    grs_destroy,
-    extract_regx,
-    retrieve_regx,
-};
-
-
-#if HAVE_TCL_H
-data1_node *grs_read_tcl (struct grs_read_info *p)
-{
-    int res;
-    struct lexSpecs *specs = (struct lexSpecs *) p->clientData;
-    struct lexSpec **curLexSpec = &specs->spec;
-    off_t start_offset;
-
-#if REGX_DEBUG
-    yaz_log (YLOG_LOG, "grs_read_tcl");
-#endif
-    if (!*curLexSpec || strcmp ((*curLexSpec)->name, specs->type))
-    {
-       Tcl_Interp *tcl_interp;
-        if (*curLexSpec)
-            lexSpecDestroy (curLexSpec);
-        *curLexSpec = lexSpecCreate (specs->type, p->dh);
-       Tcl_FindExecutable("");
-       tcl_interp = (*curLexSpec)->tcl_interp = Tcl_CreateInterp();
-       Tcl_Init(tcl_interp);
-       Tcl_CreateCommand (tcl_interp, "begin", cmd_tcl_begin, *curLexSpec, 0);
-       Tcl_CreateCommand (tcl_interp, "end", cmd_tcl_end, *curLexSpec, 0);
-       Tcl_CreateCommand (tcl_interp, "data", cmd_tcl_data, *curLexSpec, 0);
-       Tcl_CreateCommand (tcl_interp, "unread", cmd_tcl_unread,
-                          *curLexSpec, 0);
-        res = readFileSpec (*curLexSpec);
-        if (res)
-        {
-            lexSpecDestroy (curLexSpec);
-            return NULL;
-        }
-    }
-    (*curLexSpec)->dh = p->dh;
-    start_offset = p->stream->tellf(p->stream);
-    if (start_offset == 0)
-    {
-        (*curLexSpec)->f_win_start = 0;
-        (*curLexSpec)->f_win_end = 0;
-        (*curLexSpec)->f_win_rf = p->stream->readf;
-        (*curLexSpec)->f_win_sf = p->stream->seekf;
-        (*curLexSpec)->stream = p->stream;
-        (*curLexSpec)->f_win_ef = p->stream->endf;
-        (*curLexSpec)->f_win_size = 500000;
-    }
-    (*curLexSpec)->m = p->mem;
-    return lexRoot (*curLexSpec, start_offset, "main");
-}
-
-static int extract_tcl(void *clientData, struct recExtractCtrl *ctrl)
-{
-    return zebra_grs_extract(clientData, ctrl, grs_read_tcl);
-}
-
-static int retrieve_tcl(void *clientData, struct recRetrieveCtrl *ctrl)
-{
-    return zebra_grs_retrieve(clientData, ctrl, grs_read_tcl);
-}
-
-static struct recType tcl_type = {
-    0,
-    "grs.tcl",
-    grs_init,
-    grs_config,
-    grs_destroy,
-    extract_tcl,
-    retrieve_tcl,
-};
-
-#endif
-
-RecType
-#ifdef IDZEBRA_STATIC_GRS_REGX
-idzebra_filter_grs_regx
-#else
-idzebra_filter
-#endif
-
-[] = {
-    &regx_type,
-#if HAVE_TCL_H
-    &tcl_type,
-#endif
-    0,
-};
-/*
- * Local variables:
- * c-basic-offset: 4
- * indent-tabs-mode: nil
- * End:
- * vim: shiftwidth=4 tabstop=8 expandtab
- */
-
diff --git a/index/safari.c b/index/safari.c
deleted file mode 100644 (file)
index be8c567..0000000
+++ /dev/null
@@ -1,320 +0,0 @@
-/* This file is part of the Zebra server.
-   Copyright (C) 1995-2008 Index Data
-
-Zebra is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 2, or (at your option) any later
-version.
-
-Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-
-*/
-
-#include <yaz/oid_db.h>
-#include <stdio.h>
-#include <assert.h>
-#include <ctype.h>
-
-#include <idzebra/util.h>
-#include <idzebra/recctrl.h>
-
-struct filter_info {
-    int segments;
-};
-
-static void *filter_init(Res res, RecType recType)
-{
-    struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo));
-    tinfo->segments = 0;
-    return tinfo;
-}
-
-static void *filter_init2(Res res, RecType recType)
-{
-    struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo));
-    tinfo->segments = 1;
-    return tinfo;
-}
-
-static ZEBRA_RES filter_config(void *clientData, Res res, const char *args)
-{
-    return ZEBRA_OK;
-}
-
-static void filter_destroy(void *clientData)
-{
-    struct filter_info *tinfo = clientData;
-    xfree (tinfo);
-}
-
-struct fi_info {
-    struct recExtractCtrl *p;
-    char *buf;
-    int offset;
-    int max;
-};
-
-static struct fi_info *fi_open(struct recExtractCtrl *p)
-{
-    struct fi_info *fi = (struct fi_info *) xmalloc (sizeof(*fi));
-
-    fi->p = p;
-    fi->buf = (char *) xmalloc (4096);
-    fi->offset = 1;
-    fi->max = 1;
-    return fi;
-}
-
-static int fi_getchar(struct fi_info *fi, char *dst)
-{
-    if (fi->offset >= fi->max)
-    {
-        if (fi->max <= 0)
-            return 0;
-        fi->max = fi->p->stream->readf(fi->p->stream, fi->buf, 4096);
-        fi->offset = 0;
-        if (fi->max <= 0)
-            return 0;
-    }
-    *dst = fi->buf[(fi->offset)++];
-    return 1;
-}
-
-static int fi_gets(struct fi_info *fi, char *dst, int max)
-{
-    int l = 0;
-    while(1)
-    {
-       char dstbyte;
-       if (!fi_getchar(fi, &dstbyte))
-           return 0;
-       if (dstbyte == '\n')
-           break;
-       if (l < max)
-           dst[l++] = dstbyte;
-    }
-    dst[l] = '\0';
-    return 1;
-}
-
-static void fi_close (struct fi_info *fi)
-{
-    xfree (fi->buf);
-    xfree (fi);
-}
-
-static int filter_extract(void *clientData, struct recExtractCtrl *p)
-{
-    struct filter_info *tinfo = clientData;
-    char line[512];
-    RecWord recWord;
-    int ret = RECCTRL_EXTRACT_OK;
-    struct fi_info *fi = fi_open(p);
-
-#if 0
-    yaz_log(YLOG_LOG, "filter_extract off=%ld",
-           (long) (*fi->p->tellf)(fi->p->fh));
-#endif
-    (*p->init)(p, &recWord);
-
-    if (!fi_gets(fi, line, sizeof(line)-1))
-        ret = RECCTRL_EXTRACT_EOF;
-    else
-    {
-        sscanf(line, "%255s", p->match_criteria);
-        while (fi_gets(fi, line, sizeof(line)-1))
-        {
-            int nor = 0;
-            char field[40];
-            const char *cp = line;
-            char type_cstr[2];
-#if 0
-            yaz_log(YLOG_LOG, "safari line: %s", line);
-#endif
-            type_cstr[1] = '\0';
-            if (*cp >= '0' && *cp <= '9')
-                type_cstr[0] = '0'; /* the default is 0 (raw) */
-            else
-                type_cstr[0] = *cp++; /* type given */
-            type_cstr[1] = '\0';
-
-            recWord.index_type = type_cstr;
-            if (tinfo->segments)
-            {
-                if (sscanf(cp, ZINT_FORMAT " " ZINT_FORMAT " " ZINT_FORMAT 
-                           ZINT_FORMAT " %39s %n",
-                           &recWord.record_id, &recWord.section_id, 
-                           &recWord.segment,
-                           &recWord.seqno,
-                           field, &nor) < 5)
-                {
-                    yaz_log(YLOG_WARN, "Bad safari record line: %s", line);
-                    ret = RECCTRL_EXTRACT_ERROR_GENERIC;
-                    break;
-                }
-            }
-            else
-            {
-                if (sscanf(cp, ZINT_FORMAT " " ZINT_FORMAT " " ZINT_FORMAT " %39s %n",
-                           &recWord.record_id, &recWord.section_id, &recWord.seqno,
-                           field, &nor) < 4)
-                {
-                    yaz_log(YLOG_WARN, "Bad safari record line: %s", line);
-                    ret = RECCTRL_EXTRACT_ERROR_GENERIC;
-                    break;
-                }
-            }
-            for (cp = cp + nor; *cp == ' '; cp++)
-                ;
-            recWord.index_name = field;
-            recWord.term_buf = cp;
-            recWord.term_len = strlen(cp);
-            (*p->tokenAdd)(&recWord);
-        }
-    }
-    fi_close(fi);
-    return ret;
-}
-
-static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p)
-{
-    int r, filter_ptr = 0;
-    static char *filter_buf = NULL;
-    static int filter_size = 0;
-    int make_header = 1;
-    int make_body = 1;
-    const char *elementSetName = NULL;
-    int no_lines = 0;
-
-    if (p->comp && p->comp->which == Z_RecordComp_simple &&
-        p->comp->u.simple->which == Z_ElementSetNames_generic)
-        elementSetName = p->comp->u.simple->u.generic;
-
-    if (elementSetName)
-    {
-       /* don't make header for the R(aw) element set name */
-       if (!strcmp(elementSetName, "R"))
-       {
-           make_header = 0;
-           make_body = 1;
-       }
-       /* only make header for the H(eader) element set name */
-       else if (!strcmp(elementSetName, "H"))
-       {
-           make_header = 1;
-           make_body = 0;
-       }
-    }
-    while (1)
-    {
-        if (filter_ptr + 4096 >= filter_size)
-        {
-            char *nb;
-
-            filter_size = 2*filter_size + 8192;
-            nb = (char *) xmalloc (filter_size);
-            if (filter_buf)
-            {
-                memcpy (nb, filter_buf, filter_ptr);
-                xfree (filter_buf);
-            }
-            filter_buf = nb;
-        }
-        if (make_header && filter_ptr == 0)
-        {
-            if (p->score >= 0)
-            {
-                sprintf (filter_buf, "Rank: %d\n", p->score);
-                filter_ptr = strlen(filter_buf);
-            }
-            sprintf (filter_buf + filter_ptr, "Local Number: " ZINT_FORMAT "\n",
-                    p->localno);
-            filter_ptr = strlen(filter_buf);
-           if (p->fname)
-           {
-               sprintf (filter_buf + filter_ptr, "Filename: %s\n", p->fname);
-               filter_ptr = strlen(filter_buf);
-           }
-           strcpy(filter_buf+filter_ptr++, "\n");
-        }
-       if (!make_body)
-           break;
-        r = p->stream->readf(p->stream, filter_buf + filter_ptr, 4096);
-        if (r <= 0)
-            break;
-        filter_ptr += r;
-    }
-    filter_buf[filter_ptr] = '\0';
-    if (elementSetName)
-    {
-        if (!strcmp (elementSetName, "B"))
-            no_lines = 4;
-        if (!strcmp (elementSetName, "M"))
-            no_lines = 20;
-    }
-    if (no_lines)
-    {
-        char *p = filter_buf;
-        int i = 0;
-
-        while (++i <= no_lines && (p = strchr (p, '\n')))
-            p++;
-        if (p)
-        {
-            p[1] = '\0';
-            filter_ptr = p-filter_buf;
-        }
-    }
-    p->output_format = yaz_oid_recsyn_sutrs;
-    p->rec_buf = filter_buf;
-    p->rec_len = filter_ptr; 
-    return 0;
-}
-
-static struct recType filter_type = {
-    0,
-    "safari",
-    filter_init,
-    filter_config,
-    filter_destroy,
-    filter_extract,
-    filter_retrieve
-};
-
-static struct recType filter_type2 = {
-    0,
-    "safari2",
-    filter_init2,
-    filter_config,
-    filter_destroy,
-    filter_extract,
-    filter_retrieve
-};
-
-RecType
-#ifdef IDZEBRA_STATIC_SAFARI
-idzebra_filter_safari
-#else
-idzebra_filter
-#endif
-
-[] = {
-    &filter_type,
-    &filter_type2,
-    0,
-};
-/*
- * Local variables:
- * c-basic-offset: 4
- * indent-tabs-mode: nil
- * End:
- * vim: shiftwidth=4 tabstop=8 expandtab
- */
-
diff --git a/index/sgmlread.c b/index/sgmlread.c
deleted file mode 100644 (file)
index 38fcbe3..0000000
+++ /dev/null
@@ -1,143 +0,0 @@
-/* This file is part of the Zebra server.
-   Copyright (C) 1995-2008 Index Data
-
-Zebra is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 2, or (at your option) any later
-version.
-
-Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-
-*/
-
-
-#include <assert.h>
-#include <yaz/log.h>
-
-#include <idzebra/recgrs.h>
-
-struct sgml_getc_info {
-    char *buf;
-    int buf_size;
-    int size;
-    int off;
-    struct ZebraRecStream *stream;
-    off_t moffset;
-    WRBUF wrbuf;
-};
-
-int sgml_getc (void *clientData)
-{
-    struct sgml_getc_info *p = (struct sgml_getc_info *) clientData;
-    int res;
-    
-    if (p->off < p->size)
-       return p->buf[(p->off)++];
-    if (p->size < p->buf_size)
-       return 0;
-    p->moffset += p->off;
-    p->off = 0;
-    p->size = 0;
-    res = p->stream->readf(p->stream, p->buf, p->buf_size);
-    if (res > 0)
-    {
-       p->size += res;
-       return p->buf[(p->off)++];
-    }
-    return 0;
-}
-
-static data1_node *grs_read_sgml (struct grs_read_info *p)
-{
-    struct sgml_getc_info *sgi = (struct sgml_getc_info *) p->clientData;
-    data1_node *node;
-    int res;
-    
-    sgi->moffset = p->stream->tellf(p->stream);
-    sgi->stream = p->stream;
-    sgi->off = 0;
-    sgi->size = 0;
-    res = sgi->stream->readf(sgi->stream, sgi->buf, sgi->buf_size);
-    if (res > 0)
-       sgi->size += res;
-    else
-       return 0;
-    node = data1_read_nodex(p->dh, p->mem, sgml_getc, sgi, sgi->wrbuf);
-    if (node && p->stream->endf)
-    {
-        off_t end_offset = sgi->moffset + sgi->off;
-       p->stream->endf(sgi->stream, &end_offset);
-    }
-    return node;
-}
-
-static void *grs_init_sgml(Res res, RecType recType)
-{
-    struct sgml_getc_info *p = (struct sgml_getc_info *) xmalloc (sizeof(*p));
-    p->buf_size = 512;
-    p->buf = xmalloc (p->buf_size);
-    p->wrbuf = wrbuf_alloc();
-    return p;
-}
-
-static ZEBRA_RES grs_config_sgml(void *clientData, Res res, const char *args)
-{
-    return ZEBRA_OK;
-}
-
-static void grs_destroy_sgml(void *clientData)
-{
-    struct sgml_getc_info *p = (struct sgml_getc_info *) clientData;
-
-    wrbuf_destroy(p->wrbuf);
-    xfree(p->buf);
-    xfree(p);
-}
-
-static int grs_extract_sgml(void *clientData, struct recExtractCtrl *ctrl)
-{
-    return zebra_grs_extract(clientData, ctrl, grs_read_sgml);
-}
-
-static int grs_retrieve_sgml(void *clientData, struct recRetrieveCtrl *ctrl)
-{
-    return zebra_grs_retrieve(clientData, ctrl, grs_read_sgml);
-}
-
-static struct recType grs_type_sgml =
-{
-    0,
-    "grs.sgml",
-    grs_init_sgml,
-    grs_config_sgml,
-    grs_destroy_sgml,
-    grs_extract_sgml,
-    grs_retrieve_sgml
-};
-
-RecType
-#ifdef IDZEBRA_STATIC_GRS_SGML
-idzebra_filter_grs_sgml
-#else
-idzebra_filter
-#endif
-
-[] = {
-    &grs_type_sgml,
-    0,
-};
-/*
- * Local variables:
- * c-basic-offset: 4
- * indent-tabs-mode: nil
- * End:
- * vim: shiftwidth=4 tabstop=8 expandtab
- */
-
diff --git a/index/xmlread.c b/index/xmlread.c
deleted file mode 100644 (file)
index 248342b..0000000
+++ /dev/null
@@ -1,547 +0,0 @@
-/* This file is part of the Zebra server.
-   Copyright (C) 1995-2008 Index Data
-
-Zebra is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 2, or (at your option) any later
-version.
-
-Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-
-*/
-
-#if HAVE_EXPAT_H
-
-#include <assert.h>
-#include <stdio.h>
-#include <stdlib.h>
-#if HAVE_ICONV_H
-#include <errno.h>
-#include <iconv.h>
-#endif
-
-#include <yaz/log.h>
-
-#include <idzebra/recgrs.h>
-
-#include <yaz/log.h>
-#include <yaz/xmalloc.h>
-
-#include <expat.h>
-
-#define XML_CHUNK 1024
-
-struct user_info {
-    data1_node *d1_stack[256];
-    int level;
-    data1_handle dh;
-    NMEM nmem;
-    int loglevel;
-};
-
-static void cb_start (void *user, const char *el, const char **attr)
-{
-    struct user_info *ui = (struct user_info*) user;
-    if (ui->level == 1)
-        data1_set_root (ui->dh, ui->d1_stack[0], ui->nmem, el);
-    ui->d1_stack[ui->level] = data1_mk_tag (ui->dh, ui->nmem, el, attr,
-                                                ui->d1_stack[ui->level-1]);
-    ui->level++;
-    yaz_log (ui->loglevel, "cb_start %s", el);
-}
-
-static void cb_end (void *user, const char *el)
-{
-    struct user_info *ui = (struct user_info*) user;
-
-    ui->level--;
-    yaz_log (ui->loglevel, "cb_end %s", el);
-}
-
-static void cb_chardata (void *user, const char *s, int len)
-{
-    struct user_info *ui = (struct user_info*) user;
-#if 0
-    yaz_log (ui->loglevel, "cb_chardata %.*s", len, s);
-#endif
-    ui->d1_stack[ui->level] = data1_mk_text_n (ui->dh, ui->nmem, s, len,
-                                                   ui->d1_stack[ui->level -1]);
-}
-
-static void cb_decl (void *user, const char *version, const char *encoding,
-                     int standalone)
-{
-    struct user_info *ui = (struct user_info*) user;
-    const char *attr_list[7];
-
-    attr_list[0] = "version";
-    attr_list[1] = version;
-
-    attr_list[2] = "encoding";
-    attr_list[3] = "UTF-8"; /* internally it's always UTF-8 */
-
-    attr_list[4] = "standalone";
-    attr_list[5] = standalone  ? "yes" : "no";
-
-    attr_list[6] = 0;
-    
-    data1_mk_preprocess (ui->dh, ui->nmem, "xml", attr_list,
-                             ui->d1_stack[ui->level-1]);
-#if 0
-    yaz_log (YLOG_LOG, "decl version=%s encoding=%s",
-             version ? version : "null",
-             encoding ? encoding : "null");
-#endif
-}
-    
-static void cb_processing (void *user, const char *target,
-                           const char *data)
-{
-    struct user_info *ui = (struct user_info*) user;
-    data1_node *res =
-        data1_mk_preprocess (ui->dh, ui->nmem, target, 0,
-                             ui->d1_stack[ui->level-1]);
-    data1_mk_text_nf (ui->dh, ui->nmem, data, strlen(data), res);
-    
-    yaz_log (ui->loglevel, "decl processing target=%s data=%s",
-             target ? target : "null",
-             data ? data : "null");
-}
-
-static void cb_comment (void *user, const char *data)
-{
-    struct user_info *ui = (struct user_info*) user;
-    yaz_log (ui->loglevel, "decl comment data=%s", data ? data : "null");
-    data1_mk_comment (ui->dh, ui->nmem, data, ui->d1_stack[ui->level-1]);
-}
-
-static void cb_doctype_start (void *userData, const char *doctypeName,
-                              const char *sysid, const char *pubid,
-                              int has_internal_subset)
-{
-    struct user_info *ui = (struct user_info*) userData;
-    yaz_log (ui->loglevel, "doctype start doctype=%s sysid=%s pubid=%s",
-             doctypeName, sysid, pubid);
-}
-
-static void cb_doctype_end (void *userData)
-{
-    struct user_info *ui = (struct user_info*) userData;
-    yaz_log (ui->loglevel, "doctype end");
-}
-
-
-static void cb_entity_decl (void *userData, const char *entityName,
-                            int is_parameter_entity,
-                            const char *value, int value_length,
-                            const char *base, const char *systemId,
-                            const char *publicId, const char *notationName)
-{
-    struct user_info *ui = (struct user_info*) userData;
-    yaz_log (ui->loglevel,
-             "entity decl %s is_para_entry=%d value=%.*s base=%s systemId=%s"
-             " publicId=%s notationName=%s",
-             entityName, is_parameter_entity, value_length, value,
-             base, systemId, publicId, notationName);
-    
-}
-
-static int cb_external_entity(XML_Parser pparser,
-                              const char *context,
-                              const char *base,
-                              const char *systemId,
-                              const char *publicId)
-{
-    struct user_info *ui = (struct user_info*) XML_GetUserData(pparser);
-    FILE *inf;
-    int done = 0;
-    XML_Parser parser;
-
-    yaz_log (ui->loglevel,
-             "external entity context=%s base=%s systemid=%s publicid=%s",
-             context, base, systemId, publicId);
-    if (!systemId)
-        return 1;
-
-    if (!(inf = fopen (systemId, "rb")))
-    {
-        yaz_log (YLOG_WARN|YLOG_ERRNO, "fopen %s", systemId);
-        return 0;
-    }
-
-    parser = XML_ExternalEntityParserCreate (pparser, "", 0);
-    while (!done)
-    {
-        int r;
-        void *buf = XML_GetBuffer (parser, XML_CHUNK);
-        if (!buf)
-        {
-            yaz_log (YLOG_WARN, "XML_GetBuffer fail");
-            break;
-        }
-        r = fread (buf, 1, XML_CHUNK, inf);
-        if (r == 0)
-        {
-            if (ferror(inf))
-            {
-                yaz_log (YLOG_WARN|YLOG_ERRNO, "fread %s", systemId);
-                break;
-            }
-            done = 1;
-        }
-        if (!XML_ParseBuffer (parser, r, done))
-        {
-           done = 1;
-           yaz_log (YLOG_WARN, "%s:%d:%d:XML error: %s",
-                    systemId,
-                    XML_GetCurrentLineNumber(parser),
-                    XML_GetCurrentColumnNumber(parser),
-                    XML_ErrorString(XML_GetErrorCode(parser)));
-       }
-    }
-    fclose (inf);
-    XML_ParserFree (parser);
-    return done;
-}
-
-
-#if HAVE_ICONV_H
-static int cb_encoding_convert (void *data, const char *s)
-{
-    iconv_t t = (iconv_t) data;
-    size_t ret;
-    size_t outleft = 2;
-    char outbuf_[2], *outbuf = outbuf_;
-    size_t inleft = 4;
-    char *inbuf = (char *) s;
-    unsigned short code;
-
-#if 1
-    yaz_log(YLOG_LOG, "------------------------- cb_encoding_convert --- ");
-#endif
-    ret = iconv (t, &inbuf, &inleft, &outbuf, &outleft);
-    if (ret == (size_t) (-1) && errno != E2BIG)
-    {
-        iconv (t, 0, 0, 0, 0);
-        return -1;
-    }
-    if (outleft != 0)
-        return -1;
-    memcpy (&code, outbuf_, sizeof(short));
-    return code;
-}
-
-static void cb_encoding_release (void *data)
-{
-    iconv_t t = (iconv_t) data;
-    iconv_close (t);
-}
-
-static int cb_encoding_handler (void *userData, const char *name,
-                                XML_Encoding *info)
-{
-    int i = 0;
-    int no_ok = 0;
-    struct user_info *ui = (struct user_info*) userData;
-
-    iconv_t t = iconv_open ("UNICODE", name);
-    if (t == (iconv_t) (-1))
-        return 0;
-   
-    info->data = 0;  /* signal that multibyte is not in use */
-    yaz_log (ui->loglevel, "Encoding handler of %s", name);
-    for (i = 0; i<256; i++)
-    {
-        size_t ret;
-        char outbuf_[5];
-        char inbuf_[5];
-        char *inbuf = inbuf_;
-        char *outbuf = outbuf_;
-        size_t inleft = 1;
-        size_t outleft = 2;
-        inbuf_[0] = i;
-
-        iconv (t, 0, 0, 0, 0);  /* reset iconv */
-
-        ret = iconv(t, &inbuf, &inleft, &outbuf, &outleft);
-        if (ret == (size_t) (-1))
-        {
-            if (errno == EILSEQ)
-            {
-                yaz_log (ui->loglevel, "Encoding %d: invalid sequence", i);
-                info->map[i] = -1;  /* invalid sequence */
-            }
-            if (errno == EINVAL)
-            {                       /* multi byte input */
-                int len = 2;
-                int j = 0;
-                info->map[i] = -1;
-                
-                while (len <= 4)
-                {
-                    char sbuf[80];
-                    int k;
-                    inbuf = inbuf_;
-                    inleft = len;
-                    outbuf = outbuf_;
-                    outleft = 2;
-
-                    inbuf_[len-1] = j;
-                    iconv (t, 0,0,0,0);
-
-                    assert (i >= 0 && i<255);
-
-                    *sbuf = 0;
-                    for (k = 0; k<len; k++)
-                    {
-                        sprintf (sbuf+strlen(sbuf), "%d ", inbuf_[k]&255);
-                    }
-                    ret = iconv (t, &inbuf, &inleft, &outbuf, &outleft);
-                    if (ret == (size_t) (-1))
-                    {
-                        if (errno == EILSEQ || errno == E2BIG)
-                        {
-                            j++;
-                            if (j > 255)
-                                break;
-                        }
-                        else if (errno == EINVAL)
-                        {
-                            len++;
-                            j = 7;
-                        }
-                    }
-                    else if (outleft == 0)
-                    {
-                        info->map[i] = -len;
-                        info->data = t;  /* signal that multibyte is in use */
-                        break;
-                    }
-                    else
-                    {
-                        break;
-                    }
-                }
-                if (info->map[i] < -1)
-                    yaz_log (ui->loglevel, "Encoding %d: multibyte input %d",
-                             i, -info->map[i]);
-                else
-                    yaz_log (ui->loglevel, "Encoding %d: multibyte input failed",
-                             i);
-            }
-            if (errno == E2BIG)
-            {
-                info->map[i] = -1;  /* no room for output */
-                if (i != 0)
-                    yaz_log (YLOG_WARN, "Encoding %d: no room for output",
-                             i);
-            }
-        }
-        else if (outleft == 0)
-        {
-            unsigned short code;
-            memcpy (&code, outbuf_, sizeof(short));
-            info->map[i] = code;
-            no_ok++;
-        }
-        else
-        {   /* should never happen */
-            info->map[i] = -1;
-            yaz_log (YLOG_DEBUG, "Encoding %d: bad state", i);
-        }
-    }
-    if (info->data)
-    {   /* at least one multi byte */
-        info->convert = cb_encoding_convert;
-        info->release = cb_encoding_release;
-    }
-    else
-    {
-        /* no multi byte - we no longer need iconv handler */
-        iconv_close(t);
-        info->convert = 0;
-        info->release = 0;
-    }
-    if (!no_ok)
-        return 0;
-    return 1;
-}
-/* HAVE_ICONV_H */
-#endif
-
-static void cb_ns_start(void *userData, const char *prefix, const char *uri)
-{
-    struct user_info *ui = (struct user_info*) userData;
-    if (prefix && uri)
-       yaz_log(ui->loglevel, "cb_ns_start %s %s", prefix, uri);
-}
-
-static void cb_ns_end(void *userData, const char *prefix)
-{
-    struct user_info *ui = (struct user_info*) userData;
-    if (prefix)
-       yaz_log(ui->loglevel, "cb_ns_end %s", prefix);
-}
-
-data1_node *zebra_read_xml(data1_handle dh,
-                           struct ZebraRecStream *stream,
-                           NMEM m)
-{
-    XML_Parser parser;
-    struct user_info uinfo;
-    int done = 0;
-    data1_node *first_node;
-    int no_read = 0;
-
-    uinfo.loglevel = YLOG_DEBUG;
-    uinfo.level = 1;
-    uinfo.dh = dh;
-    uinfo.nmem = m;
-    uinfo.d1_stack[0] = data1_mk_node2 (dh, m, DATA1N_root, 0);
-    uinfo.d1_stack[1] = 0; /* indicate no children (see end of routine) */
-    
-    parser = XML_ParserCreate (0 /* encoding */);
-    
-    XML_SetElementHandler (parser, cb_start, cb_end);
-    XML_SetCharacterDataHandler (parser, cb_chardata);
-    XML_SetXmlDeclHandler (parser, cb_decl);
-    XML_SetProcessingInstructionHandler (parser, cb_processing);
-    XML_SetUserData (parser, &uinfo);
-    XML_SetCommentHandler (parser, cb_comment);
-    XML_SetDoctypeDeclHandler (parser, cb_doctype_start, cb_doctype_end);
-    XML_SetEntityDeclHandler (parser, cb_entity_decl);
-    XML_SetExternalEntityRefHandler (parser, cb_external_entity);
-    XML_SetNamespaceDeclHandler(parser, cb_ns_start, cb_ns_end);
-#if HAVE_ICONV_H
-    XML_SetUnknownEncodingHandler (parser, cb_encoding_handler, &uinfo);
-#endif
-    while (!done)
-    {
-        int r;
-        void *buf = XML_GetBuffer (parser, XML_CHUNK);
-        if (!buf)
-        {
-            /* error */
-            yaz_log (YLOG_WARN, "XML_GetBuffer fail");
-            break;
-        }
-        r = stream->readf(stream, buf, XML_CHUNK);
-        if (r < 0)
-        {
-            /* error */
-            yaz_log (YLOG_WARN, "XML read fail");
-            break;
-        }
-        else if (r == 0)
-            done = 1;
-       else
-           no_read += r;
-        if (no_read && !XML_ParseBuffer (parser, r, done))
-        {
-      &