From: Adam Dickmeiss Date: Thu, 11 Sep 2008 13:57:59 +0000 (+0200) Subject: renamed record filter module sources X-Git-Tag: v2.0.34~19 X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=commitdiff_plain;h=b3679b1ecd730266f70a62347f66e5964a2fa030 renamed record filter module sources --- diff --git a/index/Makefile.am b/index/Makefile.am index 88e8bd1..688e112 100644 --- a/index/Makefile.am +++ b/index/Makefile.am @@ -20,27 +20,27 @@ modlibdir=$(libdir)/$(PACKAGE)$(PACKAGE_SUFFIX)/modules tabdatadir = $(datadir)/$(PACKAGE)$(PACKAGE_SUFFIX)/tab # The shared modules -mod_grs_regx_la_SOURCES = regxread.c +mod_grs_regx_la_SOURCES = mod_grs_regx.c mod_grs_regx_la_LDFLAGS = -rpath $(modlibdir) -module -avoid-version mod_grs_regx_la_LADD = $(TCL_LIB) mod_grs_regx_la_LIBADD = $(zebralib) $(mod_grs_regx_la_LADD) -mod_grs_xml_la_SOURCES = xmlread.c +mod_grs_xml_la_SOURCES = mod_grs_xml.c mod_grs_xml_la_LDFLAGS = -rpath $(modlibdir) -module -avoid-version mod_grs_xml_la_LADD = $(EXPAT_LIBS) mod_grs_xml_la_LIBADD = $(zebralib) $(mod_grs_xml_la_LADD) -mod_grs_marc_la_SOURCES = marcread.c marcomp.c marcomp.h inline.c inline.h +mod_grs_marc_la_SOURCES = mod_grs_marc.c marcomp.c marcomp.h inline.c inline.h mod_grs_marc_la_LDFLAGS = -rpath $(modlibdir) -module -avoid-version mod_grs_marc_la_LADD = mod_grs_marc_la_LIBADD = $(zebralib) $(mod_grs_marc_la_LADD) -mod_safari_la_SOURCES = safari.c +mod_safari_la_SOURCES = mod_safari.c mod_safari_la_LDFLAGS = -rpath $(modlibdir) -module -avoid-version mod_safari_la_LADD = mod_safari_la_LIBADD = $(zebralib) $(mod_safari_la_LADD) -mod_alvis_la_SOURCES = alvis.c +mod_alvis_la_SOURCES = mod_alvis.c mod_alvis_la_LDFLAGS = -rpath $(modlibdir) -module -avoid-version mod_alvis_la_LADD = mod_alvis_la_LIBADD = $(zebralib) $(mod_alvis_la_LADD) @@ -50,7 +50,7 @@ mod_dom_la_LDFLAGS = -rpath $(modlibdir) -module -avoid-version mod_dom_la_LADD = mod_dom_la_LIBADD = $(zebralib) $(mod_dom_la_LADD) -mod_text_la_SOURCES = rectext.c +mod_text_la_SOURCES = mod_text.c mod_text_la_LDFLAGS = -rpath $(modlibdir) -module -avoid-version mod_text_la_LADD = mod_text_la_LIBADD = $(zebralib) $(mod_text_la_LADD) @@ -82,7 +82,7 @@ libidzebra_2_0_la_LDFLAGS=-export-dynamic -version-info $(ZEBRALIBS_VERSION_INFO libidzebra_2_0_la_SOURCES = \ recctrl.c \ recgrs.c \ - sgmlread.c \ + mod_grs_sgml.c \ attribute.c \ compact.c \ dir.c dirs.c \ diff --git a/index/alvis.c b/index/alvis.c deleted file mode 100644 index a752890..0000000 --- a/index/alvis.c +++ /dev/null @@ -1,729 +0,0 @@ -/* This file is part of the Zebra server. - Copyright (C) 1995-2008 Index Data - -Zebra is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free -Software Foundation; either version 2, or (at your option) any later -version. - -Zebra is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*/ - -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#if YAZ_HAVE_EXSLT -#include -#endif - -#include -#include - -struct filter_schema { - const char *name; - const char *identifier; - const char *stylesheet; - struct filter_schema *next; - const char *default_schema; - /* char default_schema; */ - xsltStylesheetPtr stylesheet_xsp; -}; - -struct filter_info { - xmlDocPtr doc; - char *fname; - char *full_name; - const char *profile_path; - int split_level; - const char *split_path; - ODR odr; - struct filter_schema *schemas; - xmlTextReaderPtr reader; -}; - -#define ZEBRA_SCHEMA_XSLT_NS "http://indexdata.dk/zebra/xslt/1" - -#define XML_STRCMP(a,b) strcmp((char*)a, b) -#define XML_STRLEN(a) strlen((char*)a) - -static const char *zebra_xslt_ns = ZEBRA_SCHEMA_XSLT_NS; - -static void set_param_str(const char **params, const char *name, - const char *value, ODR odr) -{ - char *quoted = odr_malloc(odr, 3 + strlen(value)); - sprintf(quoted, "'%s'", value); - while (*params) - params++; - params[0] = name; - params[1] = quoted; - params[2] = 0; -} - -static void set_param_int(const char **params, const char *name, - zint value, ODR odr) -{ - char *quoted = odr_malloc(odr, 30); /* 25 digits enough for 2^64 */ - while (*params) - params++; - sprintf(quoted, "'" ZINT_FORMAT "'", value); - params[0] = name; - params[1] = quoted; - params[2] = 0; -} - -#define ENABLE_INPUT_CALLBACK 0 - -#if ENABLE_INPUT_CALLBACK -static int zebra_xmlInputMatchCallback (char const *filename) -{ - yaz_log(YLOG_LOG, "match %s", filename); - return 0; -} - -static void * zebra_xmlInputOpenCallback (char const *filename) -{ - return 0; -} - -static int zebra_xmlInputReadCallback (void * context, char * buffer, int len) -{ - return 0; -} - -static int zebra_xmlInputCloseCallback (void * context) -{ - return 0; -} -#endif - -static void *filter_init(Res res, RecType recType) -{ - struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo)); - tinfo->reader = 0; - tinfo->fname = 0; - tinfo->full_name = 0; - tinfo->profile_path = 0; - tinfo->split_level = 0; - tinfo->split_path = 0; - tinfo->odr = odr_createmem(ODR_ENCODE); - tinfo->doc = 0; - tinfo->schemas = 0; - -#if YAZ_HAVE_EXSLT - exsltRegisterAll(); -#endif - -#if ENABLE_INPUT_CALLBACK - xmlRegisterDefaultInputCallbacks(); - xmlRegisterInputCallbacks(zebra_xmlInputMatchCallback, - zebra_xmlInputOpenCallback, - zebra_xmlInputReadCallback, - zebra_xmlInputCloseCallback); -#endif - return tinfo; -} - -static int attr_content(struct _xmlAttr *attr, const char *name, - const char **dst_content) -{ - if (!XML_STRCMP(attr->name, name) && attr->children - && attr->children->type == XML_TEXT_NODE) - { - *dst_content = (const char *)(attr->children->content); - return 1; - } - return 0; -} - -static void destroy_schemas(struct filter_info *tinfo) -{ - struct filter_schema *schema = tinfo->schemas; - while (schema) - { - struct filter_schema *schema_next = schema->next; - if (schema->stylesheet_xsp) - xsltFreeStylesheet(schema->stylesheet_xsp); - xfree(schema); - schema = schema_next; - } - tinfo->schemas = 0; - xfree(tinfo->fname); - if (tinfo->doc) - xmlFreeDoc(tinfo->doc); - tinfo->doc = 0; -} - -static ZEBRA_RES create_schemas(struct filter_info *tinfo, const char *fname) -{ - char tmp_full_name[1024]; - xmlNodePtr ptr; - tinfo->fname = xstrdup(fname); - - if (yaz_filepath_resolve(tinfo->fname, tinfo->profile_path, - NULL, tmp_full_name)) - tinfo->full_name = xstrdup(tmp_full_name); - else - tinfo->full_name = xstrdup(tinfo->fname); - - yaz_log(YLOG_LOG, "alvis filter: loading config file %s", tinfo->full_name); - - tinfo->doc = xmlParseFile(tinfo->full_name); - - if (!tinfo->doc) - { - yaz_log(YLOG_WARN, "alvis filter: could not parse config file %s", - tinfo->full_name); - - return ZEBRA_FAIL; - } - - ptr = xmlDocGetRootElement(tinfo->doc); - if (!ptr || ptr->type != XML_ELEMENT_NODE - || XML_STRCMP(ptr->name, "schemaInfo")) - { - yaz_log(YLOG_WARN, - "alvis filter: config file %s :" - " expected root element ", - tinfo->full_name); - return ZEBRA_FAIL; - } - - for (ptr = ptr->children; ptr; ptr = ptr->next) - { - if (ptr->type != XML_ELEMENT_NODE) - continue; - if (!XML_STRCMP(ptr->name, "schema")) - { - struct _xmlAttr *attr; - struct filter_schema *schema = xmalloc(sizeof(*schema)); - schema->name = 0; - schema->identifier = 0; - schema->stylesheet = 0; - schema->default_schema = 0; - schema->next = tinfo->schemas; - schema->stylesheet_xsp = 0; - tinfo->schemas = schema; - for (attr = ptr->properties; attr; attr = attr->next) - { - attr_content(attr, "identifier", &schema->identifier); - attr_content(attr, "name", &schema->name); - attr_content(attr, "stylesheet", &schema->stylesheet); - attr_content(attr, "default", &schema->default_schema); - } - /*yaz_log(YLOG_LOG, "XSLT add %s %s %s", - schema->name, schema->identifier, schema->stylesheet); */ - - /* find requested schema */ - - if (schema->stylesheet) - { - char tmp_xslt_full_name[1024]; - if (!yaz_filepath_resolve(schema->stylesheet, tinfo->profile_path, - NULL, tmp_xslt_full_name)) - { - yaz_log(YLOG_WARN, - "alvis filter: stylesheet %s not found in path %s", - schema->stylesheet, tinfo->profile_path); - return ZEBRA_FAIL; - } - schema->stylesheet_xsp - = xsltParseStylesheetFile((const xmlChar*) tmp_xslt_full_name); - if (!schema->stylesheet_xsp) - { - yaz_log(YLOG_WARN, - "alvis filter: could not parse xslt stylesheet %s", - tmp_xslt_full_name); - return ZEBRA_FAIL; - } - } - } - else if (!XML_STRCMP(ptr->name, "split")) - { - struct _xmlAttr *attr; - for (attr = ptr->properties; attr; attr = attr->next) - { - const char *split_level_str = 0; - attr_content(attr, "level", &split_level_str); - tinfo->split_level = - split_level_str ? atoi(split_level_str) : 0; - } - } - else - { - yaz_log(YLOG_WARN, "Bad element %s in %s", ptr->name, fname); - return ZEBRA_FAIL; - } - } - return ZEBRA_OK; -} - -static struct filter_schema *lookup_schema(struct filter_info *tinfo, - const char *est) -{ - struct filter_schema *schema; - - for (schema = tinfo->schemas; schema; schema = schema->next) - { - /* find requested schema */ - if (est) - { - if (schema->identifier && !strcmp(schema->identifier, est)) - return schema; - - if (schema->name && !strcmp(schema->name, est)) - return schema; - } - /* or return default schema if defined */ - else if (schema->default_schema) - return schema; - } - - /* return first schema if no default schema defined */ - if (tinfo->schemas) - return tinfo->schemas; - - return 0; -} - -static ZEBRA_RES filter_config(void *clientData, Res res, const char *args) -{ - struct filter_info *tinfo = clientData; - if (!args || !*args) - { - yaz_log(YLOG_WARN, "alvis filter: need config file"); - return ZEBRA_FAIL; - } - - if (tinfo->fname && !strcmp(args, tinfo->fname)) - return ZEBRA_OK; - - tinfo->profile_path = res_get(res, "profilePath"); - yaz_log(YLOG_LOG, "alvis filter: profilePath %s", tinfo->profile_path); - - destroy_schemas(tinfo); - return create_schemas(tinfo, args); -} - -static void filter_destroy(void *clientData) -{ - struct filter_info *tinfo = clientData; - destroy_schemas(tinfo); - xfree(tinfo->full_name); - if (tinfo->reader) - xmlFreeTextReader(tinfo->reader); - odr_destroy(tinfo->odr); - xfree(tinfo); -} - -static int ioread_ex(void *context, char *buffer, int len) -{ - struct recExtractCtrl *p = context; - return p->stream->readf(p->stream, buffer, len); -} - -static int ioclose_ex(void *context) -{ - return 0; -} - -static void index_cdata(struct filter_info *tinfo, struct recExtractCtrl *ctrl, - xmlNodePtr ptr, RecWord *recWord) -{ - for(; ptr; ptr = ptr->next) - { - index_cdata(tinfo, ctrl, ptr->children, recWord); - if (ptr->type != XML_TEXT_NODE) - continue; - recWord->term_buf = (const char *)ptr->content; - recWord->term_len = XML_STRLEN(ptr->content); - (*ctrl->tokenAdd)(recWord); - } -} - -static void index_node(struct filter_info *tinfo, struct recExtractCtrl *ctrl, - xmlNodePtr ptr, RecWord *recWord) -{ - for(; ptr; ptr = ptr->next) - { - index_node(tinfo, ctrl, ptr->children, recWord); - if (ptr->type != XML_ELEMENT_NODE || !ptr->ns || - XML_STRCMP(ptr->ns->href, zebra_xslt_ns)) - continue; - if (!XML_STRCMP(ptr->name, "index")) - { - const char *name_str = 0; - const char *type_str = 0; - const char *xpath_str = 0; - struct _xmlAttr *attr; - for (attr = ptr->properties; attr; attr = attr->next) - { - attr_content(attr, "name", &name_str); - attr_content(attr, "xpath", &xpath_str); - attr_content(attr, "type", &type_str); - } - if (name_str) - { - const char *prev_type = recWord->index_type; /* save default type */ - - if (type_str && *type_str) - recWord->index_type = (const char *) type_str; /* type was given */ - recWord->index_name = name_str; - index_cdata(tinfo, ctrl, ptr->children, recWord); - - recWord->index_type = prev_type; /* restore it again */ - } - } - } -} - -static void index_record(struct filter_info *tinfo,struct recExtractCtrl *ctrl, - xmlNodePtr ptr, RecWord *recWord) -{ - const char *type_str = "update"; - - if (ptr && ptr->type == XML_ELEMENT_NODE && ptr->ns && - !XML_STRCMP(ptr->ns->href, zebra_xslt_ns) - && !XML_STRCMP(ptr->name, "record")) - { - const char *id_str = 0; - const char *rank_str = 0; - struct _xmlAttr *attr; - for (attr = ptr->properties; attr; attr = attr->next) - { - attr_content(attr, "type", &type_str); - attr_content(attr, "id", &id_str); - attr_content(attr, "rank", &rank_str); - } - if (id_str) - sscanf(id_str, "%255s", ctrl->match_criteria); - - if (rank_str) - ctrl->staticrank = atozint(rank_str); - ptr = ptr->children; - } - - if (!strcmp("update", type_str)) - index_node(tinfo, ctrl, ptr, recWord); - else if (!strcmp("delete", type_str)) - yaz_log(YLOG_WARN, "alvis filter delete: to be implemented"); - else - yaz_log(YLOG_WARN, "alvis filter: unknown record type '%s'", - type_str); -} - -static int extract_doc(struct filter_info *tinfo, struct recExtractCtrl *p, - xmlDocPtr doc) -{ - RecWord recWord; - const char *params[10]; - xmlChar *buf_out; - int len_out; - - struct filter_schema *schema = lookup_schema(tinfo, zebra_xslt_ns); - - params[0] = 0; - set_param_str(params, "schema", zebra_xslt_ns, tinfo->odr); - - (*p->init)(p, &recWord); - - if (schema && schema->stylesheet_xsp) - { - xmlNodePtr root_ptr; - xmlDocPtr resDoc = - xsltApplyStylesheet(schema->stylesheet_xsp, - doc, params); - if (p->flagShowRecords) - { - xmlDocDumpMemory(resDoc, &buf_out, &len_out); - fwrite(buf_out, len_out, 1, stdout); - xmlFree(buf_out); - } - root_ptr = xmlDocGetRootElement(resDoc); - if (root_ptr) - index_record(tinfo, p, root_ptr, &recWord); - else - { - yaz_log(YLOG_WARN, "No root for index XML record." - " split_level=%d stylesheet=%s", - tinfo->split_level, schema->stylesheet); - } - xmlFreeDoc(resDoc); - } - xmlDocDumpMemory(doc, &buf_out, &len_out); - if (p->flagShowRecords) - fwrite(buf_out, len_out, 1, stdout); - if (p->setStoreData) - (*p->setStoreData)(p, buf_out, len_out); - xmlFree(buf_out); - - xmlFreeDoc(doc); - return RECCTRL_EXTRACT_OK; -} - -static int extract_split(struct filter_info *tinfo, struct recExtractCtrl *p) -{ - int ret; - - if (p->first_record) - { - if (tinfo->reader) - xmlFreeTextReader(tinfo->reader); - tinfo->reader = xmlReaderForIO(ioread_ex, ioclose_ex, - p /* I/O handler */, - 0 /* URL */, - 0 /* encoding */, - XML_PARSE_XINCLUDE - | XML_PARSE_NOENT - | XML_PARSE_NONET); - } - if (!tinfo->reader) - return RECCTRL_EXTRACT_ERROR_GENERIC; - - ret = xmlTextReaderRead(tinfo->reader); - while (ret == 1) - { - int type = xmlTextReaderNodeType(tinfo->reader); - int depth = xmlTextReaderDepth(tinfo->reader); - if (type == XML_READER_TYPE_ELEMENT && tinfo->split_level == depth) - { - xmlNodePtr ptr = xmlTextReaderExpand(tinfo->reader); - if (ptr) - { - xmlNodePtr ptr2 = xmlCopyNode(ptr, 1); - xmlDocPtr doc = xmlNewDoc((const xmlChar*) "1.0"); - - xmlDocSetRootElement(doc, ptr2); - - return extract_doc(tinfo, p, doc); - } - else - { - xmlFreeTextReader(tinfo->reader); - tinfo->reader = 0; - return RECCTRL_EXTRACT_ERROR_GENERIC; - } - } - ret = xmlTextReaderRead(tinfo->reader); - } - xmlFreeTextReader(tinfo->reader); - tinfo->reader = 0; - return RECCTRL_EXTRACT_EOF; -} - -static int extract_full(struct filter_info *tinfo, struct recExtractCtrl *p) -{ - if (p->first_record) /* only one record per stream */ - { - xmlDocPtr doc = xmlReadIO(ioread_ex, ioclose_ex, p /* I/O handler */, - 0 /* URL */, - 0 /* encoding */, - XML_PARSE_XINCLUDE - | XML_PARSE_NOENT - | XML_PARSE_NONET); - if (!doc) - return RECCTRL_EXTRACT_ERROR_GENERIC; - /* else { - xmlNodePtr root = xmlDocGetRootElement(doc); - if (!root) - return RECCTRL_EXTRACT_ERROR_GENERIC; - } */ - - return extract_doc(tinfo, p, doc); - } - else - return RECCTRL_EXTRACT_EOF; -} - -static int filter_extract(void *clientData, struct recExtractCtrl *p) -{ - struct filter_info *tinfo = clientData; - - odr_reset(tinfo->odr); - if (tinfo->split_level == 0 || p->setStoreData == 0) - return extract_full(tinfo, p); - else - return extract_split(tinfo, p); -} - -static int ioread_ret(void *context, char *buffer, int len) -{ - struct recRetrieveCtrl *p = context; - return p->stream->readf(p->stream, buffer, len); -} - -static int ioclose_ret(void *context) -{ - return 0; -} - -static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p) -{ - /* const char *esn = zebra_xslt_ns; */ - const char *esn = 0; - const char *params[32]; - struct filter_info *tinfo = clientData; - xmlDocPtr resDoc; - xmlDocPtr doc; - struct filter_schema *schema; - - if (p->comp) - { - if (p->comp->which == Z_RecordComp_simple - && p->comp->u.simple->which == Z_ElementSetNames_generic) - { - esn = p->comp->u.simple->u.generic; - } - else if (p->comp->which == Z_RecordComp_complex - && p->comp->u.complex->generic->elementSpec - && p->comp->u.complex->generic->elementSpec->which == - Z_ElementSpec_elementSetName) - { - esn = p->comp->u.complex->generic->elementSpec->u.elementSetName; - } - } - schema = lookup_schema(tinfo, esn); - if (!schema) - { - p->diagnostic = - YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_; - return 0; - } - - params[0] = 0; - set_param_int(params, "id", p->localno, p->odr); - if (p->fname) - set_param_str(params, "filename", p->fname, p->odr); - if (p->staticrank >= 0) - set_param_int(params, "rank", p->staticrank, p->odr); - - if (esn) - set_param_str(params, "schema", esn, p->odr); - else - if (schema->name) - set_param_str(params, "schema", schema->name, p->odr); - else if (schema->identifier) - set_param_str(params, "schema", schema->identifier, p->odr); - else - set_param_str(params, "schema", "", p->odr); - - if (p->score >= 0) - set_param_int(params, "score", p->score, p->odr); - set_param_int(params, "size", p->recordSize, p->odr); - - doc = xmlReadIO(ioread_ret, ioclose_ret, p /* I/O handler */, - 0 /* URL */, - 0 /* encoding */, - XML_PARSE_XINCLUDE | XML_PARSE_NOENT | XML_PARSE_NONET); - if (!doc) - { - p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS; - return 0; - } - - if (!schema->stylesheet_xsp) - resDoc = doc; - else - { - resDoc = xsltApplyStylesheet(schema->stylesheet_xsp, - doc, params); - xmlFreeDoc(doc); - } - if (!resDoc) - { - p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS; - } - else if (!p->input_format - || !oid_oidcmp(p->input_format, yaz_oid_recsyn_xml)) - { - xmlChar *buf_out; - int len_out; - - if (schema->stylesheet_xsp) - xsltSaveResultToString(&buf_out, &len_out, resDoc, - schema->stylesheet_xsp); - else - xmlDocDumpMemory(resDoc, &buf_out, &len_out); - - p->output_format = yaz_oid_recsyn_xml; - p->rec_len = len_out; - p->rec_buf = odr_malloc(p->odr, p->rec_len); - memcpy(p->rec_buf, buf_out, p->rec_len); - xmlFree(buf_out); - } - else if (!oid_oidcmp(p->output_format, yaz_oid_recsyn_sutrs)) - { - xmlChar *buf_out; - int len_out; - - if (schema->stylesheet_xsp) - xsltSaveResultToString(&buf_out, &len_out, resDoc, - schema->stylesheet_xsp); - else - xmlDocDumpMemory(resDoc, &buf_out, &len_out); - - p->output_format = yaz_oid_recsyn_sutrs; - p->rec_len = len_out; - p->rec_buf = odr_malloc(p->odr, p->rec_len); - memcpy(p->rec_buf, buf_out, p->rec_len); - - xmlFree(buf_out); - } - else - { - p->diagnostic = YAZ_BIB1_RECORD_SYNTAX_UNSUPP; - } - xmlFreeDoc(resDoc); - return 0; -} - -static struct recType filter_type = { - 0, - "alvis", - filter_init, - filter_config, - filter_destroy, - filter_extract, - filter_retrieve -}; - -RecType -#ifdef IDZEBRA_STATIC_ALVIS -idzebra_filter_alvis -#else -idzebra_filter -#endif - -[] = { - &filter_type, - 0, -}; -/* - * Local variables: - * c-basic-offset: 4 - * indent-tabs-mode: nil - * End: - * vim: shiftwidth=4 tabstop=8 expandtab - */ - diff --git a/index/marcread.c b/index/marcread.c deleted file mode 100644 index 52b59e8..0000000 --- a/index/marcread.c +++ /dev/null @@ -1,914 +0,0 @@ -/* This file is part of the Zebra server. - Copyright (C) 1995-2008 Index Data - -Zebra is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free -Software Foundation; either version 2, or (at your option) any later -version. - -Zebra is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*/ - -#include -#include -#include - -#include -#include -#include -#include "marcomp.h" -#include "inline.h" - -#define MARC_DEBUG 0 -#define MARCOMP_DEBUG 0 - -struct marc_info { - char type[256]; -}; - -static data1_node *grs_read_iso2709 (struct grs_read_info *p, int marc_xml) -{ - struct marc_info *mi = (struct marc_info*) p->clientData; - char buf[100000]; - int entry_p; - int record_length; - int indicator_length; - int identifier_length; - int base_address; - int end_of_directory; - int length_data_entry; - int length_starting; - int length_implementation; - int read_bytes; -#if MARC_DEBUG - FILE *outf = stdout; -#endif - data1_node *res_root, *res_top; - char *absynName; - data1_marctab *marctab; - - if (p->stream->readf(p->stream, buf, 5) != 5) - return NULL; - while (*buf < '0' || *buf > '9') - { - int i; - - yaz_log(YLOG_WARN, "MARC: Skipping bad byte %d (0x%02X)", - *buf & 0xff, *buf & 0xff); - for (i = 0; i<4; i++) - buf[i] = buf[i+1]; - - if (p->stream->readf(p->stream, buf+4, 1) != 1) - return NULL; - } - record_length = atoi_n (buf, 5); - if (record_length < 25) - { - yaz_log (YLOG_WARN, "MARC record length < 25, is %d", record_length); - return NULL; - } - - read_bytes = p->stream->readf(p->stream, buf+5, record_length-5); - if (read_bytes < record_length-5) - { - yaz_log (YLOG_WARN, "Couldn't read whole MARC record"); - return NULL; - } - /* skip until we meet a record separator */ - while (buf[record_length-1] != ISO2709_RS) - { - if (record_length > sizeof(buf)-2) - break; - read_bytes = p->stream->readf(p->stream, buf+record_length, 1); - if (read_bytes != 1) - break; - record_length++; - } - /* read one byte ahead to see if there is more ... */ - read_bytes = p->stream->readf(p->stream, buf+record_length, 1); - if (read_bytes == 1) - { - off_t cur_offset = p->stream->tellf(p->stream); - if (p->stream->endf) - { - off_t end_offset = cur_offset - 1; - p->stream->endf(p->stream, &end_offset); - } - } - - absynName = mi->type; - res_root = data1_mk_root (p->dh, p->mem, absynName); - if (!res_root) - { - yaz_log (YLOG_WARN, "cannot read MARC without an abstract syntax"); - return 0; - } - if (marc_xml) - { - data1_node *lead; - const char *attr[] = { "xmlns", "http://www.loc.gov/MARC21/slim", 0}; - - res_top = data1_mk_tag (p->dh, p->mem, "record", attr, res_root); - - lead = data1_mk_tag(p->dh, p->mem, "leader", 0, res_top); - data1_mk_text_n(p->dh, p->mem, buf, 24, lead); - } - else - res_top = data1_mk_tag (p->dh, p->mem, absynName, 0, res_root); - - if ((marctab = data1_absyn_getmarctab(p->dh, res_root))) - { - memcpy(marctab->leader, buf, 24); - memcpy(marctab->implementation_codes, buf+6, 4); - marctab->implementation_codes[4] = '\0'; - memcpy(marctab->user_systems, buf+17, 3); - marctab->user_systems[3] = '\0'; - } - - if (marctab && marctab->force_indicator_length >= 0) - indicator_length = marctab->force_indicator_length; - else - indicator_length = atoi_n (buf+10, 1); - if (marctab && marctab->force_identifier_length >= 0) - identifier_length = marctab->force_identifier_length; - else - identifier_length = atoi_n (buf+11, 1); - base_address = atoi_n (buf+12, 5); - - length_data_entry = atoi_n (buf+20, 1); - length_starting = atoi_n (buf+21, 1); - length_implementation = atoi_n (buf+22, 1); - - for (entry_p = 24; buf[entry_p] != ISO2709_FS; ) - { - int l = 3 + length_data_entry + length_starting; - if (entry_p + l >= record_length) - { - yaz_log(YLOG_WARN, "MARC: Directory offset %d: end of record.", - entry_p); - return 0; - } - /* check for digits in length info */ - while (--l >= 3) - if (!isdigit(*(const unsigned char *) (buf + entry_p+l))) - break; - if (l >= 3) - { - /* not all digits, so stop directory scan */ - yaz_log(YLOG_LOG, "MARC: Bad directory"); - break; - } - entry_p += 3 + length_data_entry + length_starting; - } - end_of_directory = entry_p; - if (base_address != entry_p+1) - { - yaz_log(YLOG_WARN, "MARC: Base address does not follow directory"); - } - for (entry_p = 24; entry_p != end_of_directory; ) - { - int data_length; - int data_offset; - int end_offset; - int i, i0; - char tag[4]; - data1_node *res; - data1_node *parent = res_top; - - memcpy (tag, buf+entry_p, 3); - entry_p += 3; - tag[3] = '\0'; - - if (marc_xml) - res = parent; - else - res = data1_mk_tag_n (p->dh, p->mem, tag, 3, 0 /* attr */, parent); - -#if MARC_DEBUG - fprintf (outf, "%s ", tag); -#endif - data_length = atoi_n (buf+entry_p, length_data_entry); - entry_p += length_data_entry; - data_offset = atoi_n (buf+entry_p, length_starting); - entry_p += length_starting; - i = data_offset + base_address; - end_offset = i+data_length-1; - - if (data_length <= 0 || data_offset < 0 || end_offset >= record_length) - { - yaz_log(YLOG_WARN, "MARC: Bad offsets in data. Skipping rest"); - break; - } - - if (memcmp (tag, "00", 2) && indicator_length) - { - /* generate indicator node */ - if (marc_xml) - { - const char *attr[10]; - int j; - - attr[0] = "tag"; - attr[1] = tag; - attr[2] = 0; - - res = data1_mk_tag(p->dh, p->mem, "datafield", attr, res); - - for (j = 0; jdh, p->mem, res, attr); - } - } - else - { -#if MARC_DEBUG - int j; -#endif - res = data1_mk_tag_n (p->dh, p->mem, - buf+i, indicator_length, 0 /* attr */, res); -#if MARC_DEBUG - for (j = 0; jdh, p->mem, "controlfield", attr, res); - } - } - parent = res; - /* traverse sub fields */ - i0 = i; - while (buf[i] != ISO2709_RS && buf[i] != ISO2709_FS && i < end_offset) - { - if (memcmp (tag, "00", 2) && identifier_length) - { - data1_node *res; - if (marc_xml) - { - int j; - const char *attr[3]; - char code[10]; - - for (j = 1; jdh, p->mem, "subfield", - attr, parent); - } - else - { - res = data1_mk_tag_n (p->dh, p->mem, - buf+i+1, identifier_length-1, - 0 /* attr */, parent); - } -#if MARC_DEBUG - fprintf (outf, " $"); - for (j = 1; jdh, p->mem, buf + i0, i - i0, res); - i0 = i; - } - else - { -#if MARC_DEBUG - fprintf (outf, "%c", buf[i]); -#endif - i++; - } - } - if (i > i0) - { - data1_mk_text_n (p->dh, p->mem, buf + i0, i - i0, parent); - } -#if MARC_DEBUG - fprintf (outf, "\n"); - if (i < end_offset) - fprintf (outf, "-- separator but not at end of field\n"); - if (buf[i] != ISO2709_RS && buf[i] != ISO2709_FS) - fprintf (outf, "-- no separator at end of field\n"); -#endif - } - return res_root; -} - -/* - * Locate some data under this node. This routine should handle variants - * prettily. - */ -static char *get_data(data1_node *n, int *len) -{ - char *r; - - while (n) - { - if (n->which == DATA1N_data) - { - int i; - *len = n->u.data.len; - - for (i = 0; i<*len; i++) - if (!d1_isspace(n->u.data.data[i])) - break; - while (*len && d1_isspace(n->u.data.data[*len - 1])) - (*len)--; - *len = *len - i; - if (*len > 0) - return n->u.data.data + i; - } - if (n->which == DATA1N_tag) - n = n->child; - else if (n->which == DATA1N_data) - n = n->next; - else - break; - } - r = ""; - *len = strlen(r); - return r; -} - -static data1_node *lookup_subfield(data1_node *node, const char *name) -{ - data1_node *p; - - for (p=node; p; p=p->next) - { - if (!yaz_matchstr(p->u.tag.tag, name)) - return p; - } - return 0; -} - -static inline_subfield *lookup_inline_subfield(inline_subfield *pisf, - const char *name) -{ - inline_subfield *p; - - for (p=pisf; p; p=p->next) - { - if (!yaz_matchstr(p->name, name)) - return p; - } - return 0; -} - -static inline_subfield *cat_inline_subfield(mc_subfield *psf, WRBUF buf, - inline_subfield *pisf) -{ - mc_subfield *p; - - for (p = psf; p && pisf; p = p->next) - { - if (p->which == MC_SF) - { - inline_subfield *found = lookup_inline_subfield(pisf, p->name); - - if (found) - { - if (strcmp(p->prefix, "_")) - { - wrbuf_puts(buf, " "); - wrbuf_puts(buf, p->prefix); - } - if (p->interval.start == -1) - { - wrbuf_puts(buf, found->data); - } - else - { - wrbuf_write(buf, found->data+p->interval.start, - p->interval.end-p->interval.start); - wrbuf_puts(buf, ""); - } - if (strcmp(p->suffix, "_")) - { - wrbuf_puts(buf, p->suffix); - wrbuf_puts(buf, " "); - } -#if MARCOMP_DEBUG - yaz_log(YLOG_LOG, "cat_inline_subfield(): add subfield $%s", found->name); -#endif - pisf = found->next; - } - } - else if (p->which == MC_SFVARIANT) - { - inline_subfield *next; - - do { - next = cat_inline_subfield(p->u.child, buf, pisf); - if (next == pisf) - break; - pisf = next; - } while (pisf); - } - else if (p->which == MC_SFGROUP) - { - mc_subfield *pp; - int found; - - for (pp = p->u.child, found = 0; pp; pp = pp->next) - { - if (!yaz_matchstr(pisf->name, p->name)) - { - found = 1; - break; - } - } - if (found) - { - wrbuf_puts(buf, " ("); - pisf = cat_inline_subfield(p->u.child, buf, pisf); - wrbuf_puts(buf, ") "); - } - } - } - return pisf; -} - -static void cat_inline_field(mc_field *pf, WRBUF buf, data1_node *subfield) -{ - if (!pf || !subfield) - return; - - for (;subfield;) - { - int len; - inline_field *pif=NULL; - data1_node *psubf; - - if (yaz_matchstr(subfield->u.tag.tag, "1")) - { - subfield = subfield->next; - continue; - } - - psubf = subfield; - pif = inline_mk_field(); - do - { - int i; - if ((i=inline_parse(pif, psubf->u.tag.tag, get_data(psubf, &len)))<0) - { - yaz_log(YLOG_WARN, "inline subfield ($%s): parse error", - psubf->u.tag.tag); - inline_destroy_field(pif); - return; - } - psubf = psubf->next; - } while (psubf && yaz_matchstr(psubf->u.tag.tag, "1")); - - subfield = psubf; - - if (pif && !yaz_matchstr(pif->name, pf->name)) - { - if (!pf->list && pif->list) - { - wrbuf_puts(buf, pif->list->data); - } - else - { - int ind1, ind2; - - /* - check indicators - */ - - ind1 = (pif->ind1[0] == ' ') ? '_':pif->ind1[0]; - ind2 = (pif->ind2[0] == ' ') ? '_':pif->ind2[0]; - - if (((pf->ind1[0] == '.') || (ind1 == pf->ind1[0])) && - ((pf->ind2[0] == '.') || (ind2 == pf->ind2[0]))) - { - cat_inline_subfield(pf->list, buf, pif->list); - - /* - add separator for inline fields - */ - if (wrbuf_len(buf)) - { - wrbuf_puts(buf, "\n"); - } - } - else - { - yaz_log(YLOG_WARN, "In-line field %s missed -- indicators do not match", pif->name); - } - } - } - inline_destroy_field(pif); - } -#if MARCOMP_DEBUG - yaz_log(YLOG_LOG, "cat_inline_field(): got buffer {%s}", buf); -#endif -} - -static data1_node *cat_subfield(mc_subfield *psf, WRBUF buf, - data1_node *subfield) -{ - mc_subfield *p; - - for (p = psf; p && subfield; p = p->next) - { - if (p->which == MC_SF) - { - data1_node *found = lookup_subfield(subfield, p->name); - - if (found) - { - int len; - - if (strcmp(p->prefix, "_")) - { - wrbuf_puts(buf, " "); - wrbuf_puts(buf, p->prefix); - } - - if (p->u.in_line) - { - cat_inline_field(p->u.in_line, buf, found); - } - else if (p->interval.start == -1) - { - wrbuf_puts(buf, get_data(found, &len)); - } - else - { - wrbuf_write(buf, get_data(found, &len)+p->interval.start, - p->interval.end-p->interval.start); - wrbuf_puts(buf, ""); - } - if (strcmp(p->suffix, "_")) - { - wrbuf_puts(buf, p->suffix); - wrbuf_puts(buf, " "); - } -#if MARCOMP_DEBUG - yaz_log(YLOG_LOG, "cat_subfield(): add subfield $%s", found->u.tag.tag); -#endif - subfield = found->next; - } - } - else if (p->which == MC_SFVARIANT) - { - data1_node *next; - do { - next = cat_subfield(p->u.child, buf, subfield); - if (next == subfield) - break; - subfield = next; - } while (subfield); - } - else if (p->which == MC_SFGROUP) - { - mc_subfield *pp; - int found; - - for (pp = p->u.child, found = 0; pp; pp = pp->next) - { - if (!yaz_matchstr(subfield->u.tag.tag, pp->name)) - { - found = 1; - break; - } - } - if (found) - { - wrbuf_puts(buf, " ("); - subfield = cat_subfield(p->u.child, buf, subfield); - wrbuf_puts(buf, ") "); - } - } - } - return subfield; -} - -static data1_node *cat_field(struct grs_read_info *p, mc_field *pf, - WRBUF buf, data1_node *field) -{ - data1_node *subfield; - int ind1, ind2; - - if (!pf || !field) - return 0; - - - if (yaz_matchstr(field->u.tag.tag, pf->name)) - return field->next; - - subfield = field->child; - - if (!subfield) - return field->next; - - /* - check subfield without indicators - */ - - if (!pf->list && subfield->which == DATA1N_data) - { - int len; - - if (pf->interval.start == -1) - { - wrbuf_puts(buf, get_data(field, &len)); - } - else - { - wrbuf_write(buf, get_data(field, &len)+pf->interval.start, - pf->interval.end-pf->interval.start); - wrbuf_puts(buf, ""); - } -#if MARCOMP_DEBUG - yaz_log(YLOG_LOG, "cat_field(): got buffer {%s}", buf); -#endif - return field->next; - } - - /* - check indicators - */ - - ind1 = (subfield->u.tag.tag[0] == ' ') ? '_':subfield->u.tag.tag[0]; - ind2 = (subfield->u.tag.tag[1] == ' ') ? '_':subfield->u.tag.tag[1]; - - if (!( - ((pf->ind1[0] == '.') || (ind1 == pf->ind1[0])) && - ((pf->ind2[0] == '.') || (ind2 == pf->ind2[0])) - )) - { -#if MARCOMP_DEBUG - yaz_log(YLOG_WARN, "Field %s missed -- does not match indicators", field->u.tag.tag); -#endif - return field->next; - } - - subfield = subfield->child; - - if (!subfield) - return field->next; - - cat_subfield(pf->list, buf, subfield); - -#if MARCOMP_DEBUG - yaz_log(YLOG_LOG, "cat_field(): got buffer {%s}", buf); -#endif - - return field->next; -} - -static int is_empty(char *s) -{ - char *p = s; - - for (p = s; *p; p++) - { - if (!isspace(*(unsigned char *)p)) - return 0; - } - return 1; -} - -static void parse_data1_tree(struct grs_read_info *p, const char *mc_stmnt, - data1_node *root) -{ - data1_marctab *marctab = data1_absyn_getmarctab(p->dh, root); - data1_node *top = root->child; - data1_node *field; - mc_context *c; - mc_field *pf; - WRBUF buf; - - c = mc_mk_context(mc_stmnt+3); - - if (!c) - return; - - pf = mc_getfield(c); - - if (!pf) - { - mc_destroy_context(c); - return; - } - buf = wrbuf_alloc(); -#if MARCOMP_DEBUG - yaz_log(YLOG_LOG, "parse_data1_tree(): statement -{%s}", mc_stmnt); -#endif - if (!yaz_matchstr(pf->name, "ldr")) - { - data1_node *new; -#if MARCOMP_DEBUG - yaz_log(YLOG_LOG,"parse_data1_tree(): try LEADER from {%d} to {%d} positions", - pf->interval.start, pf->interval.end); -#endif - if (marctab) - { - new = data1_mk_tag_n(p->dh, p->mem, mc_stmnt, strlen(mc_stmnt), 0, top); - data1_mk_text_n(p->dh, p->mem, marctab->leader+pf->interval.start, - pf->interval.end-pf->interval.start+1, new); - } - } - else - { - field=top->child; - - while(field) - { - if (!yaz_matchstr(field->u.tag.tag, pf->name)) - { - data1_node *new; - char *pb; -#if MARCOMP_DEBUG - yaz_log(YLOG_LOG, "parse_data1_tree(): try field {%s}", field->u.tag.tag); -#endif - wrbuf_rewind(buf); - wrbuf_puts(buf, ""); - - field = cat_field(p, pf, buf, field); - - wrbuf_cstr(buf); - pb = wrbuf_buf(buf); - for (pb = strtok(pb, "\n"); pb; pb = strtok(NULL, "\n")) - { - if (!is_empty(pb)) - { - new = data1_mk_tag_n(p->dh, p->mem, mc_stmnt, strlen(mc_stmnt), 0, top); - data1_mk_text_n(p->dh, p->mem, pb, strlen(pb), new); - } - } - } - else - { - field = field->next; - } - } - } - mc_destroy_field(pf); - mc_destroy_context(c); - wrbuf_destroy(buf); -} - -data1_node *grs_read_marcxml(struct grs_read_info *p) -{ - data1_node *root = grs_read_iso2709(p, 1); - data1_element *e; - - if (!root) - return 0; - - for (e = data1_absyn_getelements(p->dh, root); e; e=e->next) - { - data1_tag *tag = e->tag; - - if (tag && tag->which == DATA1T_string && - !yaz_matchstr(tag->value.string, "mc?")) - parse_data1_tree(p, tag->value.string, root); - } - return root; -} - -data1_node *grs_read_marc(struct grs_read_info *p) -{ - data1_node *root = grs_read_iso2709(p, 0); - data1_element *e; - - if (!root) - return 0; - - for (e = data1_absyn_getelements(p->dh, root); e; e=e->next) - { - data1_tag *tag = e->tag; - - if (tag && tag->which == DATA1T_string && - !yaz_matchstr(tag->value.string, "mc?")) - parse_data1_tree(p, tag->value.string, root); - } - return root; -} - -static void *init_marc(Res res, RecType rt) -{ - struct marc_info *p = xmalloc(sizeof(*p)); - strcpy(p->type, ""); - return p; -} - -static ZEBRA_RES config_marc(void *clientData, Res res, const char *args) -{ - struct marc_info *p = (struct marc_info*) clientData; - if (strlen(args) < sizeof(p->type)) - strcpy(p->type, args); - return ZEBRA_OK; -} - -static void destroy_marc(void *clientData) -{ - struct marc_info *p = (struct marc_info*) clientData; - xfree (p); -} - - -static int extract_marc(void *clientData, struct recExtractCtrl *ctrl) -{ - return zebra_grs_extract(clientData, ctrl, grs_read_marc); -} - -static int retrieve_marc(void *clientData, struct recRetrieveCtrl *ctrl) -{ - return zebra_grs_retrieve(clientData, ctrl, grs_read_marc); -} - -static struct recType marc_type = { - 0, - "grs.marc", - init_marc, - config_marc, - destroy_marc, - extract_marc, - retrieve_marc, -}; - -static int extract_marcxml(void *clientData, struct recExtractCtrl *ctrl) -{ - return zebra_grs_extract(clientData, ctrl, grs_read_marcxml); -} - -static int retrieve_marcxml(void *clientData, struct recRetrieveCtrl *ctrl) -{ - return zebra_grs_retrieve(clientData, ctrl, grs_read_marcxml); -} - -static struct recType marcxml_type = { - 0, - "grs.marcxml", - init_marc, - config_marc, - destroy_marc, - extract_marcxml, - retrieve_marcxml, -}; - -RecType -#ifdef IDZEBRA_STATIC_GRS_MARC -idzebra_filter_grs_marc -#else -idzebra_filter -#endif - -[] = { - &marc_type, - &marcxml_type, - 0, -}; - -/* - * Local variables: - * c-basic-offset: 4 - * indent-tabs-mode: nil - * End: - * vim: shiftwidth=4 tabstop=8 expandtab - */ - diff --git a/index/mod_alvis.c b/index/mod_alvis.c new file mode 100644 index 0000000..a752890 --- /dev/null +++ b/index/mod_alvis.c @@ -0,0 +1,729 @@ +/* This file is part of the Zebra server. + Copyright (C) 1995-2008 Index Data + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#if YAZ_HAVE_EXSLT +#include +#endif + +#include +#include + +struct filter_schema { + const char *name; + const char *identifier; + const char *stylesheet; + struct filter_schema *next; + const char *default_schema; + /* char default_schema; */ + xsltStylesheetPtr stylesheet_xsp; +}; + +struct filter_info { + xmlDocPtr doc; + char *fname; + char *full_name; + const char *profile_path; + int split_level; + const char *split_path; + ODR odr; + struct filter_schema *schemas; + xmlTextReaderPtr reader; +}; + +#define ZEBRA_SCHEMA_XSLT_NS "http://indexdata.dk/zebra/xslt/1" + +#define XML_STRCMP(a,b) strcmp((char*)a, b) +#define XML_STRLEN(a) strlen((char*)a) + +static const char *zebra_xslt_ns = ZEBRA_SCHEMA_XSLT_NS; + +static void set_param_str(const char **params, const char *name, + const char *value, ODR odr) +{ + char *quoted = odr_malloc(odr, 3 + strlen(value)); + sprintf(quoted, "'%s'", value); + while (*params) + params++; + params[0] = name; + params[1] = quoted; + params[2] = 0; +} + +static void set_param_int(const char **params, const char *name, + zint value, ODR odr) +{ + char *quoted = odr_malloc(odr, 30); /* 25 digits enough for 2^64 */ + while (*params) + params++; + sprintf(quoted, "'" ZINT_FORMAT "'", value); + params[0] = name; + params[1] = quoted; + params[2] = 0; +} + +#define ENABLE_INPUT_CALLBACK 0 + +#if ENABLE_INPUT_CALLBACK +static int zebra_xmlInputMatchCallback (char const *filename) +{ + yaz_log(YLOG_LOG, "match %s", filename); + return 0; +} + +static void * zebra_xmlInputOpenCallback (char const *filename) +{ + return 0; +} + +static int zebra_xmlInputReadCallback (void * context, char * buffer, int len) +{ + return 0; +} + +static int zebra_xmlInputCloseCallback (void * context) +{ + return 0; +} +#endif + +static void *filter_init(Res res, RecType recType) +{ + struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo)); + tinfo->reader = 0; + tinfo->fname = 0; + tinfo->full_name = 0; + tinfo->profile_path = 0; + tinfo->split_level = 0; + tinfo->split_path = 0; + tinfo->odr = odr_createmem(ODR_ENCODE); + tinfo->doc = 0; + tinfo->schemas = 0; + +#if YAZ_HAVE_EXSLT + exsltRegisterAll(); +#endif + +#if ENABLE_INPUT_CALLBACK + xmlRegisterDefaultInputCallbacks(); + xmlRegisterInputCallbacks(zebra_xmlInputMatchCallback, + zebra_xmlInputOpenCallback, + zebra_xmlInputReadCallback, + zebra_xmlInputCloseCallback); +#endif + return tinfo; +} + +static int attr_content(struct _xmlAttr *attr, const char *name, + const char **dst_content) +{ + if (!XML_STRCMP(attr->name, name) && attr->children + && attr->children->type == XML_TEXT_NODE) + { + *dst_content = (const char *)(attr->children->content); + return 1; + } + return 0; +} + +static void destroy_schemas(struct filter_info *tinfo) +{ + struct filter_schema *schema = tinfo->schemas; + while (schema) + { + struct filter_schema *schema_next = schema->next; + if (schema->stylesheet_xsp) + xsltFreeStylesheet(schema->stylesheet_xsp); + xfree(schema); + schema = schema_next; + } + tinfo->schemas = 0; + xfree(tinfo->fname); + if (tinfo->doc) + xmlFreeDoc(tinfo->doc); + tinfo->doc = 0; +} + +static ZEBRA_RES create_schemas(struct filter_info *tinfo, const char *fname) +{ + char tmp_full_name[1024]; + xmlNodePtr ptr; + tinfo->fname = xstrdup(fname); + + if (yaz_filepath_resolve(tinfo->fname, tinfo->profile_path, + NULL, tmp_full_name)) + tinfo->full_name = xstrdup(tmp_full_name); + else + tinfo->full_name = xstrdup(tinfo->fname); + + yaz_log(YLOG_LOG, "alvis filter: loading config file %s", tinfo->full_name); + + tinfo->doc = xmlParseFile(tinfo->full_name); + + if (!tinfo->doc) + { + yaz_log(YLOG_WARN, "alvis filter: could not parse config file %s", + tinfo->full_name); + + return ZEBRA_FAIL; + } + + ptr = xmlDocGetRootElement(tinfo->doc); + if (!ptr || ptr->type != XML_ELEMENT_NODE + || XML_STRCMP(ptr->name, "schemaInfo")) + { + yaz_log(YLOG_WARN, + "alvis filter: config file %s :" + " expected root element ", + tinfo->full_name); + return ZEBRA_FAIL; + } + + for (ptr = ptr->children; ptr; ptr = ptr->next) + { + if (ptr->type != XML_ELEMENT_NODE) + continue; + if (!XML_STRCMP(ptr->name, "schema")) + { + struct _xmlAttr *attr; + struct filter_schema *schema = xmalloc(sizeof(*schema)); + schema->name = 0; + schema->identifier = 0; + schema->stylesheet = 0; + schema->default_schema = 0; + schema->next = tinfo->schemas; + schema->stylesheet_xsp = 0; + tinfo->schemas = schema; + for (attr = ptr->properties; attr; attr = attr->next) + { + attr_content(attr, "identifier", &schema->identifier); + attr_content(attr, "name", &schema->name); + attr_content(attr, "stylesheet", &schema->stylesheet); + attr_content(attr, "default", &schema->default_schema); + } + /*yaz_log(YLOG_LOG, "XSLT add %s %s %s", + schema->name, schema->identifier, schema->stylesheet); */ + + /* find requested schema */ + + if (schema->stylesheet) + { + char tmp_xslt_full_name[1024]; + if (!yaz_filepath_resolve(schema->stylesheet, tinfo->profile_path, + NULL, tmp_xslt_full_name)) + { + yaz_log(YLOG_WARN, + "alvis filter: stylesheet %s not found in path %s", + schema->stylesheet, tinfo->profile_path); + return ZEBRA_FAIL; + } + schema->stylesheet_xsp + = xsltParseStylesheetFile((const xmlChar*) tmp_xslt_full_name); + if (!schema->stylesheet_xsp) + { + yaz_log(YLOG_WARN, + "alvis filter: could not parse xslt stylesheet %s", + tmp_xslt_full_name); + return ZEBRA_FAIL; + } + } + } + else if (!XML_STRCMP(ptr->name, "split")) + { + struct _xmlAttr *attr; + for (attr = ptr->properties; attr; attr = attr->next) + { + const char *split_level_str = 0; + attr_content(attr, "level", &split_level_str); + tinfo->split_level = + split_level_str ? atoi(split_level_str) : 0; + } + } + else + { + yaz_log(YLOG_WARN, "Bad element %s in %s", ptr->name, fname); + return ZEBRA_FAIL; + } + } + return ZEBRA_OK; +} + +static struct filter_schema *lookup_schema(struct filter_info *tinfo, + const char *est) +{ + struct filter_schema *schema; + + for (schema = tinfo->schemas; schema; schema = schema->next) + { + /* find requested schema */ + if (est) + { + if (schema->identifier && !strcmp(schema->identifier, est)) + return schema; + + if (schema->name && !strcmp(schema->name, est)) + return schema; + } + /* or return default schema if defined */ + else if (schema->default_schema) + return schema; + } + + /* return first schema if no default schema defined */ + if (tinfo->schemas) + return tinfo->schemas; + + return 0; +} + +static ZEBRA_RES filter_config(void *clientData, Res res, const char *args) +{ + struct filter_info *tinfo = clientData; + if (!args || !*args) + { + yaz_log(YLOG_WARN, "alvis filter: need config file"); + return ZEBRA_FAIL; + } + + if (tinfo->fname && !strcmp(args, tinfo->fname)) + return ZEBRA_OK; + + tinfo->profile_path = res_get(res, "profilePath"); + yaz_log(YLOG_LOG, "alvis filter: profilePath %s", tinfo->profile_path); + + destroy_schemas(tinfo); + return create_schemas(tinfo, args); +} + +static void filter_destroy(void *clientData) +{ + struct filter_info *tinfo = clientData; + destroy_schemas(tinfo); + xfree(tinfo->full_name); + if (tinfo->reader) + xmlFreeTextReader(tinfo->reader); + odr_destroy(tinfo->odr); + xfree(tinfo); +} + +static int ioread_ex(void *context, char *buffer, int len) +{ + struct recExtractCtrl *p = context; + return p->stream->readf(p->stream, buffer, len); +} + +static int ioclose_ex(void *context) +{ + return 0; +} + +static void index_cdata(struct filter_info *tinfo, struct recExtractCtrl *ctrl, + xmlNodePtr ptr, RecWord *recWord) +{ + for(; ptr; ptr = ptr->next) + { + index_cdata(tinfo, ctrl, ptr->children, recWord); + if (ptr->type != XML_TEXT_NODE) + continue; + recWord->term_buf = (const char *)ptr->content; + recWord->term_len = XML_STRLEN(ptr->content); + (*ctrl->tokenAdd)(recWord); + } +} + +static void index_node(struct filter_info *tinfo, struct recExtractCtrl *ctrl, + xmlNodePtr ptr, RecWord *recWord) +{ + for(; ptr; ptr = ptr->next) + { + index_node(tinfo, ctrl, ptr->children, recWord); + if (ptr->type != XML_ELEMENT_NODE || !ptr->ns || + XML_STRCMP(ptr->ns->href, zebra_xslt_ns)) + continue; + if (!XML_STRCMP(ptr->name, "index")) + { + const char *name_str = 0; + const char *type_str = 0; + const char *xpath_str = 0; + struct _xmlAttr *attr; + for (attr = ptr->properties; attr; attr = attr->next) + { + attr_content(attr, "name", &name_str); + attr_content(attr, "xpath", &xpath_str); + attr_content(attr, "type", &type_str); + } + if (name_str) + { + const char *prev_type = recWord->index_type; /* save default type */ + + if (type_str && *type_str) + recWord->index_type = (const char *) type_str; /* type was given */ + recWord->index_name = name_str; + index_cdata(tinfo, ctrl, ptr->children, recWord); + + recWord->index_type = prev_type; /* restore it again */ + } + } + } +} + +static void index_record(struct filter_info *tinfo,struct recExtractCtrl *ctrl, + xmlNodePtr ptr, RecWord *recWord) +{ + const char *type_str = "update"; + + if (ptr && ptr->type == XML_ELEMENT_NODE && ptr->ns && + !XML_STRCMP(ptr->ns->href, zebra_xslt_ns) + && !XML_STRCMP(ptr->name, "record")) + { + const char *id_str = 0; + const char *rank_str = 0; + struct _xmlAttr *attr; + for (attr = ptr->properties; attr; attr = attr->next) + { + attr_content(attr, "type", &type_str); + attr_content(attr, "id", &id_str); + attr_content(attr, "rank", &rank_str); + } + if (id_str) + sscanf(id_str, "%255s", ctrl->match_criteria); + + if (rank_str) + ctrl->staticrank = atozint(rank_str); + ptr = ptr->children; + } + + if (!strcmp("update", type_str)) + index_node(tinfo, ctrl, ptr, recWord); + else if (!strcmp("delete", type_str)) + yaz_log(YLOG_WARN, "alvis filter delete: to be implemented"); + else + yaz_log(YLOG_WARN, "alvis filter: unknown record type '%s'", + type_str); +} + +static int extract_doc(struct filter_info *tinfo, struct recExtractCtrl *p, + xmlDocPtr doc) +{ + RecWord recWord; + const char *params[10]; + xmlChar *buf_out; + int len_out; + + struct filter_schema *schema = lookup_schema(tinfo, zebra_xslt_ns); + + params[0] = 0; + set_param_str(params, "schema", zebra_xslt_ns, tinfo->odr); + + (*p->init)(p, &recWord); + + if (schema && schema->stylesheet_xsp) + { + xmlNodePtr root_ptr; + xmlDocPtr resDoc = + xsltApplyStylesheet(schema->stylesheet_xsp, + doc, params); + if (p->flagShowRecords) + { + xmlDocDumpMemory(resDoc, &buf_out, &len_out); + fwrite(buf_out, len_out, 1, stdout); + xmlFree(buf_out); + } + root_ptr = xmlDocGetRootElement(resDoc); + if (root_ptr) + index_record(tinfo, p, root_ptr, &recWord); + else + { + yaz_log(YLOG_WARN, "No root for index XML record." + " split_level=%d stylesheet=%s", + tinfo->split_level, schema->stylesheet); + } + xmlFreeDoc(resDoc); + } + xmlDocDumpMemory(doc, &buf_out, &len_out); + if (p->flagShowRecords) + fwrite(buf_out, len_out, 1, stdout); + if (p->setStoreData) + (*p->setStoreData)(p, buf_out, len_out); + xmlFree(buf_out); + + xmlFreeDoc(doc); + return RECCTRL_EXTRACT_OK; +} + +static int extract_split(struct filter_info *tinfo, struct recExtractCtrl *p) +{ + int ret; + + if (p->first_record) + { + if (tinfo->reader) + xmlFreeTextReader(tinfo->reader); + tinfo->reader = xmlReaderForIO(ioread_ex, ioclose_ex, + p /* I/O handler */, + 0 /* URL */, + 0 /* encoding */, + XML_PARSE_XINCLUDE + | XML_PARSE_NOENT + | XML_PARSE_NONET); + } + if (!tinfo->reader) + return RECCTRL_EXTRACT_ERROR_GENERIC; + + ret = xmlTextReaderRead(tinfo->reader); + while (ret == 1) + { + int type = xmlTextReaderNodeType(tinfo->reader); + int depth = xmlTextReaderDepth(tinfo->reader); + if (type == XML_READER_TYPE_ELEMENT && tinfo->split_level == depth) + { + xmlNodePtr ptr = xmlTextReaderExpand(tinfo->reader); + if (ptr) + { + xmlNodePtr ptr2 = xmlCopyNode(ptr, 1); + xmlDocPtr doc = xmlNewDoc((const xmlChar*) "1.0"); + + xmlDocSetRootElement(doc, ptr2); + + return extract_doc(tinfo, p, doc); + } + else + { + xmlFreeTextReader(tinfo->reader); + tinfo->reader = 0; + return RECCTRL_EXTRACT_ERROR_GENERIC; + } + } + ret = xmlTextReaderRead(tinfo->reader); + } + xmlFreeTextReader(tinfo->reader); + tinfo->reader = 0; + return RECCTRL_EXTRACT_EOF; +} + +static int extract_full(struct filter_info *tinfo, struct recExtractCtrl *p) +{ + if (p->first_record) /* only one record per stream */ + { + xmlDocPtr doc = xmlReadIO(ioread_ex, ioclose_ex, p /* I/O handler */, + 0 /* URL */, + 0 /* encoding */, + XML_PARSE_XINCLUDE + | XML_PARSE_NOENT + | XML_PARSE_NONET); + if (!doc) + return RECCTRL_EXTRACT_ERROR_GENERIC; + /* else { + xmlNodePtr root = xmlDocGetRootElement(doc); + if (!root) + return RECCTRL_EXTRACT_ERROR_GENERIC; + } */ + + return extract_doc(tinfo, p, doc); + } + else + return RECCTRL_EXTRACT_EOF; +} + +static int filter_extract(void *clientData, struct recExtractCtrl *p) +{ + struct filter_info *tinfo = clientData; + + odr_reset(tinfo->odr); + if (tinfo->split_level == 0 || p->setStoreData == 0) + return extract_full(tinfo, p); + else + return extract_split(tinfo, p); +} + +static int ioread_ret(void *context, char *buffer, int len) +{ + struct recRetrieveCtrl *p = context; + return p->stream->readf(p->stream, buffer, len); +} + +static int ioclose_ret(void *context) +{ + return 0; +} + +static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p) +{ + /* const char *esn = zebra_xslt_ns; */ + const char *esn = 0; + const char *params[32]; + struct filter_info *tinfo = clientData; + xmlDocPtr resDoc; + xmlDocPtr doc; + struct filter_schema *schema; + + if (p->comp) + { + if (p->comp->which == Z_RecordComp_simple + && p->comp->u.simple->which == Z_ElementSetNames_generic) + { + esn = p->comp->u.simple->u.generic; + } + else if (p->comp->which == Z_RecordComp_complex + && p->comp->u.complex->generic->elementSpec + && p->comp->u.complex->generic->elementSpec->which == + Z_ElementSpec_elementSetName) + { + esn = p->comp->u.complex->generic->elementSpec->u.elementSetName; + } + } + schema = lookup_schema(tinfo, esn); + if (!schema) + { + p->diagnostic = + YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_; + return 0; + } + + params[0] = 0; + set_param_int(params, "id", p->localno, p->odr); + if (p->fname) + set_param_str(params, "filename", p->fname, p->odr); + if (p->staticrank >= 0) + set_param_int(params, "rank", p->staticrank, p->odr); + + if (esn) + set_param_str(params, "schema", esn, p->odr); + else + if (schema->name) + set_param_str(params, "schema", schema->name, p->odr); + else if (schema->identifier) + set_param_str(params, "schema", schema->identifier, p->odr); + else + set_param_str(params, "schema", "", p->odr); + + if (p->score >= 0) + set_param_int(params, "score", p->score, p->odr); + set_param_int(params, "size", p->recordSize, p->odr); + + doc = xmlReadIO(ioread_ret, ioclose_ret, p /* I/O handler */, + 0 /* URL */, + 0 /* encoding */, + XML_PARSE_XINCLUDE | XML_PARSE_NOENT | XML_PARSE_NONET); + if (!doc) + { + p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS; + return 0; + } + + if (!schema->stylesheet_xsp) + resDoc = doc; + else + { + resDoc = xsltApplyStylesheet(schema->stylesheet_xsp, + doc, params); + xmlFreeDoc(doc); + } + if (!resDoc) + { + p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS; + } + else if (!p->input_format + || !oid_oidcmp(p->input_format, yaz_oid_recsyn_xml)) + { + xmlChar *buf_out; + int len_out; + + if (schema->stylesheet_xsp) + xsltSaveResultToString(&buf_out, &len_out, resDoc, + schema->stylesheet_xsp); + else + xmlDocDumpMemory(resDoc, &buf_out, &len_out); + + p->output_format = yaz_oid_recsyn_xml; + p->rec_len = len_out; + p->rec_buf = odr_malloc(p->odr, p->rec_len); + memcpy(p->rec_buf, buf_out, p->rec_len); + xmlFree(buf_out); + } + else if (!oid_oidcmp(p->output_format, yaz_oid_recsyn_sutrs)) + { + xmlChar *buf_out; + int len_out; + + if (schema->stylesheet_xsp) + xsltSaveResultToString(&buf_out, &len_out, resDoc, + schema->stylesheet_xsp); + else + xmlDocDumpMemory(resDoc, &buf_out, &len_out); + + p->output_format = yaz_oid_recsyn_sutrs; + p->rec_len = len_out; + p->rec_buf = odr_malloc(p->odr, p->rec_len); + memcpy(p->rec_buf, buf_out, p->rec_len); + + xmlFree(buf_out); + } + else + { + p->diagnostic = YAZ_BIB1_RECORD_SYNTAX_UNSUPP; + } + xmlFreeDoc(resDoc); + return 0; +} + +static struct recType filter_type = { + 0, + "alvis", + filter_init, + filter_config, + filter_destroy, + filter_extract, + filter_retrieve +}; + +RecType +#ifdef IDZEBRA_STATIC_ALVIS +idzebra_filter_alvis +#else +idzebra_filter +#endif + +[] = { + &filter_type, + 0, +}; +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/index/mod_grs_marc.c b/index/mod_grs_marc.c new file mode 100644 index 0000000..52b59e8 --- /dev/null +++ b/index/mod_grs_marc.c @@ -0,0 +1,914 @@ +/* This file is part of the Zebra server. + Copyright (C) 1995-2008 Index Data + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include +#include +#include + +#include +#include +#include +#include "marcomp.h" +#include "inline.h" + +#define MARC_DEBUG 0 +#define MARCOMP_DEBUG 0 + +struct marc_info { + char type[256]; +}; + +static data1_node *grs_read_iso2709 (struct grs_read_info *p, int marc_xml) +{ + struct marc_info *mi = (struct marc_info*) p->clientData; + char buf[100000]; + int entry_p; + int record_length; + int indicator_length; + int identifier_length; + int base_address; + int end_of_directory; + int length_data_entry; + int length_starting; + int length_implementation; + int read_bytes; +#if MARC_DEBUG + FILE *outf = stdout; +#endif + data1_node *res_root, *res_top; + char *absynName; + data1_marctab *marctab; + + if (p->stream->readf(p->stream, buf, 5) != 5) + return NULL; + while (*buf < '0' || *buf > '9') + { + int i; + + yaz_log(YLOG_WARN, "MARC: Skipping bad byte %d (0x%02X)", + *buf & 0xff, *buf & 0xff); + for (i = 0; i<4; i++) + buf[i] = buf[i+1]; + + if (p->stream->readf(p->stream, buf+4, 1) != 1) + return NULL; + } + record_length = atoi_n (buf, 5); + if (record_length < 25) + { + yaz_log (YLOG_WARN, "MARC record length < 25, is %d", record_length); + return NULL; + } + + read_bytes = p->stream->readf(p->stream, buf+5, record_length-5); + if (read_bytes < record_length-5) + { + yaz_log (YLOG_WARN, "Couldn't read whole MARC record"); + return NULL; + } + /* skip until we meet a record separator */ + while (buf[record_length-1] != ISO2709_RS) + { + if (record_length > sizeof(buf)-2) + break; + read_bytes = p->stream->readf(p->stream, buf+record_length, 1); + if (read_bytes != 1) + break; + record_length++; + } + /* read one byte ahead to see if there is more ... */ + read_bytes = p->stream->readf(p->stream, buf+record_length, 1); + if (read_bytes == 1) + { + off_t cur_offset = p->stream->tellf(p->stream); + if (p->stream->endf) + { + off_t end_offset = cur_offset - 1; + p->stream->endf(p->stream, &end_offset); + } + } + + absynName = mi->type; + res_root = data1_mk_root (p->dh, p->mem, absynName); + if (!res_root) + { + yaz_log (YLOG_WARN, "cannot read MARC without an abstract syntax"); + return 0; + } + if (marc_xml) + { + data1_node *lead; + const char *attr[] = { "xmlns", "http://www.loc.gov/MARC21/slim", 0}; + + res_top = data1_mk_tag (p->dh, p->mem, "record", attr, res_root); + + lead = data1_mk_tag(p->dh, p->mem, "leader", 0, res_top); + data1_mk_text_n(p->dh, p->mem, buf, 24, lead); + } + else + res_top = data1_mk_tag (p->dh, p->mem, absynName, 0, res_root); + + if ((marctab = data1_absyn_getmarctab(p->dh, res_root))) + { + memcpy(marctab->leader, buf, 24); + memcpy(marctab->implementation_codes, buf+6, 4); + marctab->implementation_codes[4] = '\0'; + memcpy(marctab->user_systems, buf+17, 3); + marctab->user_systems[3] = '\0'; + } + + if (marctab && marctab->force_indicator_length >= 0) + indicator_length = marctab->force_indicator_length; + else + indicator_length = atoi_n (buf+10, 1); + if (marctab && marctab->force_identifier_length >= 0) + identifier_length = marctab->force_identifier_length; + else + identifier_length = atoi_n (buf+11, 1); + base_address = atoi_n (buf+12, 5); + + length_data_entry = atoi_n (buf+20, 1); + length_starting = atoi_n (buf+21, 1); + length_implementation = atoi_n (buf+22, 1); + + for (entry_p = 24; buf[entry_p] != ISO2709_FS; ) + { + int l = 3 + length_data_entry + length_starting; + if (entry_p + l >= record_length) + { + yaz_log(YLOG_WARN, "MARC: Directory offset %d: end of record.", + entry_p); + return 0; + } + /* check for digits in length info */ + while (--l >= 3) + if (!isdigit(*(const unsigned char *) (buf + entry_p+l))) + break; + if (l >= 3) + { + /* not all digits, so stop directory scan */ + yaz_log(YLOG_LOG, "MARC: Bad directory"); + break; + } + entry_p += 3 + length_data_entry + length_starting; + } + end_of_directory = entry_p; + if (base_address != entry_p+1) + { + yaz_log(YLOG_WARN, "MARC: Base address does not follow directory"); + } + for (entry_p = 24; entry_p != end_of_directory; ) + { + int data_length; + int data_offset; + int end_offset; + int i, i0; + char tag[4]; + data1_node *res; + data1_node *parent = res_top; + + memcpy (tag, buf+entry_p, 3); + entry_p += 3; + tag[3] = '\0'; + + if (marc_xml) + res = parent; + else + res = data1_mk_tag_n (p->dh, p->mem, tag, 3, 0 /* attr */, parent); + +#if MARC_DEBUG + fprintf (outf, "%s ", tag); +#endif + data_length = atoi_n (buf+entry_p, length_data_entry); + entry_p += length_data_entry; + data_offset = atoi_n (buf+entry_p, length_starting); + entry_p += length_starting; + i = data_offset + base_address; + end_offset = i+data_length-1; + + if (data_length <= 0 || data_offset < 0 || end_offset >= record_length) + { + yaz_log(YLOG_WARN, "MARC: Bad offsets in data. Skipping rest"); + break; + } + + if (memcmp (tag, "00", 2) && indicator_length) + { + /* generate indicator node */ + if (marc_xml) + { + const char *attr[10]; + int j; + + attr[0] = "tag"; + attr[1] = tag; + attr[2] = 0; + + res = data1_mk_tag(p->dh, p->mem, "datafield", attr, res); + + for (j = 0; jdh, p->mem, res, attr); + } + } + else + { +#if MARC_DEBUG + int j; +#endif + res = data1_mk_tag_n (p->dh, p->mem, + buf+i, indicator_length, 0 /* attr */, res); +#if MARC_DEBUG + for (j = 0; jdh, p->mem, "controlfield", attr, res); + } + } + parent = res; + /* traverse sub fields */ + i0 = i; + while (buf[i] != ISO2709_RS && buf[i] != ISO2709_FS && i < end_offset) + { + if (memcmp (tag, "00", 2) && identifier_length) + { + data1_node *res; + if (marc_xml) + { + int j; + const char *attr[3]; + char code[10]; + + for (j = 1; jdh, p->mem, "subfield", + attr, parent); + } + else + { + res = data1_mk_tag_n (p->dh, p->mem, + buf+i+1, identifier_length-1, + 0 /* attr */, parent); + } +#if MARC_DEBUG + fprintf (outf, " $"); + for (j = 1; jdh, p->mem, buf + i0, i - i0, res); + i0 = i; + } + else + { +#if MARC_DEBUG + fprintf (outf, "%c", buf[i]); +#endif + i++; + } + } + if (i > i0) + { + data1_mk_text_n (p->dh, p->mem, buf + i0, i - i0, parent); + } +#if MARC_DEBUG + fprintf (outf, "\n"); + if (i < end_offset) + fprintf (outf, "-- separator but not at end of field\n"); + if (buf[i] != ISO2709_RS && buf[i] != ISO2709_FS) + fprintf (outf, "-- no separator at end of field\n"); +#endif + } + return res_root; +} + +/* + * Locate some data under this node. This routine should handle variants + * prettily. + */ +static char *get_data(data1_node *n, int *len) +{ + char *r; + + while (n) + { + if (n->which == DATA1N_data) + { + int i; + *len = n->u.data.len; + + for (i = 0; i<*len; i++) + if (!d1_isspace(n->u.data.data[i])) + break; + while (*len && d1_isspace(n->u.data.data[*len - 1])) + (*len)--; + *len = *len - i; + if (*len > 0) + return n->u.data.data + i; + } + if (n->which == DATA1N_tag) + n = n->child; + else if (n->which == DATA1N_data) + n = n->next; + else + break; + } + r = ""; + *len = strlen(r); + return r; +} + +static data1_node *lookup_subfield(data1_node *node, const char *name) +{ + data1_node *p; + + for (p=node; p; p=p->next) + { + if (!yaz_matchstr(p->u.tag.tag, name)) + return p; + } + return 0; +} + +static inline_subfield *lookup_inline_subfield(inline_subfield *pisf, + const char *name) +{ + inline_subfield *p; + + for (p=pisf; p; p=p->next) + { + if (!yaz_matchstr(p->name, name)) + return p; + } + return 0; +} + +static inline_subfield *cat_inline_subfield(mc_subfield *psf, WRBUF buf, + inline_subfield *pisf) +{ + mc_subfield *p; + + for (p = psf; p && pisf; p = p->next) + { + if (p->which == MC_SF) + { + inline_subfield *found = lookup_inline_subfield(pisf, p->name); + + if (found) + { + if (strcmp(p->prefix, "_")) + { + wrbuf_puts(buf, " "); + wrbuf_puts(buf, p->prefix); + } + if (p->interval.start == -1) + { + wrbuf_puts(buf, found->data); + } + else + { + wrbuf_write(buf, found->data+p->interval.start, + p->interval.end-p->interval.start); + wrbuf_puts(buf, ""); + } + if (strcmp(p->suffix, "_")) + { + wrbuf_puts(buf, p->suffix); + wrbuf_puts(buf, " "); + } +#if MARCOMP_DEBUG + yaz_log(YLOG_LOG, "cat_inline_subfield(): add subfield $%s", found->name); +#endif + pisf = found->next; + } + } + else if (p->which == MC_SFVARIANT) + { + inline_subfield *next; + + do { + next = cat_inline_subfield(p->u.child, buf, pisf); + if (next == pisf) + break; + pisf = next; + } while (pisf); + } + else if (p->which == MC_SFGROUP) + { + mc_subfield *pp; + int found; + + for (pp = p->u.child, found = 0; pp; pp = pp->next) + { + if (!yaz_matchstr(pisf->name, p->name)) + { + found = 1; + break; + } + } + if (found) + { + wrbuf_puts(buf, " ("); + pisf = cat_inline_subfield(p->u.child, buf, pisf); + wrbuf_puts(buf, ") "); + } + } + } + return pisf; +} + +static void cat_inline_field(mc_field *pf, WRBUF buf, data1_node *subfield) +{ + if (!pf || !subfield) + return; + + for (;subfield;) + { + int len; + inline_field *pif=NULL; + data1_node *psubf; + + if (yaz_matchstr(subfield->u.tag.tag, "1")) + { + subfield = subfield->next; + continue; + } + + psubf = subfield; + pif = inline_mk_field(); + do + { + int i; + if ((i=inline_parse(pif, psubf->u.tag.tag, get_data(psubf, &len)))<0) + { + yaz_log(YLOG_WARN, "inline subfield ($%s): parse error", + psubf->u.tag.tag); + inline_destroy_field(pif); + return; + } + psubf = psubf->next; + } while (psubf && yaz_matchstr(psubf->u.tag.tag, "1")); + + subfield = psubf; + + if (pif && !yaz_matchstr(pif->name, pf->name)) + { + if (!pf->list && pif->list) + { + wrbuf_puts(buf, pif->list->data); + } + else + { + int ind1, ind2; + + /* + check indicators + */ + + ind1 = (pif->ind1[0] == ' ') ? '_':pif->ind1[0]; + ind2 = (pif->ind2[0] == ' ') ? '_':pif->ind2[0]; + + if (((pf->ind1[0] == '.') || (ind1 == pf->ind1[0])) && + ((pf->ind2[0] == '.') || (ind2 == pf->ind2[0]))) + { + cat_inline_subfield(pf->list, buf, pif->list); + + /* + add separator for inline fields + */ + if (wrbuf_len(buf)) + { + wrbuf_puts(buf, "\n"); + } + } + else + { + yaz_log(YLOG_WARN, "In-line field %s missed -- indicators do not match", pif->name); + } + } + } + inline_destroy_field(pif); + } +#if MARCOMP_DEBUG + yaz_log(YLOG_LOG, "cat_inline_field(): got buffer {%s}", buf); +#endif +} + +static data1_node *cat_subfield(mc_subfield *psf, WRBUF buf, + data1_node *subfield) +{ + mc_subfield *p; + + for (p = psf; p && subfield; p = p->next) + { + if (p->which == MC_SF) + { + data1_node *found = lookup_subfield(subfield, p->name); + + if (found) + { + int len; + + if (strcmp(p->prefix, "_")) + { + wrbuf_puts(buf, " "); + wrbuf_puts(buf, p->prefix); + } + + if (p->u.in_line) + { + cat_inline_field(p->u.in_line, buf, found); + } + else if (p->interval.start == -1) + { + wrbuf_puts(buf, get_data(found, &len)); + } + else + { + wrbuf_write(buf, get_data(found, &len)+p->interval.start, + p->interval.end-p->interval.start); + wrbuf_puts(buf, ""); + } + if (strcmp(p->suffix, "_")) + { + wrbuf_puts(buf, p->suffix); + wrbuf_puts(buf, " "); + } +#if MARCOMP_DEBUG + yaz_log(YLOG_LOG, "cat_subfield(): add subfield $%s", found->u.tag.tag); +#endif + subfield = found->next; + } + } + else if (p->which == MC_SFVARIANT) + { + data1_node *next; + do { + next = cat_subfield(p->u.child, buf, subfield); + if (next == subfield) + break; + subfield = next; + } while (subfield); + } + else if (p->which == MC_SFGROUP) + { + mc_subfield *pp; + int found; + + for (pp = p->u.child, found = 0; pp; pp = pp->next) + { + if (!yaz_matchstr(subfield->u.tag.tag, pp->name)) + { + found = 1; + break; + } + } + if (found) + { + wrbuf_puts(buf, " ("); + subfield = cat_subfield(p->u.child, buf, subfield); + wrbuf_puts(buf, ") "); + } + } + } + return subfield; +} + +static data1_node *cat_field(struct grs_read_info *p, mc_field *pf, + WRBUF buf, data1_node *field) +{ + data1_node *subfield; + int ind1, ind2; + + if (!pf || !field) + return 0; + + + if (yaz_matchstr(field->u.tag.tag, pf->name)) + return field->next; + + subfield = field->child; + + if (!subfield) + return field->next; + + /* + check subfield without indicators + */ + + if (!pf->list && subfield->which == DATA1N_data) + { + int len; + + if (pf->interval.start == -1) + { + wrbuf_puts(buf, get_data(field, &len)); + } + else + { + wrbuf_write(buf, get_data(field, &len)+pf->interval.start, + pf->interval.end-pf->interval.start); + wrbuf_puts(buf, ""); + } +#if MARCOMP_DEBUG + yaz_log(YLOG_LOG, "cat_field(): got buffer {%s}", buf); +#endif + return field->next; + } + + /* + check indicators + */ + + ind1 = (subfield->u.tag.tag[0] == ' ') ? '_':subfield->u.tag.tag[0]; + ind2 = (subfield->u.tag.tag[1] == ' ') ? '_':subfield->u.tag.tag[1]; + + if (!( + ((pf->ind1[0] == '.') || (ind1 == pf->ind1[0])) && + ((pf->ind2[0] == '.') || (ind2 == pf->ind2[0])) + )) + { +#if MARCOMP_DEBUG + yaz_log(YLOG_WARN, "Field %s missed -- does not match indicators", field->u.tag.tag); +#endif + return field->next; + } + + subfield = subfield->child; + + if (!subfield) + return field->next; + + cat_subfield(pf->list, buf, subfield); + +#if MARCOMP_DEBUG + yaz_log(YLOG_LOG, "cat_field(): got buffer {%s}", buf); +#endif + + return field->next; +} + +static int is_empty(char *s) +{ + char *p = s; + + for (p = s; *p; p++) + { + if (!isspace(*(unsigned char *)p)) + return 0; + } + return 1; +} + +static void parse_data1_tree(struct grs_read_info *p, const char *mc_stmnt, + data1_node *root) +{ + data1_marctab *marctab = data1_absyn_getmarctab(p->dh, root); + data1_node *top = root->child; + data1_node *field; + mc_context *c; + mc_field *pf; + WRBUF buf; + + c = mc_mk_context(mc_stmnt+3); + + if (!c) + return; + + pf = mc_getfield(c); + + if (!pf) + { + mc_destroy_context(c); + return; + } + buf = wrbuf_alloc(); +#if MARCOMP_DEBUG + yaz_log(YLOG_LOG, "parse_data1_tree(): statement -{%s}", mc_stmnt); +#endif + if (!yaz_matchstr(pf->name, "ldr")) + { + data1_node *new; +#if MARCOMP_DEBUG + yaz_log(YLOG_LOG,"parse_data1_tree(): try LEADER from {%d} to {%d} positions", + pf->interval.start, pf->interval.end); +#endif + if (marctab) + { + new = data1_mk_tag_n(p->dh, p->mem, mc_stmnt, strlen(mc_stmnt), 0, top); + data1_mk_text_n(p->dh, p->mem, marctab->leader+pf->interval.start, + pf->interval.end-pf->interval.start+1, new); + } + } + else + { + field=top->child; + + while(field) + { + if (!yaz_matchstr(field->u.tag.tag, pf->name)) + { + data1_node *new; + char *pb; +#if MARCOMP_DEBUG + yaz_log(YLOG_LOG, "parse_data1_tree(): try field {%s}", field->u.tag.tag); +#endif + wrbuf_rewind(buf); + wrbuf_puts(buf, ""); + + field = cat_field(p, pf, buf, field); + + wrbuf_cstr(buf); + pb = wrbuf_buf(buf); + for (pb = strtok(pb, "\n"); pb; pb = strtok(NULL, "\n")) + { + if (!is_empty(pb)) + { + new = data1_mk_tag_n(p->dh, p->mem, mc_stmnt, strlen(mc_stmnt), 0, top); + data1_mk_text_n(p->dh, p->mem, pb, strlen(pb), new); + } + } + } + else + { + field = field->next; + } + } + } + mc_destroy_field(pf); + mc_destroy_context(c); + wrbuf_destroy(buf); +} + +data1_node *grs_read_marcxml(struct grs_read_info *p) +{ + data1_node *root = grs_read_iso2709(p, 1); + data1_element *e; + + if (!root) + return 0; + + for (e = data1_absyn_getelements(p->dh, root); e; e=e->next) + { + data1_tag *tag = e->tag; + + if (tag && tag->which == DATA1T_string && + !yaz_matchstr(tag->value.string, "mc?")) + parse_data1_tree(p, tag->value.string, root); + } + return root; +} + +data1_node *grs_read_marc(struct grs_read_info *p) +{ + data1_node *root = grs_read_iso2709(p, 0); + data1_element *e; + + if (!root) + return 0; + + for (e = data1_absyn_getelements(p->dh, root); e; e=e->next) + { + data1_tag *tag = e->tag; + + if (tag && tag->which == DATA1T_string && + !yaz_matchstr(tag->value.string, "mc?")) + parse_data1_tree(p, tag->value.string, root); + } + return root; +} + +static void *init_marc(Res res, RecType rt) +{ + struct marc_info *p = xmalloc(sizeof(*p)); + strcpy(p->type, ""); + return p; +} + +static ZEBRA_RES config_marc(void *clientData, Res res, const char *args) +{ + struct marc_info *p = (struct marc_info*) clientData; + if (strlen(args) < sizeof(p->type)) + strcpy(p->type, args); + return ZEBRA_OK; +} + +static void destroy_marc(void *clientData) +{ + struct marc_info *p = (struct marc_info*) clientData; + xfree (p); +} + + +static int extract_marc(void *clientData, struct recExtractCtrl *ctrl) +{ + return zebra_grs_extract(clientData, ctrl, grs_read_marc); +} + +static int retrieve_marc(void *clientData, struct recRetrieveCtrl *ctrl) +{ + return zebra_grs_retrieve(clientData, ctrl, grs_read_marc); +} + +static struct recType marc_type = { + 0, + "grs.marc", + init_marc, + config_marc, + destroy_marc, + extract_marc, + retrieve_marc, +}; + +static int extract_marcxml(void *clientData, struct recExtractCtrl *ctrl) +{ + return zebra_grs_extract(clientData, ctrl, grs_read_marcxml); +} + +static int retrieve_marcxml(void *clientData, struct recRetrieveCtrl *ctrl) +{ + return zebra_grs_retrieve(clientData, ctrl, grs_read_marcxml); +} + +static struct recType marcxml_type = { + 0, + "grs.marcxml", + init_marc, + config_marc, + destroy_marc, + extract_marcxml, + retrieve_marcxml, +}; + +RecType +#ifdef IDZEBRA_STATIC_GRS_MARC +idzebra_filter_grs_marc +#else +idzebra_filter +#endif + +[] = { + &marc_type, + &marcxml_type, + 0, +}; + +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/index/mod_grs_regx.c b/index/mod_grs_regx.c new file mode 100644 index 0000000..ecf8f95 --- /dev/null +++ b/index/mod_grs_regx.c @@ -0,0 +1,2037 @@ +/* This file is part of the Zebra server. + Copyright (C) 1995-2008 Index Data + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#if HAVE_TCL_H +#include + +#if MAJOR_VERSION >= 8 +#define HAVE_TCL_OBJECTS +#endif +#endif + +#define REGX_DEBUG 0 + +#define F_WIN_EOF 2000000000 +#define F_WIN_READ 1 + +#define REGX_EOF 0 +#define REGX_PATTERN 1 +#define REGX_BODY 2 +#define REGX_BEGIN 3 +#define REGX_END 4 +#define REGX_CODE 5 +#define REGX_CONTEXT 6 +#define REGX_INIT 7 + +struct regxCode { + char *str; +#if HAVE_TCL_OBJECTS + Tcl_Obj *tcl_obj; +#endif +}; + +struct lexRuleAction { + int which; + union { + struct { + struct DFA *dfa; /* REGX_PATTERN */ + int body; + } pattern; + struct regxCode *code; /* REGX_CODE */ + } u; + struct lexRuleAction *next; +}; + +struct lexRuleInfo { + int no; + struct lexRuleAction *actionList; +}; + +struct lexRule { + struct lexRuleInfo info; + struct lexRule *next; +}; + +struct lexContext { + char *name; + struct DFA *dfa; + struct lexRule *rules; + struct lexRuleInfo **fastRule; + int ruleNo; + int initFlag; + + struct lexRuleAction *beginActionList; + struct lexRuleAction *endActionList; + struct lexRuleAction *initActionList; + struct lexContext *next; +}; + +struct lexConcatBuf { + int max; + char *buf; +}; + +struct lexSpec { + char *name; + struct lexContext *context; + + struct lexContext **context_stack; + int context_stack_size; + int context_stack_top; + + int lineNo; + NMEM m; + data1_handle dh; +#if HAVE_TCL_H + Tcl_Interp *tcl_interp; +#endif + struct ZebraRecStream *stream; + off_t (*f_win_ef)(struct ZebraRecStream *s, off_t *); + + int f_win_start; /* first byte of buffer is this file offset */ + int f_win_end; /* last byte of buffer is this offset - 1 */ + int f_win_size; /* size of buffer */ + char *f_win_buf; /* buffer itself */ + int (*f_win_rf)(struct ZebraRecStream *, char *, size_t); + off_t (*f_win_sf)(struct ZebraRecStream *, off_t); + + struct lexConcatBuf *concatBuf; + int maxLevel; + data1_node **d1_stack; + int d1_level; + int stop_flag; + + int *arg_start; + int *arg_end; + int arg_no; + int ptr; +}; + +struct lexSpecs { + struct lexSpec *spec; + char type[256]; +}; + +static char *f_win_get (struct lexSpec *spec, off_t start_pos, off_t end_pos, + int *size) +{ + int i, r, off = start_pos - spec->f_win_start; + + if (off >= 0 && end_pos <= spec->f_win_end) + { + *size = end_pos - start_pos; + return spec->f_win_buf + off; + } + if (off < 0 || start_pos >= spec->f_win_end) + { + (*spec->f_win_sf)(spec->stream, start_pos); + spec->f_win_start = start_pos; + + if (!spec->f_win_buf) + spec->f_win_buf = (char *) xmalloc (spec->f_win_size); + *size = (*spec->f_win_rf)(spec->stream, spec->f_win_buf, + spec->f_win_size); + spec->f_win_end = spec->f_win_start + *size; + + if (*size > end_pos - start_pos) + *size = end_pos - start_pos; + return spec->f_win_buf; + } + for (i = 0; if_win_end - start_pos; i++) + spec->f_win_buf[i] = spec->f_win_buf[i + off]; + r = (*spec->f_win_rf)(spec->stream, + spec->f_win_buf + i, + spec->f_win_size - i); + spec->f_win_start = start_pos; + spec->f_win_end += r; + *size = i + r; + if (*size > end_pos - start_pos) + *size = end_pos - start_pos; + return spec->f_win_buf; +} + +static int f_win_advance (struct lexSpec *spec, int *pos) +{ + int size; + char *buf; + + if (*pos >= spec->f_win_start && *pos < spec->f_win_end) + return spec->f_win_buf[(*pos)++ - spec->f_win_start]; + if (*pos == F_WIN_EOF) + return 0; + buf = f_win_get (spec, *pos, *pos+1, &size); + if (size == 1) + { + (*pos)++; + return *buf; + } + *pos = F_WIN_EOF; + return 0; +} + +static void regxCodeDel (struct regxCode **pp) +{ + struct regxCode *p = *pp; + if (p) + { +#if HAVE_TCL_OBJECTS + if (p->tcl_obj) + Tcl_DecrRefCount (p->tcl_obj); +#endif + xfree (p->str); + xfree (p); + *pp = NULL; + } +} + +static void regxCodeMk (struct regxCode **pp, const char *buf, int len) +{ + struct regxCode *p; + + p = (struct regxCode *) xmalloc (sizeof(*p)); + p->str = (char *) xmalloc (len+1); + memcpy (p->str, buf, len); + p->str[len] = '\0'; +#if HAVE_TCL_OBJECTS + p->tcl_obj = Tcl_NewStringObj ((char *) buf, len); + if (p->tcl_obj) + Tcl_IncrRefCount (p->tcl_obj); +#endif + *pp = p; +} + +static struct DFA *lexSpecDFA (void) +{ + struct DFA *dfa; + + dfa = dfa_init (); + dfa_parse_cmap_del (dfa, ' '); + dfa_parse_cmap_del (dfa, '\t'); + dfa_parse_cmap_add (dfa, '/', 0); + return dfa; +} + +static void actionListDel (struct lexRuleAction **rap) +{ + struct lexRuleAction *ra1, *ra; + + for (ra = *rap; ra; ra = ra1) + { + ra1 = ra->next; + switch (ra->which) + { + case REGX_PATTERN: + dfa_delete (&ra->u.pattern.dfa); + break; + case REGX_CODE: + regxCodeDel (&ra->u.code); + break; + } + xfree (ra); + } + *rap = NULL; +} + +static struct lexContext *lexContextCreate (const char *name) +{ + struct lexContext *p = (struct lexContext *) xmalloc (sizeof(*p)); + + p->name = xstrdup (name); + p->ruleNo = 1; + p->initFlag = 0; + p->dfa = lexSpecDFA (); + p->rules = NULL; + p->fastRule = NULL; + p->beginActionList = NULL; + p->endActionList = NULL; + p->initActionList = NULL; + p->next = NULL; + return p; +} + +static void lexContextDestroy (struct lexContext *p) +{ + struct lexRule *rp, *rp1; + + dfa_delete (&p->dfa); + xfree (p->fastRule); + for (rp = p->rules; rp; rp = rp1) + { + rp1 = rp->next; + actionListDel (&rp->info.actionList); + xfree (rp); + } + actionListDel (&p->beginActionList); + actionListDel (&p->endActionList); + actionListDel (&p->initActionList); + xfree (p->name); + xfree (p); +} + +static struct lexSpec *lexSpecCreate (const char *name, data1_handle dh) +{ + struct lexSpec *p; + int i; + + p = (struct lexSpec *) xmalloc (sizeof(*p)); + p->name = (char *) xmalloc (strlen(name)+1); + strcpy (p->name, name); + +#if HAVE_TCL_H + p->tcl_interp = 0; +#endif + p->dh = dh; + p->context = NULL; + p->context_stack_size = 100; + p->context_stack = (struct lexContext **) + xmalloc (sizeof(*p->context_stack) * p->context_stack_size); + p->f_win_buf = NULL; + + p->maxLevel = 128; + p->concatBuf = (struct lexConcatBuf *) + xmalloc (sizeof(*p->concatBuf) * p->maxLevel); + for (i = 0; i < p->maxLevel; i++) + { + p->concatBuf[i].max = 0; + p->concatBuf[i].buf = 0; + } + p->d1_stack = (data1_node **) xmalloc (sizeof(*p->d1_stack) * p->maxLevel); + p->d1_level = 0; + return p; +} + +static void lexSpecDestroy (struct lexSpec **pp) +{ + struct lexSpec *p; + struct lexContext *lt; + int i; + + assert (pp); + p = *pp; + if (!p) + return ; + + for (i = 0; i < p->maxLevel; i++) + xfree (p->concatBuf[i].buf); + xfree (p->concatBuf); + + lt = p->context; + while (lt) + { + struct lexContext *lt_next = lt->next; + lexContextDestroy (lt); + lt = lt_next; + } +#if HAVE_TCL_OBJECTS + if (p->tcl_interp) + Tcl_DeleteInterp (p->tcl_interp); +#endif + xfree (p->name); + xfree (p->f_win_buf); + xfree (p->context_stack); + xfree (p->d1_stack); + xfree (p); + *pp = NULL; +} + +static int readParseToken (const char **cpp, int *len) +{ + const char *cp = *cpp; + char cmd[32]; + int i, level; + + while (*cp == ' ' || *cp == '\t' || *cp == '\n' || *cp == '\r') + cp++; + switch (*cp) + { + case '\0': + return 0; + case '/': + *cpp = cp+1; + return REGX_PATTERN; + case '{': + *cpp = cp+1; + level = 1; + while (*++cp) + { + if (*cp == '{') + level++; + else if (*cp == '}') + { + level--; + if (level == 0) + break; + } + } + *len = cp - *cpp; + return REGX_CODE; + default: + i = 0; + while (1) + { + if (*cp >= 'a' && *cp <= 'z') + cmd[i] = *cp; + else if (*cp >= 'A' && *cp <= 'Z') + cmd[i] = *cp + 'a' - 'A'; + else + break; + if (i < (int) sizeof(cmd)-2) + i++; + cp++; + } + cmd[i] = '\0'; + if (i == 0) + { + yaz_log (YLOG_WARN, "bad character %d %c", *cp, *cp); + cp++; + while (*cp && *cp != ' ' && *cp != '\t' && + *cp != '\n' && *cp != '\r') + cp++; + *cpp = cp; + return 0; + } + *cpp = cp; + if (!strcmp (cmd, "begin")) + return REGX_BEGIN; + else if (!strcmp (cmd, "end")) + return REGX_END; + else if (!strcmp (cmd, "body")) + return REGX_BODY; + else if (!strcmp (cmd, "context")) + return REGX_CONTEXT; + else if (!strcmp (cmd, "init")) + return REGX_INIT; + else + { + yaz_log (YLOG_WARN, "bad command %s", cmd); + return 0; + } + } +} + +static int actionListMk (struct lexSpec *spec, const char *s, + struct lexRuleAction **ap) +{ + int r, tok, len; + int bodyMark = 0; + const char *s0; + + while ((tok = readParseToken (&s, &len))) + { + switch (tok) + { + case REGX_BODY: + bodyMark = 1; + continue; + case REGX_CODE: + *ap = (struct lexRuleAction *) xmalloc (sizeof(**ap)); + (*ap)->which = tok; + regxCodeMk (&(*ap)->u.code, s, len); + s += len+1; + break; + case REGX_PATTERN: + *ap = (struct lexRuleAction *) xmalloc (sizeof(**ap)); + (*ap)->which = tok; + (*ap)->u.pattern.body = bodyMark; + bodyMark = 0; + (*ap)->u.pattern.dfa = lexSpecDFA (); + s0 = s; + r = dfa_parse ((*ap)->u.pattern.dfa, &s); + if (r || *s != '/') + { + int pos = s - s0; + xfree (*ap); + *ap = NULL; + yaz_log(YLOG_WARN, "regular expression error '%.*s'", pos, s0); + return -1; + } + else + { + int pos = s - s0; + if (debug_dfa_tran) + printf("pattern: %.*s\n", pos, s0); + dfa_mkstate((*ap)->u.pattern.dfa); + s++; + } + break; + case REGX_BEGIN: + yaz_log (YLOG_WARN, "cannot use BEGIN here"); + continue; + case REGX_INIT: + yaz_log (YLOG_WARN, "cannot use INIT here"); + continue; + case REGX_END: + *ap = (struct lexRuleAction *) xmalloc (sizeof(**ap)); + (*ap)->which = tok; + break; + } + ap = &(*ap)->next; + } + *ap = NULL; + return 0; +} + +int readOneSpec (struct lexSpec *spec, const char *s) +{ + int len, r, tok; + struct lexRule *rp; + struct lexContext *lc; + + tok = readParseToken (&s, &len); + if (tok == REGX_CONTEXT) + { + char context_name[32]; + tok = readParseToken (&s, &len); + if (tok != REGX_CODE) + { + yaz_log (YLOG_WARN, "missing name after CONTEXT keyword"); + return 0; + } + if (len > 31) + len = 31; + memcpy (context_name, s, len); + context_name[len] = '\0'; + lc = lexContextCreate (context_name); + lc->next = spec->context; + spec->context = lc; + return 0; + } + if (!spec->context) + spec->context = lexContextCreate ("main"); + + switch (tok) + { + case REGX_BEGIN: + actionListDel (&spec->context->beginActionList); + actionListMk (spec, s, &spec->context->beginActionList); + break; + case REGX_END: + actionListDel (&spec->context->endActionList); + actionListMk (spec, s, &spec->context->endActionList); + break; + case REGX_INIT: + actionListDel (&spec->context->initActionList); + actionListMk (spec, s, &spec->context->initActionList); + break; + case REGX_PATTERN: +#if REGX_DEBUG + yaz_log (YLOG_LOG, "rule %d %s", spec->context->ruleNo, s); +#endif + r = dfa_parse (spec->context->dfa, &s); + if (r) + { + yaz_log (YLOG_WARN, "regular expression error. r=%d", r); + return -1; + } + if (*s != '/') + { + yaz_log (YLOG_WARN, "expects / at end of pattern. got %c", *s); + return -1; + } + s++; + rp = (struct lexRule *) xmalloc (sizeof(*rp)); + rp->info.no = spec->context->ruleNo++; + rp->next = spec->context->rules; + spec->context->rules = rp; + actionListMk (spec, s, &rp->info.actionList); + } + return 0; +} + +int readFileSpec (struct lexSpec *spec) +{ + struct lexContext *lc; + int c, i, errors = 0; + FILE *spec_inf = 0; + WRBUF lineBuf; + char fname[256]; + +#if HAVE_TCL_H + if (spec->tcl_interp) + { + sprintf (fname, "%s.tflt", spec->name); + spec_inf = data1_path_fopen (spec->dh, fname, "r"); + } +#endif + if (!spec_inf) + { + sprintf (fname, "%s.flt", spec->name); + spec_inf = data1_path_fopen (spec->dh, fname, "r"); + } + if (!spec_inf) + { + yaz_log (YLOG_ERRNO|YLOG_WARN, "cannot read spec file %s", spec->name); + return -1; + } + yaz_log (YLOG_LOG, "reading regx filter %s", fname); +#if HAVE_TCL_H + if (spec->tcl_interp) + yaz_log (YLOG_LOG, "Tcl enabled"); +#endif + +#if 0 + debug_dfa_trav = 0; + debug_dfa_tran = 1; + debug_dfa_followpos = 0; + dfa_verbose = 1; +#endif + + lineBuf = wrbuf_alloc(); + spec->lineNo = 0; + c = getc (spec_inf); + while (c != EOF) + { + wrbuf_rewind (lineBuf); + if (c == '#' || c == '\n' || c == ' ' || c == '\t' || c == '\r') + { + while (c != '\n' && c != EOF) + c = getc (spec_inf); + spec->lineNo++; + if (c == '\n') + c = getc (spec_inf); + } + else + { + int addLine = 0; + + while (1) + { + int c1 = c; + wrbuf_putc(lineBuf, c); + c = getc (spec_inf); + while (c == '\r') + c = getc (spec_inf); + if (c == EOF) + break; + if (c1 == '\n') + { + if (c != ' ' && c != '\t') + break; + addLine++; + } + } + wrbuf_putc(lineBuf, '\0'); + readOneSpec (spec, wrbuf_buf(lineBuf)); + spec->lineNo += addLine; + } + } + fclose (spec_inf); + wrbuf_destroy(lineBuf); + + for (lc = spec->context; lc; lc = lc->next) + { + struct lexRule *rp; + lc->fastRule = (struct lexRuleInfo **) + xmalloc (sizeof(*lc->fastRule) * lc->ruleNo); + for (i = 0; i < lc->ruleNo; i++) + lc->fastRule[i] = NULL; + for (rp = lc->rules; rp; rp = rp->next) + lc->fastRule[rp->info.no] = &rp->info; + dfa_mkstate (lc->dfa); + } + if (errors) + return -1; + + return 0; +} + +#if 0 +static struct lexSpec *curLexSpec = NULL; +#endif + +static void execData (struct lexSpec *spec, + const char *ebuf, int elen, int formatted_text, + const char *attribute_str, int attribute_len) +{ + struct data1_node *res, *parent; + int org_len; + + if (elen == 0) /* shouldn't happen, but it does! */ + return ; +#if REGX_DEBUG + if (elen > 80) + yaz_log (YLOG_LOG, "data(%d bytes) %.40s ... %.*s", elen, + ebuf, 40, ebuf + elen-40); + else if (elen == 1 && ebuf[0] == '\n') + { + yaz_log (YLOG_LOG, "data(new line)"); + } + else if (elen > 0) + yaz_log (YLOG_LOG, "data(%d bytes) %.*s", elen, elen, ebuf); + else + yaz_log (YLOG_LOG, "data(%d bytes)", elen); +#endif + + if (spec->d1_level <= 1) + return; + + parent = spec->d1_stack[spec->d1_level -1]; + assert (parent); + + if (attribute_str) + { + data1_xattr **ap; + res = parent; + if (res->which != DATA1N_tag) + return; + /* sweep through exising attributes.. */ + for (ap = &res->u.tag.attributes; *ap; ap = &(*ap)->next) + if (strlen((*ap)->name) == attribute_len && + !memcmp((*ap)->name, attribute_str, attribute_len)) + break; + if (!*ap) + { + /* new attribute. Create it with name + value */ + *ap = nmem_malloc(spec->m, sizeof(**ap)); + + (*ap)->name = nmem_malloc(spec->m, attribute_len+1); + memcpy((*ap)->name, attribute_str, attribute_len); + (*ap)->name[attribute_len] = '\0'; + + (*ap)->value = nmem_malloc(spec->m, elen+1); + memcpy((*ap)->value, ebuf, elen); + (*ap)->value[elen] = '\0'; + (*ap)->next = 0; + } + else + { + /* append to value if attribute already exists */ + char *nv = nmem_malloc(spec->m, elen + 1 + strlen((*ap)->value)); + strcpy(nv, (*ap)->value); + memcpy (nv + strlen(nv), ebuf, elen); + nv[strlen(nv)+elen] = '\0'; + (*ap)->value = nv; + } + } + else + { + if ((res = spec->d1_stack[spec->d1_level]) && + res->which == DATA1N_data) + org_len = res->u.data.len; + else + { + org_len = 0; + + res = data1_mk_node2 (spec->dh, spec->m, DATA1N_data, parent); + res->u.data.what = DATA1I_text; + res->u.data.len = 0; + res->u.data.formatted_text = formatted_text; + res->u.data.data = 0; + + if (spec->d1_stack[spec->d1_level]) + spec->d1_stack[spec->d1_level]->next = res; + spec->d1_stack[spec->d1_level] = res; + } + if (org_len + elen >= spec->concatBuf[spec->d1_level].max) + { + char *old_buf, *new_buf; + + spec->concatBuf[spec->d1_level].max = org_len + elen + 256; + new_buf = (char *) xmalloc (spec->concatBuf[spec->d1_level].max); + if ((old_buf = spec->concatBuf[spec->d1_level].buf)) + { + memcpy (new_buf, old_buf, org_len); + xfree (old_buf); + } + spec->concatBuf[spec->d1_level].buf = new_buf; + } + memcpy (spec->concatBuf[spec->d1_level].buf + org_len, ebuf, elen); + res->u.data.len += elen; + } +} + +static void execDataP (struct lexSpec *spec, + const char *ebuf, int elen, int formatted_text) +{ + execData (spec, ebuf, elen, formatted_text, 0, 0); +} + +static void tagDataRelease (struct lexSpec *spec) +{ + data1_node *res; + + if ((res = spec->d1_stack[spec->d1_level]) && + res->which == DATA1N_data && + res->u.data.what == DATA1I_text) + { + assert (!res->u.data.data); + assert (res->u.data.len > 0); + if (res->u.data.len > DATA1_LOCALDATA) + res->u.data.data = (char *) nmem_malloc (spec->m, res->u.data.len); + else + res->u.data.data = res->lbuf; + memcpy (res->u.data.data, spec->concatBuf[spec->d1_level].buf, + res->u.data.len); + } +} + +static void variantBegin (struct lexSpec *spec, + const char *class_str, int class_len, + const char *type_str, int type_len, + const char *value_str, int value_len) +{ + struct data1_node *parent = spec->d1_stack[spec->d1_level -1]; + char tclass[DATA1_MAX_SYMBOL], ttype[DATA1_MAX_SYMBOL]; + data1_vartype *tp; + int i; + data1_node *res; + + if (spec->d1_level == 0) + { + yaz_log (YLOG_WARN, "in variant begin. No record type defined"); + return ; + } + if (class_len >= DATA1_MAX_SYMBOL) + class_len = DATA1_MAX_SYMBOL-1; + memcpy (tclass, class_str, class_len); + tclass[class_len] = '\0'; + + if (type_len >= DATA1_MAX_SYMBOL) + type_len = DATA1_MAX_SYMBOL-1; + memcpy (ttype, type_str, type_len); + ttype[type_len] = '\0'; + +#if REGX_DEBUG + yaz_log (YLOG_LOG, "variant begin(%s,%s,%d)", tclass, ttype, + spec->d1_level); +#endif + + if (!(tp = + data1_getvartypeby_absyn(spec->dh, parent->root->u.root.absyn, + tclass, ttype))) + return; + + if (parent->which != DATA1N_variant) + { + res = data1_mk_node2 (spec->dh, spec->m, DATA1N_variant, parent); + if (spec->d1_stack[spec->d1_level]) + tagDataRelease (spec); + spec->d1_stack[spec->d1_level] = res; + spec->d1_stack[++(spec->d1_level)] = NULL; + } + for (i = spec->d1_level-1; spec->d1_stack[i]->which == DATA1N_variant; i--) + if (spec->d1_stack[i]->u.variant.type == tp) + { + spec->d1_level = i; + break; + } + +#if REGX_DEBUG + yaz_log (YLOG_LOG, "variant node(%d)", spec->d1_level); +#endif + parent = spec->d1_stack[spec->d1_level-1]; + res = data1_mk_node2 (spec->dh, spec->m, DATA1N_variant, parent); + res->u.variant.type = tp; + + if (value_len >= DATA1_LOCALDATA) + value_len =DATA1_LOCALDATA-1; + memcpy (res->lbuf, value_str, value_len); + res->lbuf[value_len] = '\0'; + + res->u.variant.value = res->lbuf; + + if (spec->d1_stack[spec->d1_level]) + tagDataRelease (spec); + spec->d1_stack[spec->d1_level] = res; + spec->d1_stack[++(spec->d1_level)] = NULL; +} + +static void tagStrip (const char **tag, int *len) +{ + int i; + + for (i = *len; i > 0 && isspace((*tag)[i-1]); --i) + ; + *len = i; + for (i = 0; i < *len && isspace((*tag)[i]); i++) + ; + *tag += i; + *len -= i; +} + +static void tagBegin (struct lexSpec *spec, + const char *tag, int len) +{ + if (spec->d1_level == 0) + { + yaz_log (YLOG_WARN, "in element begin. No record type defined"); + return ; + } + tagStrip (&tag, &len); + if (spec->d1_stack[spec->d1_level]) + tagDataRelease (spec); + +#if REGX_DEBUG + yaz_log (YLOG_LOG, "begin tag(%.*s, %d)", len, tag, spec->d1_level); +#endif + + spec->d1_stack[spec->d1_level] = data1_mk_tag_n ( + spec->dh, spec->m, tag, len, 0, spec->d1_stack[spec->d1_level -1]); + spec->d1_stack[++(spec->d1_level)] = NULL; +} + +static void tagEnd (struct lexSpec *spec, int min_level, + const char *tag, int len) +{ + tagStrip (&tag, &len); + while (spec->d1_level > min_level) + { + tagDataRelease (spec); + (spec->d1_level)--; + if (spec->d1_level == 0) + break; + if ((spec->d1_stack[spec->d1_level]->which == DATA1N_tag) && + (!tag || + (strlen(spec->d1_stack[spec->d1_level]->u.tag.tag) == + (size_t) len && + !memcmp (spec->d1_stack[spec->d1_level]->u.tag.tag, tag, len)))) + break; + } +#if REGX_DEBUG + yaz_log (YLOG_LOG, "end tag(%d)", spec->d1_level); +#endif +} + + +static int tryMatch (struct lexSpec *spec, int *pptr, int *mptr, + struct DFA *dfa, int greedy) +{ + struct DFA_state *state = dfa->states[0]; + struct DFA_tran *t; + unsigned char c = 0; + unsigned char c_prev = 0; + int ptr = *pptr; /* current pointer */ + int start_ptr = *pptr; /* first char of match */ + int last_ptr = 0; /* last char of match */ + int last_rule = 0; /* rule number of current match */ + int restore_ptr = 0; + int i; + + if (ptr) + { + --ptr; + c = f_win_advance (spec, &ptr); + } + while (1) + { + if (dfa->states[0] == state) + { + c_prev = c; + restore_ptr = ptr; + } + c = f_win_advance (spec, &ptr); + + if (ptr == F_WIN_EOF) + { + if (last_rule) + { + *mptr = start_ptr; + *pptr = last_ptr; + return 1; + } + break; + } + + t = state->trans; + i = state->tran_no; + while (1) + if (--i < 0) /* no transition for character c */ + { + if (last_rule) + { + *mptr = start_ptr; /* match starts here */ + *pptr = last_ptr; /* match end here (+1) */ + return 1; + } + state = dfa->states[0]; + + ptr = restore_ptr; + c = f_win_advance (spec, &ptr); + + start_ptr = ptr; + + break; + } + else if (c >= t->ch[0] && c <= t->ch[1]) + { + state = dfa->states[t->to]; + if (state->rule_no && c_prev == '\n') + { + last_rule = state->rule_no; + last_ptr = ptr; + } + else if (state->rule_nno) + { + last_rule = state->rule_nno; + last_ptr = ptr; + } + break; + } + else + t++; + } + return 0; +} + +static int execTok (struct lexSpec *spec, const char **src, + const char **tokBuf, int *tokLen) +{ + const char *s = *src; + + while (*s == ' ' || *s == '\t') + s++; + if (!*s) + return 0; + if (*s == '$' && s[1] >= '0' && s[1] <= '9') + { + int n = 0; + s++; + while (*s >= '0' && *s <= '9') + n = n*10 + (*s++ -'0'); + if (spec->arg_no == 0) + { + *tokBuf = ""; + *tokLen = 0; + } + else + { + if (n >= spec->arg_no) + n = spec->arg_no-1; + *tokBuf = f_win_get (spec, spec->arg_start[n], spec->arg_end[n], + tokLen); + } + } + else if (*s == '\"') + { + *tokBuf = ++s; + while (*s && *s != '\"') + s++; + *tokLen = s - *tokBuf; + if (*s) + s++; + *src = s; + } + else if (*s == '\n' || *s == ';') + { + *src = s+1; + return 1; + } + else if (*s == '-') + { + *tokBuf = s++; + while (*s && *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r' && + *s != ';') + s++; + *tokLen = s - *tokBuf; + *src = s; + return 3; + } + else + { + *tokBuf = s++; + while (*s && *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r' && + *s != ';') + s++; + *tokLen = s - *tokBuf; + } + *src = s; + return 2; +} + +static char *regxStrz (const char *src, int len, char *str) +{ + if (len > 63) + len = 63; + memcpy (str, src, len); + str[len] = '\0'; + return str; +} + +#if HAVE_TCL_H +static int cmd_tcl_begin (ClientData clientData, Tcl_Interp *interp, + int argc, const char **argv) +{ + struct lexSpec *spec = (struct lexSpec *) clientData; + if (argc < 2) + return TCL_ERROR; + if (!strcmp(argv[1], "record") && argc == 3) + { + const char *absynName = argv[2]; + data1_node *res; + +#if REGX_DEBUG + yaz_log (YLOG_LOG, "begin record %s", absynName); +#endif + res = data1_mk_root (spec->dh, spec->m, absynName); + + spec->d1_level = 0; + + spec->d1_stack[spec->d1_level++] = res; + + res = data1_mk_tag (spec->dh, spec->m, absynName, 0, res); + + spec->d1_stack[spec->d1_level++] = res; + + spec->d1_stack[spec->d1_level] = NULL; + } + else if (!strcmp(argv[1], "element") && argc == 3) + { + tagBegin (spec, argv[2], strlen(argv[2])); + } + else if (!strcmp (argv[1], "variant") && argc == 5) + { + variantBegin (spec, argv[2], strlen(argv[2]), + argv[3], strlen(argv[3]), + argv[4], strlen(argv[4])); + } + else if (!strcmp (argv[1], "context") && argc == 3) + { + struct lexContext *lc = spec->context; +#if REGX_DEBUG + yaz_log (YLOG_LOG, "begin context %s",argv[2]); +#endif + while (lc && strcmp (argv[2], lc->name)) + lc = lc->next; + if (lc) + { + spec->context_stack[++(spec->context_stack_top)] = lc; + } + else + yaz_log (YLOG_WARN, "unknown context %s", argv[2]); + } + else + return TCL_ERROR; + return TCL_OK; +} + +static int cmd_tcl_end (ClientData clientData, Tcl_Interp *interp, + int argc, const char **argv) +{ + struct lexSpec *spec = (struct lexSpec *) clientData; + if (argc < 2) + return TCL_ERROR; + + if (!strcmp (argv[1], "record")) + { + while (spec->d1_level) + { + tagDataRelease (spec); + (spec->d1_level)--; + } +#if REGX_DEBUG + yaz_log (YLOG_LOG, "end record"); +#endif + spec->stop_flag = 1; + } + else if (!strcmp (argv[1], "element")) + { + int min_level = 2; + const char *element = 0; + if (argc >= 3 && !strcmp(argv[2], "-record")) + { + min_level = 0; + if (argc == 4) + element = argv[3]; + } + else + if (argc == 3) + element = argv[2]; + tagEnd (spec, min_level, element, (element ? strlen(element) : 0)); + if (spec->d1_level <= 1) + { +#if REGX_DEBUG + yaz_log (YLOG_LOG, "end element end records"); +#endif + spec->stop_flag = 1; + } + } + else if (!strcmp (argv[1], "context")) + { +#if REGX_DEBUG + yaz_log (YLOG_LOG, "end context"); +#endif + if (spec->context_stack_top) + (spec->context_stack_top)--; + } + else + return TCL_ERROR; + return TCL_OK; +} + +static int cmd_tcl_data (ClientData clientData, Tcl_Interp *interp, + int argc, const char **argv) +{ + int argi = 1; + int textFlag = 0; + const char *element = 0; + const char *attribute = 0; + struct lexSpec *spec = (struct lexSpec *) clientData; + + while (argi < argc) + { + if (!strcmp("-text", argv[argi])) + { + textFlag = 1; + argi++; + } + else if (!strcmp("-element", argv[argi])) + { + argi++; + if (argi < argc) + element = argv[argi++]; + } + else if (!strcmp("-attribute", argv[argi])) + { + argi++; + if (argi < argc) + attribute = argv[argi++]; + } + else + break; + } + if (element) + tagBegin (spec, element, strlen(element)); + + while (argi < argc) + { +#if TCL_MAJOR_VERSION > 8 || (TCL_MAJOR_VERSION == 8 && TCL_MINOR_VERSION > 0) + Tcl_DString ds; + char *native = Tcl_UtfToExternalDString(0, argv[argi], -1, &ds); + execData (spec, native, strlen(native), textFlag, attribute, + attribute ? strlen(attribute) : 0); + Tcl_DStringFree (&ds); +#else + execData (spec, argv[argi], strlen(argv[argi]), textFlag, attribute, + attribute ? strlen(attribute) : 0); +#endif + argi++; + } + if (element) + tagEnd (spec, 2, NULL, 0); + return TCL_OK; +} + +static int cmd_tcl_unread (ClientData clientData, Tcl_Interp *interp, + int argc, const char **argv) +{ + struct lexSpec *spec = (struct lexSpec *) clientData; + int argi = 1; + int offset = 0; + int no; + + while (argi < argc) + { + if (!strcmp("-offset", argv[argi])) + { + argi++; + if (argi < argc) + { + offset = atoi(argv[argi]); + argi++; + } + } + else + break; + } + if (argi != argc-1) + return TCL_ERROR; + no = atoi(argv[argi]); + if (no >= spec->arg_no) + no = spec->arg_no - 1; + spec->ptr = spec->arg_start[no] + offset; + return TCL_OK; +} + +static void execTcl (struct lexSpec *spec, struct regxCode *code) +{ + int i; + int ret; + for (i = 0; i < spec->arg_no; i++) + { + char var_name[10], *var_buf; + int var_len, ch; + + sprintf (var_name, "%d", i); + var_buf = f_win_get (spec, spec->arg_start[i], spec->arg_end[i], + &var_len); + if (var_buf) + { + ch = var_buf[var_len]; + var_buf[var_len] = '\0'; + Tcl_SetVar (spec->tcl_interp, var_name, var_buf, 0); + var_buf[var_len] = ch; + } + } +#if HAVE_TCL_OBJECTS + ret = Tcl_GlobalEvalObj(spec->tcl_interp, code->tcl_obj); +#else + ret = Tcl_GlobalEval (spec->tcl_interp, code->str); +#endif + if (ret != TCL_OK) + { + const char *err = Tcl_GetVar(spec->tcl_interp, "errorInfo", 0); + yaz_log(YLOG_FATAL, "Tcl error, line=%d, \"%s\"\n%s", + spec->tcl_interp->errorLine, + spec->tcl_interp->result, + err ? err : "[NO ERRORINFO]"); + } +} +/* HAVE_TCL_H */ +#endif + +static void execCode (struct lexSpec *spec, struct regxCode *code) +{ + const char *s = code->str; + int cmd_len, r; + const char *cmd_str; + + r = execTok (spec, &s, &cmd_str, &cmd_len); + while (r) + { + char *p, ptmp[64]; + + if (r == 1) + { + r = execTok (spec, &s, &cmd_str, &cmd_len); + continue; + } + p = regxStrz (cmd_str, cmd_len, ptmp); + if (!strcmp (p, "begin")) + { + r = execTok (spec, &s, &cmd_str, &cmd_len); + if (r < 2) + { + yaz_log (YLOG_WARN, "missing keyword after 'begin'"); + continue; + } + p = regxStrz (cmd_str, cmd_len, ptmp); + if (!strcmp (p, "record")) + { + r = execTok (spec, &s, &cmd_str, &cmd_len); + if (r < 2) + continue; + if (spec->d1_level <= 1) + { + static char absynName[64]; + data1_node *res; + + if (cmd_len > 63) + cmd_len = 63; + memcpy (absynName, cmd_str, cmd_len); + absynName[cmd_len] = '\0'; +#if REGX_DEBUG + yaz_log (YLOG_LOG, "begin record %s", absynName); +#endif + res = data1_mk_root (spec->dh, spec->m, absynName); + + spec->d1_level = 0; + + spec->d1_stack[spec->d1_level++] = res; + + res = data1_mk_tag (spec->dh, spec->m, absynName, 0, res); + + spec->d1_stack[spec->d1_level++] = res; + + spec->d1_stack[spec->d1_level] = NULL; + } + r = execTok (spec, &s, &cmd_str, &cmd_len); + } + else if (!strcmp (p, "element")) + { + r = execTok (spec, &s, &cmd_str, &cmd_len); + if (r < 2) + continue; + tagBegin (spec, cmd_str, cmd_len); + r = execTok (spec, &s, &cmd_str, &cmd_len); + } + else if (!strcmp (p, "variant")) + { + int class_len; + const char *class_str = NULL; + int type_len; + const char *type_str = NULL; + int value_len; + const char *value_str = NULL; + r = execTok (spec, &s, &cmd_str, &cmd_len); + if (r < 2) + continue; + class_str = cmd_str; + class_len = cmd_len; + r = execTok (spec, &s, &cmd_str, &cmd_len); + if (r < 2) + continue; + type_str = cmd_str; + type_len = cmd_len; + + r = execTok (spec, &s, &cmd_str, &cmd_len); + if (r < 2) + continue; + value_str = cmd_str; + value_len = cmd_len; + + variantBegin (spec, class_str, class_len, + type_str, type_len, value_str, value_len); + + + r = execTok (spec, &s, &cmd_str, &cmd_len); + } + else if (!strcmp (p, "context")) + { + if (r > 1) + { + struct lexContext *lc = spec->context; + r = execTok (spec, &s, &cmd_str, &cmd_len); + p = regxStrz (cmd_str, cmd_len, ptmp); +#if REGX_DEBUG + yaz_log (YLOG_LOG, "begin context %s", p); +#endif + while (lc && strcmp (p, lc->name)) + lc = lc->next; + if (lc) + spec->context_stack[++(spec->context_stack_top)] = lc; + else + yaz_log (YLOG_WARN, "unknown context %s", p); + + } + r = execTok (spec, &s, &cmd_str, &cmd_len); + } + else + { + yaz_log (YLOG_WARN, "bad keyword '%s' after begin", p); + } + } + else if (!strcmp (p, "end")) + { + r = execTok (spec, &s, &cmd_str, &cmd_len); + if (r < 2) + { + yaz_log (YLOG_WARN, "missing keyword after 'end'"); + continue; + } + p = regxStrz (cmd_str, cmd_len, ptmp); + if (!strcmp (p, "record")) + { + while (spec->d1_level) + { + tagDataRelease (spec); + (spec->d1_level)--; + } + r = execTok (spec, &s, &cmd_str, &cmd_len); +#if REGX_DEBUG + yaz_log (YLOG_LOG, "end record"); +#endif + spec->stop_flag = 1; + } + else if (!strcmp (p, "element")) + { + int min_level = 2; + while ((r = execTok (spec, &s, &cmd_str, &cmd_len)) == 3) + { + if (cmd_len==7 && !memcmp ("-record", cmd_str, cmd_len)) + min_level = 0; + } + if (r > 2) + { + tagEnd (spec, min_level, cmd_str, cmd_len); + r = execTok (spec, &s, &cmd_str, &cmd_len); + } + else + tagEnd (spec, min_level, NULL, 0); + if (spec->d1_level <= 1) + { +#if REGX_DEBUG + yaz_log (YLOG_LOG, "end element end records"); +#endif + spec->stop_flag = 1; + } + + } + else if (!strcmp (p, "context")) + { +#if REGX_DEBUG + yaz_log (YLOG_LOG, "end context"); +#endif + if (spec->context_stack_top) + (spec->context_stack_top)--; + r = execTok (spec, &s, &cmd_str, &cmd_len); + } + else + yaz_log (YLOG_WARN, "bad keyword '%s' after end", p); + } + else if (!strcmp (p, "data")) + { + int textFlag = 0; + int element_len; + const char *element_str = NULL; + int attribute_len; + const char *attribute_str = NULL; + + while ((r = execTok (spec, &s, &cmd_str, &cmd_len)) == 3) + { + if (cmd_len==5 && !memcmp ("-text", cmd_str, cmd_len)) + textFlag = 1; + else if (cmd_len==8 && !memcmp ("-element", cmd_str, cmd_len)) + { + r = execTok (spec, &s, &element_str, &element_len); + if (r < 2) + break; + } + else if (cmd_len==10 && !memcmp ("-attribute", cmd_str, + cmd_len)) + { + r = execTok (spec, &s, &attribute_str, &attribute_len); + if (r < 2) + break; + } + else + yaz_log (YLOG_WARN, "bad data option: %.*s", + cmd_len, cmd_str); + } + if (r != 2) + { + yaz_log (YLOG_WARN, "missing data item after data"); + continue; + } + if (element_str) + tagBegin (spec, element_str, element_len); + do + { + execData (spec, cmd_str, cmd_len, textFlag, + attribute_str, attribute_len); + r = execTok (spec, &s, &cmd_str, &cmd_len); + } while (r > 1); + if (element_str) + tagEnd (spec, 2, NULL, 0); + } + else if (!strcmp (p, "unread")) + { + int no, offset; + r = execTok (spec, &s, &cmd_str, &cmd_len); + if (r==3 && cmd_len == 7 && !memcmp ("-offset", cmd_str, cmd_len)) + { + r = execTok (spec, &s, &cmd_str, &cmd_len); + if (r < 2) + { + yaz_log (YLOG_WARN, "missing number after -offset"); + continue; + } + p = regxStrz (cmd_str, cmd_len, ptmp); + offset = atoi (p); + r = execTok (spec, &s, &cmd_str, &cmd_len); + } + else + offset = 0; + if (r < 2) + { + yaz_log (YLOG_WARN, "missing index after unread command"); + continue; + } + if (cmd_len != 1 || *cmd_str < '0' || *cmd_str > '9') + { + yaz_log (YLOG_WARN, "bad index after unread command"); + continue; + } + else + { + no = *cmd_str - '0'; + if (no >= spec->arg_no) + no = spec->arg_no - 1; + spec->ptr = spec->arg_start[no] + offset; + } + r = execTok (spec, &s, &cmd_str, &cmd_len); + } + else if (!strcmp (p, "context")) + { + if (r > 1) + { + struct lexContext *lc = spec->context; + r = execTok (spec, &s, &cmd_str, &cmd_len); + p = regxStrz (cmd_str, cmd_len, ptmp); + + while (lc && strcmp (p, lc->name)) + lc = lc->next; + if (lc) + spec->context_stack[spec->context_stack_top] = lc; + else + yaz_log (YLOG_WARN, "unknown context %s", p); + + } + r = execTok (spec, &s, &cmd_str, &cmd_len); + } + else + { + yaz_log (YLOG_WARN, "unknown code command '%.*s'", cmd_len, cmd_str); + r = execTok (spec, &s, &cmd_str, &cmd_len); + continue; + } + if (r > 1) + { + yaz_log (YLOG_WARN, "ignoring token %.*s", cmd_len, cmd_str); + do { + r = execTok (spec, &s, &cmd_str, &cmd_len); + } while (r > 1); + } + } +} + + +static int execAction (struct lexSpec *spec, struct lexRuleAction *ap, + int start_ptr, int *pptr) +{ + int sptr; + int arg_start[20]; + int arg_end[20]; + int arg_no = 1; + + if (!ap) + return 1; + arg_start[0] = start_ptr; + arg_end[0] = *pptr; + spec->arg_start = arg_start; + spec->arg_end = arg_end; + + while (ap) + { + switch (ap->which) + { + case REGX_PATTERN: + if (ap->u.pattern.body) + { + arg_start[arg_no] = *pptr; + if (!tryMatch (spec, pptr, &sptr, ap->u.pattern.dfa, 0)) + { + arg_end[arg_no] = F_WIN_EOF; + arg_no++; + arg_start[arg_no] = F_WIN_EOF; + arg_end[arg_no] = F_WIN_EOF; + yaz_log(YLOG_DEBUG, "Pattern match rest of record"); + *pptr = F_WIN_EOF; + } + else + { + arg_end[arg_no] = sptr; + arg_no++; + arg_start[arg_no] = sptr; + arg_end[arg_no] = *pptr; + } + } + else + { + arg_start[arg_no] = *pptr; + if (!tryMatch (spec, pptr, &sptr, ap->u.pattern.dfa, 1)) + return 1; + if (sptr != arg_start[arg_no]) + return 1; + arg_end[arg_no] = *pptr; + } + arg_no++; + break; + case REGX_CODE: + spec->arg_no = arg_no; + spec->ptr = *pptr; +#if HAVE_TCL_H + if (spec->tcl_interp) + execTcl(spec, ap->u.code); + else + execCode (spec, ap->u.code); +#else + execCode (spec, ap->u.code); +#endif + *pptr = spec->ptr; + if (spec->stop_flag) + return 0; + break; + case REGX_END: + arg_start[arg_no] = *pptr; + arg_end[arg_no] = F_WIN_EOF; + arg_no++; + *pptr = F_WIN_EOF; + } + ap = ap->next; + } + return 1; +} + +static int execRule (struct lexSpec *spec, struct lexContext *context, + int ruleNo, int start_ptr, int *pptr) +{ +#if REGX_DEBUG + yaz_log (YLOG_LOG, "exec rule %d", ruleNo); +#endif + return execAction (spec, context->fastRule[ruleNo]->actionList, + start_ptr, pptr); +} + +int lexNode (struct lexSpec *spec, int *ptr) +{ + struct lexContext *context = spec->context_stack[spec->context_stack_top]; + struct DFA_state *state = context->dfa->states[0]; + struct DFA_tran *t; + unsigned char c; + unsigned char c_prev = '\n'; + int i; + int last_rule = 0; /* rule number of current match */ + int last_ptr = *ptr; /* last char of match */ + int start_ptr = *ptr; /* first char of match */ + int skip_ptr = *ptr; /* first char of run */ + int more = 0; + + while (1) + { + c = f_win_advance (spec, ptr); + if (*ptr == F_WIN_EOF) + { + /* end of file met */ + if (last_rule) + { + /* there was a match */ + if (skip_ptr < start_ptr) + { + /* deal with chars that didn't match */ + int size; + char *buf; + buf = f_win_get (spec, skip_ptr, start_ptr, &size); + execDataP (spec, buf, size, 0); + } + /* restore pointer */ + *ptr = last_ptr; + /* execute rule */ + if (!execRule (spec, context, last_rule, start_ptr, ptr)) + return more; + /* restore skip pointer */ + skip_ptr = *ptr; + last_rule = 0; + } + else if (skip_ptr < *ptr) + { + /* deal with chars that didn't match */ + int size; + char *buf; + buf = f_win_get (spec, skip_ptr, *ptr, &size); + execDataP (spec, buf, size, 0); + } + state = context->dfa->states[0]; + if (*ptr == F_WIN_EOF) + return more; + } + t = state->trans; + i = state->tran_no; + while (1) + if (--i < 0) + { /* no transition for character c ... */ + if (last_rule) + { + if (skip_ptr < start_ptr) + { + /* deal with chars that didn't match */ + int size; + char *buf; + buf = f_win_get (spec, skip_ptr, start_ptr, &size); + execDataP (spec, buf, size, 0); + } + /* restore pointer */ + *ptr = last_ptr; + if (!execRule (spec, context, last_rule, start_ptr, ptr)) + { + if (spec->f_win_ef && *ptr != F_WIN_EOF) + { + off_t end_offset = *ptr; +#if REGX_DEBUG + yaz_log (YLOG_LOG, "regx: endf ptr=%d", *ptr); +#endif + (*spec->f_win_ef)(spec->stream, &end_offset); + } + return more; + } + context = spec->context_stack[spec->context_stack_top]; + skip_ptr = *ptr; + last_rule = 0; + last_ptr = start_ptr = *ptr; + if (start_ptr > 0) + { + --start_ptr; + c_prev = f_win_advance (spec, &start_ptr); + } + } + else + { + c_prev = f_win_advance (spec, &start_ptr); + *ptr = start_ptr; + } + state = context->dfa->states[0]; + break; + } + else if (c >= t->ch[0] && c <= t->ch[1]) + { /* transition ... */ + state = context->dfa->states[t->to]; + if (state->rule_no) + { + if (c_prev == '\n') + { + last_rule = state->rule_no; + last_ptr = *ptr; + } + else if (state->rule_nno) + { + last_rule = state->rule_nno; + last_ptr = *ptr; + } + more = 1; + } + break; + } + else + t++; + } + return more; +} + +static data1_node *lexRoot (struct lexSpec *spec, off_t offset, + const char *context_name) +{ + struct lexContext *lt = spec->context; + int ptr = offset; + int ret; + + spec->stop_flag = 0; + spec->d1_level = 0; + spec->context_stack_top = 0; + while (lt) + { + if (!strcmp (lt->name, context_name)) + break; + lt = lt->next; + } + if (!lt) + { + yaz_log (YLOG_WARN, "cannot find context %s", context_name); + return NULL; + } + spec->context_stack[spec->context_stack_top] = lt; + spec->d1_stack[spec->d1_level] = NULL; +#if 1 + if (!lt->initFlag) + { + lt->initFlag = 1; + execAction (spec, lt->initActionList, ptr, &ptr); + } +#endif + execAction (spec, lt->beginActionList, ptr, &ptr); + + ret = lexNode (spec, &ptr); + while (spec->d1_level) + { + tagDataRelease (spec); + (spec->d1_level)--; + } + if (!ret) + return 0; + execAction (spec, lt->endActionList, ptr, &ptr); + return spec->d1_stack[0]; +} + +void grs_destroy(void *clientData) +{ + struct lexSpecs *specs = (struct lexSpecs *) clientData; + if (specs->spec) + { + lexSpecDestroy(&specs->spec); + } + xfree (specs); +} + +void *grs_init(Res res, RecType recType) +{ + struct lexSpecs *specs = (struct lexSpecs *) xmalloc (sizeof(*specs)); + specs->spec = 0; + strcpy(specs->type, ""); + return specs; +} + + +ZEBRA_RES grs_config(void *clientData, Res res, const char *args) +{ + struct lexSpecs *specs = (struct lexSpecs *) clientData; + if (strlen(args) < sizeof(specs->type)) + strcpy(specs->type, args); + return ZEBRA_OK; +} + +data1_node *grs_read_regx (struct grs_read_info *p) +{ + int res; + struct lexSpecs *specs = (struct lexSpecs *) p->clientData; + struct lexSpec **curLexSpec = &specs->spec; + off_t start_offset; + +#if REGX_DEBUG + yaz_log (YLOG_LOG, "grs_read_regx"); +#endif + if (!*curLexSpec || strcmp ((*curLexSpec)->name, specs->type)) + { + if (*curLexSpec) + lexSpecDestroy (curLexSpec); + *curLexSpec = lexSpecCreate (specs->type, p->dh); + res = readFileSpec (*curLexSpec); + if (res) + { + lexSpecDestroy (curLexSpec); + return NULL; + } + } + (*curLexSpec)->dh = p->dh; + start_offset = p->stream->tellf(p->stream); + if (start_offset == 0) + { + (*curLexSpec)->f_win_start = 0; + (*curLexSpec)->f_win_end = 0; + (*curLexSpec)->f_win_rf = p->stream->readf; + (*curLexSpec)->f_win_sf = p->stream->seekf; + (*curLexSpec)->stream = p->stream; + (*curLexSpec)->f_win_ef = p->stream->endf; + (*curLexSpec)->f_win_size = 500000; + } + (*curLexSpec)->m = p->mem; + return lexRoot (*curLexSpec, start_offset, "main"); +} + +static int extract_regx(void *clientData, struct recExtractCtrl *ctrl) +{ + return zebra_grs_extract(clientData, ctrl, grs_read_regx); +} + +static int retrieve_regx(void *clientData, struct recRetrieveCtrl *ctrl) +{ + return zebra_grs_retrieve(clientData, ctrl, grs_read_regx); +} + +static struct recType regx_type = { + 0, + "grs.regx", + grs_init, + grs_config, + grs_destroy, + extract_regx, + retrieve_regx, +}; + + +#if HAVE_TCL_H +data1_node *grs_read_tcl (struct grs_read_info *p) +{ + int res; + struct lexSpecs *specs = (struct lexSpecs *) p->clientData; + struct lexSpec **curLexSpec = &specs->spec; + off_t start_offset; + +#if REGX_DEBUG + yaz_log (YLOG_LOG, "grs_read_tcl"); +#endif + if (!*curLexSpec || strcmp ((*curLexSpec)->name, specs->type)) + { + Tcl_Interp *tcl_interp; + if (*curLexSpec) + lexSpecDestroy (curLexSpec); + *curLexSpec = lexSpecCreate (specs->type, p->dh); + Tcl_FindExecutable(""); + tcl_interp = (*curLexSpec)->tcl_interp = Tcl_CreateInterp(); + Tcl_Init(tcl_interp); + Tcl_CreateCommand (tcl_interp, "begin", cmd_tcl_begin, *curLexSpec, 0); + Tcl_CreateCommand (tcl_interp, "end", cmd_tcl_end, *curLexSpec, 0); + Tcl_CreateCommand (tcl_interp, "data", cmd_tcl_data, *curLexSpec, 0); + Tcl_CreateCommand (tcl_interp, "unread", cmd_tcl_unread, + *curLexSpec, 0); + res = readFileSpec (*curLexSpec); + if (res) + { + lexSpecDestroy (curLexSpec); + return NULL; + } + } + (*curLexSpec)->dh = p->dh; + start_offset = p->stream->tellf(p->stream); + if (start_offset == 0) + { + (*curLexSpec)->f_win_start = 0; + (*curLexSpec)->f_win_end = 0; + (*curLexSpec)->f_win_rf = p->stream->readf; + (*curLexSpec)->f_win_sf = p->stream->seekf; + (*curLexSpec)->stream = p->stream; + (*curLexSpec)->f_win_ef = p->stream->endf; + (*curLexSpec)->f_win_size = 500000; + } + (*curLexSpec)->m = p->mem; + return lexRoot (*curLexSpec, start_offset, "main"); +} + +static int extract_tcl(void *clientData, struct recExtractCtrl *ctrl) +{ + return zebra_grs_extract(clientData, ctrl, grs_read_tcl); +} + +static int retrieve_tcl(void *clientData, struct recRetrieveCtrl *ctrl) +{ + return zebra_grs_retrieve(clientData, ctrl, grs_read_tcl); +} + +static struct recType tcl_type = { + 0, + "grs.tcl", + grs_init, + grs_config, + grs_destroy, + extract_tcl, + retrieve_tcl, +}; + +#endif + +RecType +#ifdef IDZEBRA_STATIC_GRS_REGX +idzebra_filter_grs_regx +#else +idzebra_filter +#endif + +[] = { + ®x_type, +#if HAVE_TCL_H + &tcl_type, +#endif + 0, +}; +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/index/mod_grs_sgml.c b/index/mod_grs_sgml.c new file mode 100644 index 0000000..38fcbe3 --- /dev/null +++ b/index/mod_grs_sgml.c @@ -0,0 +1,143 @@ +/* This file is part of the Zebra server. + Copyright (C) 1995-2008 Index Data + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#include +#include + +#include + +struct sgml_getc_info { + char *buf; + int buf_size; + int size; + int off; + struct ZebraRecStream *stream; + off_t moffset; + WRBUF wrbuf; +}; + +int sgml_getc (void *clientData) +{ + struct sgml_getc_info *p = (struct sgml_getc_info *) clientData; + int res; + + if (p->off < p->size) + return p->buf[(p->off)++]; + if (p->size < p->buf_size) + return 0; + p->moffset += p->off; + p->off = 0; + p->size = 0; + res = p->stream->readf(p->stream, p->buf, p->buf_size); + if (res > 0) + { + p->size += res; + return p->buf[(p->off)++]; + } + return 0; +} + +static data1_node *grs_read_sgml (struct grs_read_info *p) +{ + struct sgml_getc_info *sgi = (struct sgml_getc_info *) p->clientData; + data1_node *node; + int res; + + sgi->moffset = p->stream->tellf(p->stream); + sgi->stream = p->stream; + sgi->off = 0; + sgi->size = 0; + res = sgi->stream->readf(sgi->stream, sgi->buf, sgi->buf_size); + if (res > 0) + sgi->size += res; + else + return 0; + node = data1_read_nodex(p->dh, p->mem, sgml_getc, sgi, sgi->wrbuf); + if (node && p->stream->endf) + { + off_t end_offset = sgi->moffset + sgi->off; + p->stream->endf(sgi->stream, &end_offset); + } + return node; +} + +static void *grs_init_sgml(Res res, RecType recType) +{ + struct sgml_getc_info *p = (struct sgml_getc_info *) xmalloc (sizeof(*p)); + p->buf_size = 512; + p->buf = xmalloc (p->buf_size); + p->wrbuf = wrbuf_alloc(); + return p; +} + +static ZEBRA_RES grs_config_sgml(void *clientData, Res res, const char *args) +{ + return ZEBRA_OK; +} + +static void grs_destroy_sgml(void *clientData) +{ + struct sgml_getc_info *p = (struct sgml_getc_info *) clientData; + + wrbuf_destroy(p->wrbuf); + xfree(p->buf); + xfree(p); +} + +static int grs_extract_sgml(void *clientData, struct recExtractCtrl *ctrl) +{ + return zebra_grs_extract(clientData, ctrl, grs_read_sgml); +} + +static int grs_retrieve_sgml(void *clientData, struct recRetrieveCtrl *ctrl) +{ + return zebra_grs_retrieve(clientData, ctrl, grs_read_sgml); +} + +static struct recType grs_type_sgml = +{ + 0, + "grs.sgml", + grs_init_sgml, + grs_config_sgml, + grs_destroy_sgml, + grs_extract_sgml, + grs_retrieve_sgml +}; + +RecType +#ifdef IDZEBRA_STATIC_GRS_SGML +idzebra_filter_grs_sgml +#else +idzebra_filter +#endif + +[] = { + &grs_type_sgml, + 0, +}; +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/index/mod_grs_xml.c b/index/mod_grs_xml.c new file mode 100644 index 0000000..248342b --- /dev/null +++ b/index/mod_grs_xml.c @@ -0,0 +1,547 @@ +/* This file is part of the Zebra server. + Copyright (C) 1995-2008 Index Data + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#if HAVE_EXPAT_H + +#include +#include +#include +#if HAVE_ICONV_H +#include +#include +#endif + +#include + +#include + +#include +#include + +#include + +#define XML_CHUNK 1024 + +struct user_info { + data1_node *d1_stack[256]; + int level; + data1_handle dh; + NMEM nmem; + int loglevel; +}; + +static void cb_start (void *user, const char *el, const char **attr) +{ + struct user_info *ui = (struct user_info*) user; + if (ui->level == 1) + data1_set_root (ui->dh, ui->d1_stack[0], ui->nmem, el); + ui->d1_stack[ui->level] = data1_mk_tag (ui->dh, ui->nmem, el, attr, + ui->d1_stack[ui->level-1]); + ui->level++; + yaz_log (ui->loglevel, "cb_start %s", el); +} + +static void cb_end (void *user, const char *el) +{ + struct user_info *ui = (struct user_info*) user; + + ui->level--; + yaz_log (ui->loglevel, "cb_end %s", el); +} + +static void cb_chardata (void *user, const char *s, int len) +{ + struct user_info *ui = (struct user_info*) user; +#if 0 + yaz_log (ui->loglevel, "cb_chardata %.*s", len, s); +#endif + ui->d1_stack[ui->level] = data1_mk_text_n (ui->dh, ui->nmem, s, len, + ui->d1_stack[ui->level -1]); +} + +static void cb_decl (void *user, const char *version, const char *encoding, + int standalone) +{ + struct user_info *ui = (struct user_info*) user; + const char *attr_list[7]; + + attr_list[0] = "version"; + attr_list[1] = version; + + attr_list[2] = "encoding"; + attr_list[3] = "UTF-8"; /* internally it's always UTF-8 */ + + attr_list[4] = "standalone"; + attr_list[5] = standalone ? "yes" : "no"; + + attr_list[6] = 0; + + data1_mk_preprocess (ui->dh, ui->nmem, "xml", attr_list, + ui->d1_stack[ui->level-1]); +#if 0 + yaz_log (YLOG_LOG, "decl version=%s encoding=%s", + version ? version : "null", + encoding ? encoding : "null"); +#endif +} + +static void cb_processing (void *user, const char *target, + const char *data) +{ + struct user_info *ui = (struct user_info*) user; + data1_node *res = + data1_mk_preprocess (ui->dh, ui->nmem, target, 0, + ui->d1_stack[ui->level-1]); + data1_mk_text_nf (ui->dh, ui->nmem, data, strlen(data), res); + + yaz_log (ui->loglevel, "decl processing target=%s data=%s", + target ? target : "null", + data ? data : "null"); +} + +static void cb_comment (void *user, const char *data) +{ + struct user_info *ui = (struct user_info*) user; + yaz_log (ui->loglevel, "decl comment data=%s", data ? data : "null"); + data1_mk_comment (ui->dh, ui->nmem, data, ui->d1_stack[ui->level-1]); +} + +static void cb_doctype_start (void *userData, const char *doctypeName, + const char *sysid, const char *pubid, + int has_internal_subset) +{ + struct user_info *ui = (struct user_info*) userData; + yaz_log (ui->loglevel, "doctype start doctype=%s sysid=%s pubid=%s", + doctypeName, sysid, pubid); +} + +static void cb_doctype_end (void *userData) +{ + struct user_info *ui = (struct user_info*) userData; + yaz_log (ui->loglevel, "doctype end"); +} + + +static void cb_entity_decl (void *userData, const char *entityName, + int is_parameter_entity, + const char *value, int value_length, + const char *base, const char *systemId, + const char *publicId, const char *notationName) +{ + struct user_info *ui = (struct user_info*) userData; + yaz_log (ui->loglevel, + "entity decl %s is_para_entry=%d value=%.*s base=%s systemId=%s" + " publicId=%s notationName=%s", + entityName, is_parameter_entity, value_length, value, + base, systemId, publicId, notationName); + +} + +static int cb_external_entity(XML_Parser pparser, + const char *context, + const char *base, + const char *systemId, + const char *publicId) +{ + struct user_info *ui = (struct user_info*) XML_GetUserData(pparser); + FILE *inf; + int done = 0; + XML_Parser parser; + + yaz_log (ui->loglevel, + "external entity context=%s base=%s systemid=%s publicid=%s", + context, base, systemId, publicId); + if (!systemId) + return 1; + + if (!(inf = fopen (systemId, "rb"))) + { + yaz_log (YLOG_WARN|YLOG_ERRNO, "fopen %s", systemId); + return 0; + } + + parser = XML_ExternalEntityParserCreate (pparser, "", 0); + while (!done) + { + int r; + void *buf = XML_GetBuffer (parser, XML_CHUNK); + if (!buf) + { + yaz_log (YLOG_WARN, "XML_GetBuffer fail"); + break; + } + r = fread (buf, 1, XML_CHUNK, inf); + if (r == 0) + { + if (ferror(inf)) + { + yaz_log (YLOG_WARN|YLOG_ERRNO, "fread %s", systemId); + break; + } + done = 1; + } + if (!XML_ParseBuffer (parser, r, done)) + { + done = 1; + yaz_log (YLOG_WARN, "%s:%d:%d:XML error: %s", + systemId, + XML_GetCurrentLineNumber(parser), + XML_GetCurrentColumnNumber(parser), + XML_ErrorString(XML_GetErrorCode(parser))); + } + } + fclose (inf); + XML_ParserFree (parser); + return done; +} + + +#if HAVE_ICONV_H +static int cb_encoding_convert (void *data, const char *s) +{ + iconv_t t = (iconv_t) data; + size_t ret; + size_t outleft = 2; + char outbuf_[2], *outbuf = outbuf_; + size_t inleft = 4; + char *inbuf = (char *) s; + unsigned short code; + +#if 1 + yaz_log(YLOG_LOG, "------------------------- cb_encoding_convert --- "); +#endif + ret = iconv (t, &inbuf, &inleft, &outbuf, &outleft); + if (ret == (size_t) (-1) && errno != E2BIG) + { + iconv (t, 0, 0, 0, 0); + return -1; + } + if (outleft != 0) + return -1; + memcpy (&code, outbuf_, sizeof(short)); + return code; +} + +static void cb_encoding_release (void *data) +{ + iconv_t t = (iconv_t) data; + iconv_close (t); +} + +static int cb_encoding_handler (void *userData, const char *name, + XML_Encoding *info) +{ + int i = 0; + int no_ok = 0; + struct user_info *ui = (struct user_info*) userData; + + iconv_t t = iconv_open ("UNICODE", name); + if (t == (iconv_t) (-1)) + return 0; + + info->data = 0; /* signal that multibyte is not in use */ + yaz_log (ui->loglevel, "Encoding handler of %s", name); + for (i = 0; i<256; i++) + { + size_t ret; + char outbuf_[5]; + char inbuf_[5]; + char *inbuf = inbuf_; + char *outbuf = outbuf_; + size_t inleft = 1; + size_t outleft = 2; + inbuf_[0] = i; + + iconv (t, 0, 0, 0, 0); /* reset iconv */ + + ret = iconv(t, &inbuf, &inleft, &outbuf, &outleft); + if (ret == (size_t) (-1)) + { + if (errno == EILSEQ) + { + yaz_log (ui->loglevel, "Encoding %d: invalid sequence", i); + info->map[i] = -1; /* invalid sequence */ + } + if (errno == EINVAL) + { /* multi byte input */ + int len = 2; + int j = 0; + info->map[i] = -1; + + while (len <= 4) + { + char sbuf[80]; + int k; + inbuf = inbuf_; + inleft = len; + outbuf = outbuf_; + outleft = 2; + + inbuf_[len-1] = j; + iconv (t, 0,0,0,0); + + assert (i >= 0 && i<255); + + *sbuf = 0; + for (k = 0; k 255) + break; + } + else if (errno == EINVAL) + { + len++; + j = 7; + } + } + else if (outleft == 0) + { + info->map[i] = -len; + info->data = t; /* signal that multibyte is in use */ + break; + } + else + { + break; + } + } + if (info->map[i] < -1) + yaz_log (ui->loglevel, "Encoding %d: multibyte input %d", + i, -info->map[i]); + else + yaz_log (ui->loglevel, "Encoding %d: multibyte input failed", + i); + } + if (errno == E2BIG) + { + info->map[i] = -1; /* no room for output */ + if (i != 0) + yaz_log (YLOG_WARN, "Encoding %d: no room for output", + i); + } + } + else if (outleft == 0) + { + unsigned short code; + memcpy (&code, outbuf_, sizeof(short)); + info->map[i] = code; + no_ok++; + } + else + { /* should never happen */ + info->map[i] = -1; + yaz_log (YLOG_DEBUG, "Encoding %d: bad state", i); + } + } + if (info->data) + { /* at least one multi byte */ + info->convert = cb_encoding_convert; + info->release = cb_encoding_release; + } + else + { + /* no multi byte - we no longer need iconv handler */ + iconv_close(t); + info->convert = 0; + info->release = 0; + } + if (!no_ok) + return 0; + return 1; +} +/* HAVE_ICONV_H */ +#endif + +static void cb_ns_start(void *userData, const char *prefix, const char *uri) +{ + struct user_info *ui = (struct user_info*) userData; + if (prefix && uri) + yaz_log(ui->loglevel, "cb_ns_start %s %s", prefix, uri); +} + +static void cb_ns_end(void *userData, const char *prefix) +{ + struct user_info *ui = (struct user_info*) userData; + if (prefix) + yaz_log(ui->loglevel, "cb_ns_end %s", prefix); +} + +data1_node *zebra_read_xml(data1_handle dh, + struct ZebraRecStream *stream, + NMEM m) +{ + XML_Parser parser; + struct user_info uinfo; + int done = 0; + data1_node *first_node; + int no_read = 0; + + uinfo.loglevel = YLOG_DEBUG; + uinfo.level = 1; + uinfo.dh = dh; + uinfo.nmem = m; + uinfo.d1_stack[0] = data1_mk_node2 (dh, m, DATA1N_root, 0); + uinfo.d1_stack[1] = 0; /* indicate no children (see end of routine) */ + + parser = XML_ParserCreate (0 /* encoding */); + + XML_SetElementHandler (parser, cb_start, cb_end); + XML_SetCharacterDataHandler (parser, cb_chardata); + XML_SetXmlDeclHandler (parser, cb_decl); + XML_SetProcessingInstructionHandler (parser, cb_processing); + XML_SetUserData (parser, &uinfo); + XML_SetCommentHandler (parser, cb_comment); + XML_SetDoctypeDeclHandler (parser, cb_doctype_start, cb_doctype_end); + XML_SetEntityDeclHandler (parser, cb_entity_decl); + XML_SetExternalEntityRefHandler (parser, cb_external_entity); + XML_SetNamespaceDeclHandler(parser, cb_ns_start, cb_ns_end); +#if HAVE_ICONV_H + XML_SetUnknownEncodingHandler (parser, cb_encoding_handler, &uinfo); +#endif + while (!done) + { + int r; + void *buf = XML_GetBuffer (parser, XML_CHUNK); + if (!buf) + { + /* error */ + yaz_log (YLOG_WARN, "XML_GetBuffer fail"); + break; + } + r = stream->readf(stream, buf, XML_CHUNK); + if (r < 0) + { + /* error */ + yaz_log (YLOG_WARN, "XML read fail"); + break; + } + else if (r == 0) + done = 1; + else + no_read += r; + if (no_read && !XML_ParseBuffer (parser, r, done)) + { + done = 1; + yaz_log (YLOG_WARN, "%d:%d:XML error: %s", + XML_GetCurrentLineNumber(parser), + XML_GetCurrentColumnNumber(parser), + XML_ErrorString(XML_GetErrorCode(parser))); + } + } + XML_ParserFree (parser); + if (no_read == 0) + return 0; + if (!uinfo.d1_stack[1] || !done) + return 0; + /* insert XML header if not present .. */ + first_node = uinfo.d1_stack[0]->child; + if (first_node->which != DATA1N_preprocess || + strcmp(first_node->u.preprocess.target, "xml")) + { + const char *attr_list[5]; + + attr_list[0] = "version"; + attr_list[1] = "1.0"; + + attr_list[2] = "encoding"; + attr_list[3] = "UTF-8"; /* encoding */ + + attr_list[4] = 0; + + data1_insert_preprocess (uinfo.dh, uinfo.nmem, "xml", attr_list, + uinfo.d1_stack[0]); + } + return uinfo.d1_stack[0]; +} + +struct xml_info { + XML_Expat_Version expat_version; +}; + +static data1_node *grs_read_xml(struct grs_read_info *p) +{ + return zebra_read_xml(p->dh, p->stream, p->mem); +} + +static void *filter_init(Res res, RecType recType) +{ + struct xml_info *p = (struct xml_info *) xmalloc (sizeof(*p)); + + p->expat_version = XML_ExpatVersionInfo(); + + return p; +} + +static void filter_destroy(void *clientData) +{ + struct xml_info *p = (struct xml_info *) clientData; + + xfree (p); +} + +static int filter_extract(void *clientData, struct recExtractCtrl *ctrl) +{ + return zebra_grs_extract(clientData, ctrl, grs_read_xml); +} + +static int filter_retrieve(void *clientData, struct recRetrieveCtrl *ctrl) +{ + return zebra_grs_retrieve(clientData, ctrl, grs_read_xml); +} + +static struct recType filter_type = { + 0, + "grs.xml", + filter_init, + 0, + filter_destroy, + filter_extract, + filter_retrieve, +}; + +RecType +#ifdef IDZEBRA_STATIC_GRS_XML +idzebra_filter_grs_xml +#else +idzebra_filter +#endif + +[] = { + &filter_type, + 0, +}; + +#endif + +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/index/mod_safari.c b/index/mod_safari.c new file mode 100644 index 0000000..be8c567 --- /dev/null +++ b/index/mod_safari.c @@ -0,0 +1,320 @@ +/* This file is part of the Zebra server. + Copyright (C) 1995-2008 Index Data + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#include +#include +#include +#include + +#include +#include + +struct filter_info { + int segments; +}; + +static void *filter_init(Res res, RecType recType) +{ + struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo)); + tinfo->segments = 0; + return tinfo; +} + +static void *filter_init2(Res res, RecType recType) +{ + struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo)); + tinfo->segments = 1; + return tinfo; +} + +static ZEBRA_RES filter_config(void *clientData, Res res, const char *args) +{ + return ZEBRA_OK; +} + +static void filter_destroy(void *clientData) +{ + struct filter_info *tinfo = clientData; + xfree (tinfo); +} + +struct fi_info { + struct recExtractCtrl *p; + char *buf; + int offset; + int max; +}; + +static struct fi_info *fi_open(struct recExtractCtrl *p) +{ + struct fi_info *fi = (struct fi_info *) xmalloc (sizeof(*fi)); + + fi->p = p; + fi->buf = (char *) xmalloc (4096); + fi->offset = 1; + fi->max = 1; + return fi; +} + +static int fi_getchar(struct fi_info *fi, char *dst) +{ + if (fi->offset >= fi->max) + { + if (fi->max <= 0) + return 0; + fi->max = fi->p->stream->readf(fi->p->stream, fi->buf, 4096); + fi->offset = 0; + if (fi->max <= 0) + return 0; + } + *dst = fi->buf[(fi->offset)++]; + return 1; +} + +static int fi_gets(struct fi_info *fi, char *dst, int max) +{ + int l = 0; + while(1) + { + char dstbyte; + if (!fi_getchar(fi, &dstbyte)) + return 0; + if (dstbyte == '\n') + break; + if (l < max) + dst[l++] = dstbyte; + } + dst[l] = '\0'; + return 1; +} + +static void fi_close (struct fi_info *fi) +{ + xfree (fi->buf); + xfree (fi); +} + +static int filter_extract(void *clientData, struct recExtractCtrl *p) +{ + struct filter_info *tinfo = clientData; + char line[512]; + RecWord recWord; + int ret = RECCTRL_EXTRACT_OK; + struct fi_info *fi = fi_open(p); + +#if 0 + yaz_log(YLOG_LOG, "filter_extract off=%ld", + (long) (*fi->p->tellf)(fi->p->fh)); +#endif + (*p->init)(p, &recWord); + + if (!fi_gets(fi, line, sizeof(line)-1)) + ret = RECCTRL_EXTRACT_EOF; + else + { + sscanf(line, "%255s", p->match_criteria); + while (fi_gets(fi, line, sizeof(line)-1)) + { + int nor = 0; + char field[40]; + const char *cp = line; + char type_cstr[2]; +#if 0 + yaz_log(YLOG_LOG, "safari line: %s", line); +#endif + type_cstr[1] = '\0'; + if (*cp >= '0' && *cp <= '9') + type_cstr[0] = '0'; /* the default is 0 (raw) */ + else + type_cstr[0] = *cp++; /* type given */ + type_cstr[1] = '\0'; + + recWord.index_type = type_cstr; + if (tinfo->segments) + { + if (sscanf(cp, ZINT_FORMAT " " ZINT_FORMAT " " ZINT_FORMAT + ZINT_FORMAT " %39s %n", + &recWord.record_id, &recWord.section_id, + &recWord.segment, + &recWord.seqno, + field, &nor) < 5) + { + yaz_log(YLOG_WARN, "Bad safari record line: %s", line); + ret = RECCTRL_EXTRACT_ERROR_GENERIC; + break; + } + } + else + { + if (sscanf(cp, ZINT_FORMAT " " ZINT_FORMAT " " ZINT_FORMAT " %39s %n", + &recWord.record_id, &recWord.section_id, &recWord.seqno, + field, &nor) < 4) + { + yaz_log(YLOG_WARN, "Bad safari record line: %s", line); + ret = RECCTRL_EXTRACT_ERROR_GENERIC; + break; + } + } + for (cp = cp + nor; *cp == ' '; cp++) + ; + recWord.index_name = field; + recWord.term_buf = cp; + recWord.term_len = strlen(cp); + (*p->tokenAdd)(&recWord); + } + } + fi_close(fi); + return ret; +} + +static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p) +{ + int r, filter_ptr = 0; + static char *filter_buf = NULL; + static int filter_size = 0; + int make_header = 1; + int make_body = 1; + const char *elementSetName = NULL; + int no_lines = 0; + + if (p->comp && p->comp->which == Z_RecordComp_simple && + p->comp->u.simple->which == Z_ElementSetNames_generic) + elementSetName = p->comp->u.simple->u.generic; + + if (elementSetName) + { + /* don't make header for the R(aw) element set name */ + if (!strcmp(elementSetName, "R")) + { + make_header = 0; + make_body = 1; + } + /* only make header for the H(eader) element set name */ + else if (!strcmp(elementSetName, "H")) + { + make_header = 1; + make_body = 0; + } + } + while (1) + { + if (filter_ptr + 4096 >= filter_size) + { + char *nb; + + filter_size = 2*filter_size + 8192; + nb = (char *) xmalloc (filter_size); + if (filter_buf) + { + memcpy (nb, filter_buf, filter_ptr); + xfree (filter_buf); + } + filter_buf = nb; + } + if (make_header && filter_ptr == 0) + { + if (p->score >= 0) + { + sprintf (filter_buf, "Rank: %d\n", p->score); + filter_ptr = strlen(filter_buf); + } + sprintf (filter_buf + filter_ptr, "Local Number: " ZINT_FORMAT "\n", + p->localno); + filter_ptr = strlen(filter_buf); + if (p->fname) + { + sprintf (filter_buf + filter_ptr, "Filename: %s\n", p->fname); + filter_ptr = strlen(filter_buf); + } + strcpy(filter_buf+filter_ptr++, "\n"); + } + if (!make_body) + break; + r = p->stream->readf(p->stream, filter_buf + filter_ptr, 4096); + if (r <= 0) + break; + filter_ptr += r; + } + filter_buf[filter_ptr] = '\0'; + if (elementSetName) + { + if (!strcmp (elementSetName, "B")) + no_lines = 4; + if (!strcmp (elementSetName, "M")) + no_lines = 20; + } + if (no_lines) + { + char *p = filter_buf; + int i = 0; + + while (++i <= no_lines && (p = strchr (p, '\n'))) + p++; + if (p) + { + p[1] = '\0'; + filter_ptr = p-filter_buf; + } + } + p->output_format = yaz_oid_recsyn_sutrs; + p->rec_buf = filter_buf; + p->rec_len = filter_ptr; + return 0; +} + +static struct recType filter_type = { + 0, + "safari", + filter_init, + filter_config, + filter_destroy, + filter_extract, + filter_retrieve +}; + +static struct recType filter_type2 = { + 0, + "safari2", + filter_init2, + filter_config, + filter_destroy, + filter_extract, + filter_retrieve +}; + +RecType +#ifdef IDZEBRA_STATIC_SAFARI +idzebra_filter_safari +#else +idzebra_filter +#endif + +[] = { + &filter_type, + &filter_type2, + 0, +}; +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/index/mod_text.c b/index/mod_text.c new file mode 100644 index 0000000..cfa97c0 --- /dev/null +++ b/index/mod_text.c @@ -0,0 +1,264 @@ +/* This file is part of the Zebra server. + Copyright (C) 1995-2008 Index Data + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + + +#include +#include +#include + +#include +#include +#include + +struct filter_info { + char *sep; +}; + +static void *filter_init(Res res, RecType recType) +{ + struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo)); + tinfo->sep = 0; + return tinfo; +} + +static ZEBRA_RES filter_config(void *clientData, Res res, const char *args) +{ + struct filter_info *tinfo = (struct filter_info*) clientData; + xfree(tinfo->sep); + tinfo->sep = 0; + if (args && *args) + tinfo->sep = xstrdup(args); + return ZEBRA_OK; +} + +static void filter_destroy(void *clientData) +{ + struct filter_info *tinfo = clientData; + xfree(tinfo->sep); + xfree(tinfo); +} + +struct buf_info { + struct recExtractCtrl *p; + char *buf; + int offset; + int max; +}; + +static struct buf_info *buf_open(struct recExtractCtrl *p) +{ + struct buf_info *fi = (struct buf_info *) xmalloc(sizeof(*fi)); + + fi->p = p; + fi->buf = (char *) xmalloc(4096); + fi->offset = 1; + fi->max = 1; + return fi; +} + +static int buf_getchar(struct filter_info *tinfo, struct buf_info *fi, char *dst) +{ + if (fi->offset >= fi->max) + { + if (fi->max <= 0) + return 0; + fi->max = fi->p->stream->readf(fi->p->stream, fi->buf, 4096); + fi->offset = 0; + if (fi->max <= 0) + return 0; + } + *dst = fi->buf[(fi->offset)++]; + if (tinfo->sep && *dst == *tinfo->sep) + { + off_t off = fi->p->stream->tellf(fi->p->stream); + off_t end_offset = off - (fi->max - fi->offset); + fi->p->stream->endf(fi->p->stream, &end_offset); + return 0; + } + return 1; +} + +static void buf_close(struct buf_info *fi) +{ + xfree(fi->buf); + xfree(fi); +} + +static int filter_extract(void *clientData, struct recExtractCtrl *p) +{ + struct filter_info *tinfo = clientData; + char w[512]; + RecWord recWord; + int r; + struct buf_info *fi = buf_open(p); + int no_read = 0; + +#if 0 + yaz_log(YLOG_LOG, "filter_extract off=%ld", + (long) (*fi->p->tellf)(fi->p->fh)); +#endif + (*p->init)(p, &recWord); + do + { + int i = 0; + + r = buf_getchar(tinfo, fi, w); + while (r > 0 && i < 511 && w[i] != '\n' && w[i] != '\r') + { + i++; + r = buf_getchar(tinfo, fi, w + i); + } + if (i) + { + no_read += i; + recWord.term_buf = w; + recWord.term_len = i; + (*p->tokenAdd)(&recWord); + } + } while (r > 0); + buf_close(fi); + if (no_read == 0) + return RECCTRL_EXTRACT_EOF; + return RECCTRL_EXTRACT_OK; +} + +static int filter_retrieve(void *clientData, struct recRetrieveCtrl *p) +{ + int r, filter_ptr = 0; + static char *filter_buf = NULL; + static int filter_size = 0; + int make_header = 1; + int make_body = 1; + const char *elementSetName = NULL; + int no_lines = 0; + + if (p->comp && p->comp->which == Z_RecordComp_simple && + p->comp->u.simple->which == Z_ElementSetNames_generic) + elementSetName = p->comp->u.simple->u.generic; + + if (elementSetName) + { + /* don't make header for the R(aw) element set name */ + if (!strcmp(elementSetName, "R")) + { + make_header = 0; + make_body = 1; + } + /* only make header for the H(eader) element set name */ + else if (!strcmp(elementSetName, "H")) + { + make_header = 1; + make_body = 0; + } + } + while (1) + { + if (filter_ptr + 4096 >= filter_size) + { + char *nb; + + filter_size = 2*filter_size + 8192; + nb = (char *) xmalloc(filter_size); + if (filter_buf) + { + memcpy(nb, filter_buf, filter_ptr); + xfree(filter_buf); + } + filter_buf = nb; + } + if (make_header && filter_ptr == 0) + { + if (p->score >= 0) + { + sprintf(filter_buf, "Rank: %d\n", p->score); + filter_ptr = strlen(filter_buf); + } + sprintf(filter_buf + filter_ptr, "Local Number: " ZINT_FORMAT "\n", + p->localno); + filter_ptr = strlen(filter_buf); + if (p->fname) + { + sprintf(filter_buf + filter_ptr, "Filename: %s\n", p->fname); + filter_ptr = strlen(filter_buf); + } + strcpy(filter_buf+filter_ptr++, "\n"); + } + if (!make_body) + break; + r = p->stream->readf(p->stream, filter_buf + filter_ptr, 4096); + if (r <= 0) + break; + filter_ptr += r; + } + filter_buf[filter_ptr] = '\0'; + if (elementSetName) + { + if (!strcmp(elementSetName, "B")) + no_lines = 4; + if (!strcmp(elementSetName, "M")) + no_lines = 20; + } + if (no_lines) + { + char *p = filter_buf; + int i = 0; + + while (++i <= no_lines && (p = strchr(p, '\n'))) + p++; + if (p) + { + p[1] = '\0'; + filter_ptr = p-filter_buf; + } + } + p->output_format = yaz_oid_recsyn_sutrs; + p->rec_buf = filter_buf; + p->rec_len = filter_ptr; + return 0; +} + +static struct recType filter_type = { + 0, + "text", + filter_init, + filter_config, + filter_destroy, + filter_extract, + filter_retrieve +}; + +RecType +#ifdef IDZEBRA_STATIC_TEXT +idzebra_filter_text +#else +idzebra_filter +#endif + +[] = { + &filter_type, + 0, +}; +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/index/rectext.c b/index/rectext.c deleted file mode 100644 index cfa97c0..0000000 --- a/index/rectext.c +++ /dev/null @@ -1,264 +0,0 @@ -/* This file is part of the Zebra server. - Copyright (C) 1995-2008 Index Data - -Zebra is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free -Software Foundation; either version 2, or (at your option) any later -version. - -Zebra is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*/ - - -#include -#include -#include - -#include -#include -#include - -struct filter_info { - char *sep; -}; - -static void *filter_init(Res res, RecType recType) -{ - struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo)); - tinfo->sep = 0; - return tinfo; -} - -static ZEBRA_RES filter_config(void *clientData, Res res, const char *args) -{ - struct filter_info *tinfo = (struct filter_info*) clientData; - xfree(tinfo->sep); - tinfo->sep = 0; - if (args && *args) - tinfo->sep = xstrdup(args); - return ZEBRA_OK; -} - -static void filter_destroy(void *clientData) -{ - struct filter_info *tinfo = clientData; - xfree(tinfo->sep); - xfree(tinfo); -} - -struct buf_info { - struct recExtractCtrl *p; - char *buf; - int offset; - int max; -}; - -static struct buf_info *buf_open(struct recExtractCtrl *p) -{ - struct buf_info *fi = (struct buf_info *) xmalloc(sizeof(*fi)); - - fi->p = p; - fi->buf = (char *) xmalloc(4096); - fi->offset = 1; - fi->max = 1; - return fi; -} - -static int buf_getchar(struct filter_info *tinfo, struct buf_info *fi, char *dst) -{ - if (fi->offset >= fi->max) - { - if (fi->max <= 0) - return 0; - fi->max = fi->p->stream->readf(fi->p->stream, fi->buf, 4096); - fi->offset = 0; - if (fi->max <= 0) - return 0; - } - *dst = fi->buf[(fi->offset)++]; - if (tinfo->sep && *dst == *tinfo->sep) - { - off_t off = fi->p->stream->tellf(fi->p->stream); - off_t end_offset = off - (fi->max - fi->offset); - fi->p->stream->endf(fi->p->stream, &end_offset); - return 0; - } - return 1; -} - -static void buf_close(struct buf_info *fi) -{ - xfree(fi->buf); - xfree(fi); -} - -static int filter_extract(void *clientData, struct recExtractCtrl *p) -{ - struct filter_info *tinfo = clientData; - char w[512]; - RecWord recWord; - int r; - struct buf_info *fi = buf_open(p); - int no_read = 0; - -#if 0 - yaz_log(YLOG_LOG, "filter_extract off=%ld", - (long) (*fi->p->tellf)(fi->p->fh)); -#endif - (*p->init)(p, &recWord); - do - { - int i = 0; - - r = buf_getchar(tinfo, fi, w); - while (r > 0 && i < 511 && w[i] != '\n' && w[i] != '\r') - { - i++; - r = buf_getchar(tinfo, fi, w + i); - } - if (i) - { - no_read += i; - recWord.term_buf = w; - recWord.term_len = i; - (*p->tokenAdd)(&recWord); - } - } while (r > 0); - buf_close(fi); - if (no_read == 0) - return RECCTRL_EXTRACT_EOF; - return RECCTRL_EXTRACT_OK; -} - -static int filter_retrieve(void *clientData, struct recRetrieveCtrl *p) -{ - int r, filter_ptr = 0; - static char *filter_buf = NULL; - static int filter_size = 0; - int make_header = 1; - int make_body = 1; - const char *elementSetName = NULL; - int no_lines = 0; - - if (p->comp && p->comp->which == Z_RecordComp_simple && - p->comp->u.simple->which == Z_ElementSetNames_generic) - elementSetName = p->comp->u.simple->u.generic; - - if (elementSetName) - { - /* don't make header for the R(aw) element set name */ - if (!strcmp(elementSetName, "R")) - { - make_header = 0; - make_body = 1; - } - /* only make header for the H(eader) element set name */ - else if (!strcmp(elementSetName, "H")) - { - make_header = 1; - make_body = 0; - } - } - while (1) - { - if (filter_ptr + 4096 >= filter_size) - { - char *nb; - - filter_size = 2*filter_size + 8192; - nb = (char *) xmalloc(filter_size); - if (filter_buf) - { - memcpy(nb, filter_buf, filter_ptr); - xfree(filter_buf); - } - filter_buf = nb; - } - if (make_header && filter_ptr == 0) - { - if (p->score >= 0) - { - sprintf(filter_buf, "Rank: %d\n", p->score); - filter_ptr = strlen(filter_buf); - } - sprintf(filter_buf + filter_ptr, "Local Number: " ZINT_FORMAT "\n", - p->localno); - filter_ptr = strlen(filter_buf); - if (p->fname) - { - sprintf(filter_buf + filter_ptr, "Filename: %s\n", p->fname); - filter_ptr = strlen(filter_buf); - } - strcpy(filter_buf+filter_ptr++, "\n"); - } - if (!make_body) - break; - r = p->stream->readf(p->stream, filter_buf + filter_ptr, 4096); - if (r <= 0) - break; - filter_ptr += r; - } - filter_buf[filter_ptr] = '\0'; - if (elementSetName) - { - if (!strcmp(elementSetName, "B")) - no_lines = 4; - if (!strcmp(elementSetName, "M")) - no_lines = 20; - } - if (no_lines) - { - char *p = filter_buf; - int i = 0; - - while (++i <= no_lines && (p = strchr(p, '\n'))) - p++; - if (p) - { - p[1] = '\0'; - filter_ptr = p-filter_buf; - } - } - p->output_format = yaz_oid_recsyn_sutrs; - p->rec_buf = filter_buf; - p->rec_len = filter_ptr; - return 0; -} - -static struct recType filter_type = { - 0, - "text", - filter_init, - filter_config, - filter_destroy, - filter_extract, - filter_retrieve -}; - -RecType -#ifdef IDZEBRA_STATIC_TEXT -idzebra_filter_text -#else -idzebra_filter -#endif - -[] = { - &filter_type, - 0, -}; -/* - * Local variables: - * c-basic-offset: 4 - * indent-tabs-mode: nil - * End: - * vim: shiftwidth=4 tabstop=8 expandtab - */ - diff --git a/index/regxread.c b/index/regxread.c deleted file mode 100644 index ecf8f95..0000000 --- a/index/regxread.c +++ /dev/null @@ -1,2037 +0,0 @@ -/* This file is part of the Zebra server. - Copyright (C) 1995-2008 Index Data - -Zebra is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free -Software Foundation; either version 2, or (at your option) any later -version. - -Zebra is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*/ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#if HAVE_TCL_H -#include - -#if MAJOR_VERSION >= 8 -#define HAVE_TCL_OBJECTS -#endif -#endif - -#define REGX_DEBUG 0 - -#define F_WIN_EOF 2000000000 -#define F_WIN_READ 1 - -#define REGX_EOF 0 -#define REGX_PATTERN 1 -#define REGX_BODY 2 -#define REGX_BEGIN 3 -#define REGX_END 4 -#define REGX_CODE 5 -#define REGX_CONTEXT 6 -#define REGX_INIT 7 - -struct regxCode { - char *str; -#if HAVE_TCL_OBJECTS - Tcl_Obj *tcl_obj; -#endif -}; - -struct lexRuleAction { - int which; - union { - struct { - struct DFA *dfa; /* REGX_PATTERN */ - int body; - } pattern; - struct regxCode *code; /* REGX_CODE */ - } u; - struct lexRuleAction *next; -}; - -struct lexRuleInfo { - int no; - struct lexRuleAction *actionList; -}; - -struct lexRule { - struct lexRuleInfo info; - struct lexRule *next; -}; - -struct lexContext { - char *name; - struct DFA *dfa; - struct lexRule *rules; - struct lexRuleInfo **fastRule; - int ruleNo; - int initFlag; - - struct lexRuleAction *beginActionList; - struct lexRuleAction *endActionList; - struct lexRuleAction *initActionList; - struct lexContext *next; -}; - -struct lexConcatBuf { - int max; - char *buf; -}; - -struct lexSpec { - char *name; - struct lexContext *context; - - struct lexContext **context_stack; - int context_stack_size; - int context_stack_top; - - int lineNo; - NMEM m; - data1_handle dh; -#if HAVE_TCL_H - Tcl_Interp *tcl_interp; -#endif - struct ZebraRecStream *stream; - off_t (*f_win_ef)(struct ZebraRecStream *s, off_t *); - - int f_win_start; /* first byte of buffer is this file offset */ - int f_win_end; /* last byte of buffer is this offset - 1 */ - int f_win_size; /* size of buffer */ - char *f_win_buf; /* buffer itself */ - int (*f_win_rf)(struct ZebraRecStream *, char *, size_t); - off_t (*f_win_sf)(struct ZebraRecStream *, off_t); - - struct lexConcatBuf *concatBuf; - int maxLevel; - data1_node **d1_stack; - int d1_level; - int stop_flag; - - int *arg_start; - int *arg_end; - int arg_no; - int ptr; -}; - -struct lexSpecs { - struct lexSpec *spec; - char type[256]; -}; - -static char *f_win_get (struct lexSpec *spec, off_t start_pos, off_t end_pos, - int *size) -{ - int i, r, off = start_pos - spec->f_win_start; - - if (off >= 0 && end_pos <= spec->f_win_end) - { - *size = end_pos - start_pos; - return spec->f_win_buf + off; - } - if (off < 0 || start_pos >= spec->f_win_end) - { - (*spec->f_win_sf)(spec->stream, start_pos); - spec->f_win_start = start_pos; - - if (!spec->f_win_buf) - spec->f_win_buf = (char *) xmalloc (spec->f_win_size); - *size = (*spec->f_win_rf)(spec->stream, spec->f_win_buf, - spec->f_win_size); - spec->f_win_end = spec->f_win_start + *size; - - if (*size > end_pos - start_pos) - *size = end_pos - start_pos; - return spec->f_win_buf; - } - for (i = 0; if_win_end - start_pos; i++) - spec->f_win_buf[i] = spec->f_win_buf[i + off]; - r = (*spec->f_win_rf)(spec->stream, - spec->f_win_buf + i, - spec->f_win_size - i); - spec->f_win_start = start_pos; - spec->f_win_end += r; - *size = i + r; - if (*size > end_pos - start_pos) - *size = end_pos - start_pos; - return spec->f_win_buf; -} - -static int f_win_advance (struct lexSpec *spec, int *pos) -{ - int size; - char *buf; - - if (*pos >= spec->f_win_start && *pos < spec->f_win_end) - return spec->f_win_buf[(*pos)++ - spec->f_win_start]; - if (*pos == F_WIN_EOF) - return 0; - buf = f_win_get (spec, *pos, *pos+1, &size); - if (size == 1) - { - (*pos)++; - return *buf; - } - *pos = F_WIN_EOF; - return 0; -} - -static void regxCodeDel (struct regxCode **pp) -{ - struct regxCode *p = *pp; - if (p) - { -#if HAVE_TCL_OBJECTS - if (p->tcl_obj) - Tcl_DecrRefCount (p->tcl_obj); -#endif - xfree (p->str); - xfree (p); - *pp = NULL; - } -} - -static void regxCodeMk (struct regxCode **pp, const char *buf, int len) -{ - struct regxCode *p; - - p = (struct regxCode *) xmalloc (sizeof(*p)); - p->str = (char *) xmalloc (len+1); - memcpy (p->str, buf, len); - p->str[len] = '\0'; -#if HAVE_TCL_OBJECTS - p->tcl_obj = Tcl_NewStringObj ((char *) buf, len); - if (p->tcl_obj) - Tcl_IncrRefCount (p->tcl_obj); -#endif - *pp = p; -} - -static struct DFA *lexSpecDFA (void) -{ - struct DFA *dfa; - - dfa = dfa_init (); - dfa_parse_cmap_del (dfa, ' '); - dfa_parse_cmap_del (dfa, '\t'); - dfa_parse_cmap_add (dfa, '/', 0); - return dfa; -} - -static void actionListDel (struct lexRuleAction **rap) -{ - struct lexRuleAction *ra1, *ra; - - for (ra = *rap; ra; ra = ra1) - { - ra1 = ra->next; - switch (ra->which) - { - case REGX_PATTERN: - dfa_delete (&ra->u.pattern.dfa); - break; - case REGX_CODE: - regxCodeDel (&ra->u.code); - break; - } - xfree (ra); - } - *rap = NULL; -} - -static struct lexContext *lexContextCreate (const char *name) -{ - struct lexContext *p = (struct lexContext *) xmalloc (sizeof(*p)); - - p->name = xstrdup (name); - p->ruleNo = 1; - p->initFlag = 0; - p->dfa = lexSpecDFA (); - p->rules = NULL; - p->fastRule = NULL; - p->beginActionList = NULL; - p->endActionList = NULL; - p->initActionList = NULL; - p->next = NULL; - return p; -} - -static void lexContextDestroy (struct lexContext *p) -{ - struct lexRule *rp, *rp1; - - dfa_delete (&p->dfa); - xfree (p->fastRule); - for (rp = p->rules; rp; rp = rp1) - { - rp1 = rp->next; - actionListDel (&rp->info.actionList); - xfree (rp); - } - actionListDel (&p->beginActionList); - actionListDel (&p->endActionList); - actionListDel (&p->initActionList); - xfree (p->name); - xfree (p); -} - -static struct lexSpec *lexSpecCreate (const char *name, data1_handle dh) -{ - struct lexSpec *p; - int i; - - p = (struct lexSpec *) xmalloc (sizeof(*p)); - p->name = (char *) xmalloc (strlen(name)+1); - strcpy (p->name, name); - -#if HAVE_TCL_H - p->tcl_interp = 0; -#endif - p->dh = dh; - p->context = NULL; - p->context_stack_size = 100; - p->context_stack = (struct lexContext **) - xmalloc (sizeof(*p->context_stack) * p->context_stack_size); - p->f_win_buf = NULL; - - p->maxLevel = 128; - p->concatBuf = (struct lexConcatBuf *) - xmalloc (sizeof(*p->concatBuf) * p->maxLevel); - for (i = 0; i < p->maxLevel; i++) - { - p->concatBuf[i].max = 0; - p->concatBuf[i].buf = 0; - } - p->d1_stack = (data1_node **) xmalloc (sizeof(*p->d1_stack) * p->maxLevel); - p->d1_level = 0; - return p; -} - -static void lexSpecDestroy (struct lexSpec **pp) -{ - struct lexSpec *p; - struct lexContext *lt; - int i; - - assert (pp); - p = *pp; - if (!p) - return ; - - for (i = 0; i < p->maxLevel; i++) - xfree (p->concatBuf[i].buf); - xfree (p->concatBuf); - - lt = p->context; - while (lt) - { - struct lexContext *lt_next = lt->next; - lexContextDestroy (lt); - lt = lt_next; - } -#if HAVE_TCL_OBJECTS - if (p->tcl_interp) - Tcl_DeleteInterp (p->tcl_interp); -#endif - xfree (p->name); - xfree (p->f_win_buf); - xfree (p->context_stack); - xfree (p->d1_stack); - xfree (p); - *pp = NULL; -} - -static int readParseToken (const char **cpp, int *len) -{ - const char *cp = *cpp; - char cmd[32]; - int i, level; - - while (*cp == ' ' || *cp == '\t' || *cp == '\n' || *cp == '\r') - cp++; - switch (*cp) - { - case '\0': - return 0; - case '/': - *cpp = cp+1; - return REGX_PATTERN; - case '{': - *cpp = cp+1; - level = 1; - while (*++cp) - { - if (*cp == '{') - level++; - else if (*cp == '}') - { - level--; - if (level == 0) - break; - } - } - *len = cp - *cpp; - return REGX_CODE; - default: - i = 0; - while (1) - { - if (*cp >= 'a' && *cp <= 'z') - cmd[i] = *cp; - else if (*cp >= 'A' && *cp <= 'Z') - cmd[i] = *cp + 'a' - 'A'; - else - break; - if (i < (int) sizeof(cmd)-2) - i++; - cp++; - } - cmd[i] = '\0'; - if (i == 0) - { - yaz_log (YLOG_WARN, "bad character %d %c", *cp, *cp); - cp++; - while (*cp && *cp != ' ' && *cp != '\t' && - *cp != '\n' && *cp != '\r') - cp++; - *cpp = cp; - return 0; - } - *cpp = cp; - if (!strcmp (cmd, "begin")) - return REGX_BEGIN; - else if (!strcmp (cmd, "end")) - return REGX_END; - else if (!strcmp (cmd, "body")) - return REGX_BODY; - else if (!strcmp (cmd, "context")) - return REGX_CONTEXT; - else if (!strcmp (cmd, "init")) - return REGX_INIT; - else - { - yaz_log (YLOG_WARN, "bad command %s", cmd); - return 0; - } - } -} - -static int actionListMk (struct lexSpec *spec, const char *s, - struct lexRuleAction **ap) -{ - int r, tok, len; - int bodyMark = 0; - const char *s0; - - while ((tok = readParseToken (&s, &len))) - { - switch (tok) - { - case REGX_BODY: - bodyMark = 1; - continue; - case REGX_CODE: - *ap = (struct lexRuleAction *) xmalloc (sizeof(**ap)); - (*ap)->which = tok; - regxCodeMk (&(*ap)->u.code, s, len); - s += len+1; - break; - case REGX_PATTERN: - *ap = (struct lexRuleAction *) xmalloc (sizeof(**ap)); - (*ap)->which = tok; - (*ap)->u.pattern.body = bodyMark; - bodyMark = 0; - (*ap)->u.pattern.dfa = lexSpecDFA (); - s0 = s; - r = dfa_parse ((*ap)->u.pattern.dfa, &s); - if (r || *s != '/') - { - int pos = s - s0; - xfree (*ap); - *ap = NULL; - yaz_log(YLOG_WARN, "regular expression error '%.*s'", pos, s0); - return -1; - } - else - { - int pos = s - s0; - if (debug_dfa_tran) - printf("pattern: %.*s\n", pos, s0); - dfa_mkstate((*ap)->u.pattern.dfa); - s++; - } - break; - case REGX_BEGIN: - yaz_log (YLOG_WARN, "cannot use BEGIN here"); - continue; - case REGX_INIT: - yaz_log (YLOG_WARN, "cannot use INIT here"); - continue; - case REGX_END: - *ap = (struct lexRuleAction *) xmalloc (sizeof(**ap)); - (*ap)->which = tok; - break; - } - ap = &(*ap)->next; - } - *ap = NULL; - return 0; -} - -int readOneSpec (struct lexSpec *spec, const char *s) -{ - int len, r, tok; - struct lexRule *rp; - struct lexContext *lc; - - tok = readParseToken (&s, &len); - if (tok == REGX_CONTEXT) - { - char context_name[32]; - tok = readParseToken (&s, &len); - if (tok != REGX_CODE) - { - yaz_log (YLOG_WARN, "missing name after CONTEXT keyword"); - return 0; - } - if (len > 31) - len = 31; - memcpy (context_name, s, len); - context_name[len] = '\0'; - lc = lexContextCreate (context_name); - lc->next = spec->context; - spec->context = lc; - return 0; - } - if (!spec->context) - spec->context = lexContextCreate ("main"); - - switch (tok) - { - case REGX_BEGIN: - actionListDel (&spec->context->beginActionList); - actionListMk (spec, s, &spec->context->beginActionList); - break; - case REGX_END: - actionListDel (&spec->context->endActionList); - actionListMk (spec, s, &spec->context->endActionList); - break; - case REGX_INIT: - actionListDel (&spec->context->initActionList); - actionListMk (spec, s, &spec->context->initActionList); - break; - case REGX_PATTERN: -#if REGX_DEBUG - yaz_log (YLOG_LOG, "rule %d %s", spec->context->ruleNo, s); -#endif - r = dfa_parse (spec->context->dfa, &s); - if (r) - { - yaz_log (YLOG_WARN, "regular expression error. r=%d", r); - return -1; - } - if (*s != '/') - { - yaz_log (YLOG_WARN, "expects / at end of pattern. got %c", *s); - return -1; - } - s++; - rp = (struct lexRule *) xmalloc (sizeof(*rp)); - rp->info.no = spec->context->ruleNo++; - rp->next = spec->context->rules; - spec->context->rules = rp; - actionListMk (spec, s, &rp->info.actionList); - } - return 0; -} - -int readFileSpec (struct lexSpec *spec) -{ - struct lexContext *lc; - int c, i, errors = 0; - FILE *spec_inf = 0; - WRBUF lineBuf; - char fname[256]; - -#if HAVE_TCL_H - if (spec->tcl_interp) - { - sprintf (fname, "%s.tflt", spec->name); - spec_inf = data1_path_fopen (spec->dh, fname, "r"); - } -#endif - if (!spec_inf) - { - sprintf (fname, "%s.flt", spec->name); - spec_inf = data1_path_fopen (spec->dh, fname, "r"); - } - if (!spec_inf) - { - yaz_log (YLOG_ERRNO|YLOG_WARN, "cannot read spec file %s", spec->name); - return -1; - } - yaz_log (YLOG_LOG, "reading regx filter %s", fname); -#if HAVE_TCL_H - if (spec->tcl_interp) - yaz_log (YLOG_LOG, "Tcl enabled"); -#endif - -#if 0 - debug_dfa_trav = 0; - debug_dfa_tran = 1; - debug_dfa_followpos = 0; - dfa_verbose = 1; -#endif - - lineBuf = wrbuf_alloc(); - spec->lineNo = 0; - c = getc (spec_inf); - while (c != EOF) - { - wrbuf_rewind (lineBuf); - if (c == '#' || c == '\n' || c == ' ' || c == '\t' || c == '\r') - { - while (c != '\n' && c != EOF) - c = getc (spec_inf); - spec->lineNo++; - if (c == '\n') - c = getc (spec_inf); - } - else - { - int addLine = 0; - - while (1) - { - int c1 = c; - wrbuf_putc(lineBuf, c); - c = getc (spec_inf); - while (c == '\r') - c = getc (spec_inf); - if (c == EOF) - break; - if (c1 == '\n') - { - if (c != ' ' && c != '\t') - break; - addLine++; - } - } - wrbuf_putc(lineBuf, '\0'); - readOneSpec (spec, wrbuf_buf(lineBuf)); - spec->lineNo += addLine; - } - } - fclose (spec_inf); - wrbuf_destroy(lineBuf); - - for (lc = spec->context; lc; lc = lc->next) - { - struct lexRule *rp; - lc->fastRule = (struct lexRuleInfo **) - xmalloc (sizeof(*lc->fastRule) * lc->ruleNo); - for (i = 0; i < lc->ruleNo; i++) - lc->fastRule[i] = NULL; - for (rp = lc->rules; rp; rp = rp->next) - lc->fastRule[rp->info.no] = &rp->info; - dfa_mkstate (lc->dfa); - } - if (errors) - return -1; - - return 0; -} - -#if 0 -static struct lexSpec *curLexSpec = NULL; -#endif - -static void execData (struct lexSpec *spec, - const char *ebuf, int elen, int formatted_text, - const char *attribute_str, int attribute_len) -{ - struct data1_node *res, *parent; - int org_len; - - if (elen == 0) /* shouldn't happen, but it does! */ - return ; -#if REGX_DEBUG - if (elen > 80) - yaz_log (YLOG_LOG, "data(%d bytes) %.40s ... %.*s", elen, - ebuf, 40, ebuf + elen-40); - else if (elen == 1 && ebuf[0] == '\n') - { - yaz_log (YLOG_LOG, "data(new line)"); - } - else if (elen > 0) - yaz_log (YLOG_LOG, "data(%d bytes) %.*s", elen, elen, ebuf); - else - yaz_log (YLOG_LOG, "data(%d bytes)", elen); -#endif - - if (spec->d1_level <= 1) - return; - - parent = spec->d1_stack[spec->d1_level -1]; - assert (parent); - - if (attribute_str) - { - data1_xattr **ap; - res = parent; - if (res->which != DATA1N_tag) - return; - /* sweep through exising attributes.. */ - for (ap = &res->u.tag.attributes; *ap; ap = &(*ap)->next) - if (strlen((*ap)->name) == attribute_len && - !memcmp((*ap)->name, attribute_str, attribute_len)) - break; - if (!*ap) - { - /* new attribute. Create it with name + value */ - *ap = nmem_malloc(spec->m, sizeof(**ap)); - - (*ap)->name = nmem_malloc(spec->m, attribute_len+1); - memcpy((*ap)->name, attribute_str, attribute_len); - (*ap)->name[attribute_len] = '\0'; - - (*ap)->value = nmem_malloc(spec->m, elen+1); - memcpy((*ap)->value, ebuf, elen); - (*ap)->value[elen] = '\0'; - (*ap)->next = 0; - } - else - { - /* append to value if attribute already exists */ - char *nv = nmem_malloc(spec->m, elen + 1 + strlen((*ap)->value)); - strcpy(nv, (*ap)->value); - memcpy (nv + strlen(nv), ebuf, elen); - nv[strlen(nv)+elen] = '\0'; - (*ap)->value = nv; - } - } - else - { - if ((res = spec->d1_stack[spec->d1_level]) && - res->which == DATA1N_data) - org_len = res->u.data.len; - else - { - org_len = 0; - - res = data1_mk_node2 (spec->dh, spec->m, DATA1N_data, parent); - res->u.data.what = DATA1I_text; - res->u.data.len = 0; - res->u.data.formatted_text = formatted_text; - res->u.data.data = 0; - - if (spec->d1_stack[spec->d1_level]) - spec->d1_stack[spec->d1_level]->next = res; - spec->d1_stack[spec->d1_level] = res; - } - if (org_len + elen >= spec->concatBuf[spec->d1_level].max) - { - char *old_buf, *new_buf; - - spec->concatBuf[spec->d1_level].max = org_len + elen + 256; - new_buf = (char *) xmalloc (spec->concatBuf[spec->d1_level].max); - if ((old_buf = spec->concatBuf[spec->d1_level].buf)) - { - memcpy (new_buf, old_buf, org_len); - xfree (old_buf); - } - spec->concatBuf[spec->d1_level].buf = new_buf; - } - memcpy (spec->concatBuf[spec->d1_level].buf + org_len, ebuf, elen); - res->u.data.len += elen; - } -} - -static void execDataP (struct lexSpec *spec, - const char *ebuf, int elen, int formatted_text) -{ - execData (spec, ebuf, elen, formatted_text, 0, 0); -} - -static void tagDataRelease (struct lexSpec *spec) -{ - data1_node *res; - - if ((res = spec->d1_stack[spec->d1_level]) && - res->which == DATA1N_data && - res->u.data.what == DATA1I_text) - { - assert (!res->u.data.data); - assert (res->u.data.len > 0); - if (res->u.data.len > DATA1_LOCALDATA) - res->u.data.data = (char *) nmem_malloc (spec->m, res->u.data.len); - else - res->u.data.data = res->lbuf; - memcpy (res->u.data.data, spec->concatBuf[spec->d1_level].buf, - res->u.data.len); - } -} - -static void variantBegin (struct lexSpec *spec, - const char *class_str, int class_len, - const char *type_str, int type_len, - const char *value_str, int value_len) -{ - struct data1_node *parent = spec->d1_stack[spec->d1_level -1]; - char tclass[DATA1_MAX_SYMBOL], ttype[DATA1_MAX_SYMBOL]; - data1_vartype *tp; - int i; - data1_node *res; - - if (spec->d1_level == 0) - { - yaz_log (YLOG_WARN, "in variant begin. No record type defined"); - return ; - } - if (class_len >= DATA1_MAX_SYMBOL) - class_len = DATA1_MAX_SYMBOL-1; - memcpy (tclass, class_str, class_len); - tclass[class_len] = '\0'; - - if (type_len >= DATA1_MAX_SYMBOL) - type_len = DATA1_MAX_SYMBOL-1; - memcpy (ttype, type_str, type_len); - ttype[type_len] = '\0'; - -#if REGX_DEBUG - yaz_log (YLOG_LOG, "variant begin(%s,%s,%d)", tclass, ttype, - spec->d1_level); -#endif - - if (!(tp = - data1_getvartypeby_absyn(spec->dh, parent->root->u.root.absyn, - tclass, ttype))) - return; - - if (parent->which != DATA1N_variant) - { - res = data1_mk_node2 (spec->dh, spec->m, DATA1N_variant, parent); - if (spec->d1_stack[spec->d1_level]) - tagDataRelease (spec); - spec->d1_stack[spec->d1_level] = res; - spec->d1_stack[++(spec->d1_level)] = NULL; - } - for (i = spec->d1_level-1; spec->d1_stack[i]->which == DATA1N_variant; i--) - if (spec->d1_stack[i]->u.variant.type == tp) - { - spec->d1_level = i; - break; - } - -#if REGX_DEBUG - yaz_log (YLOG_LOG, "variant node(%d)", spec->d1_level); -#endif - parent = spec->d1_stack[spec->d1_level-1]; - res = data1_mk_node2 (spec->dh, spec->m, DATA1N_variant, parent); - res->u.variant.type = tp; - - if (value_len >= DATA1_LOCALDATA) - value_len =DATA1_LOCALDATA-1; - memcpy (res->lbuf, value_str, value_len); - res->lbuf[value_len] = '\0'; - - res->u.variant.value = res->lbuf; - - if (spec->d1_stack[spec->d1_level]) - tagDataRelease (spec); - spec->d1_stack[spec->d1_level] = res; - spec->d1_stack[++(spec->d1_level)] = NULL; -} - -static void tagStrip (const char **tag, int *len) -{ - int i; - - for (i = *len; i > 0 && isspace((*tag)[i-1]); --i) - ; - *len = i; - for (i = 0; i < *len && isspace((*tag)[i]); i++) - ; - *tag += i; - *len -= i; -} - -static void tagBegin (struct lexSpec *spec, - const char *tag, int len) -{ - if (spec->d1_level == 0) - { - yaz_log (YLOG_WARN, "in element begin. No record type defined"); - return ; - } - tagStrip (&tag, &len); - if (spec->d1_stack[spec->d1_level]) - tagDataRelease (spec); - -#if REGX_DEBUG - yaz_log (YLOG_LOG, "begin tag(%.*s, %d)", len, tag, spec->d1_level); -#endif - - spec->d1_stack[spec->d1_level] = data1_mk_tag_n ( - spec->dh, spec->m, tag, len, 0, spec->d1_stack[spec->d1_level -1]); - spec->d1_stack[++(spec->d1_level)] = NULL; -} - -static void tagEnd (struct lexSpec *spec, int min_level, - const char *tag, int len) -{ - tagStrip (&tag, &len); - while (spec->d1_level > min_level) - { - tagDataRelease (spec); - (spec->d1_level)--; - if (spec->d1_level == 0) - break; - if ((spec->d1_stack[spec->d1_level]->which == DATA1N_tag) && - (!tag || - (strlen(spec->d1_stack[spec->d1_level]->u.tag.tag) == - (size_t) len && - !memcmp (spec->d1_stack[spec->d1_level]->u.tag.tag, tag, len)))) - break; - } -#if REGX_DEBUG - yaz_log (YLOG_LOG, "end tag(%d)", spec->d1_level); -#endif -} - - -static int tryMatch (struct lexSpec *spec, int *pptr, int *mptr, - struct DFA *dfa, int greedy) -{ - struct DFA_state *state = dfa->states[0]; - struct DFA_tran *t; - unsigned char c = 0; - unsigned char c_prev = 0; - int ptr = *pptr; /* current pointer */ - int start_ptr = *pptr; /* first char of match */ - int last_ptr = 0; /* last char of match */ - int last_rule = 0; /* rule number of current match */ - int restore_ptr = 0; - int i; - - if (ptr) - { - --ptr; - c = f_win_advance (spec, &ptr); - } - while (1) - { - if (dfa->states[0] == state) - { - c_prev = c; - restore_ptr = ptr; - } - c = f_win_advance (spec, &ptr); - - if (ptr == F_WIN_EOF) - { - if (last_rule) - { - *mptr = start_ptr; - *pptr = last_ptr; - return 1; - } - break; - } - - t = state->trans; - i = state->tran_no; - while (1) - if (--i < 0) /* no transition for character c */ - { - if (last_rule) - { - *mptr = start_ptr; /* match starts here */ - *pptr = last_ptr; /* match end here (+1) */ - return 1; - } - state = dfa->states[0]; - - ptr = restore_ptr; - c = f_win_advance (spec, &ptr); - - start_ptr = ptr; - - break; - } - else if (c >= t->ch[0] && c <= t->ch[1]) - { - state = dfa->states[t->to]; - if (state->rule_no && c_prev == '\n') - { - last_rule = state->rule_no; - last_ptr = ptr; - } - else if (state->rule_nno) - { - last_rule = state->rule_nno; - last_ptr = ptr; - } - break; - } - else - t++; - } - return 0; -} - -static int execTok (struct lexSpec *spec, const char **src, - const char **tokBuf, int *tokLen) -{ - const char *s = *src; - - while (*s == ' ' || *s == '\t') - s++; - if (!*s) - return 0; - if (*s == '$' && s[1] >= '0' && s[1] <= '9') - { - int n = 0; - s++; - while (*s >= '0' && *s <= '9') - n = n*10 + (*s++ -'0'); - if (spec->arg_no == 0) - { - *tokBuf = ""; - *tokLen = 0; - } - else - { - if (n >= spec->arg_no) - n = spec->arg_no-1; - *tokBuf = f_win_get (spec, spec->arg_start[n], spec->arg_end[n], - tokLen); - } - } - else if (*s == '\"') - { - *tokBuf = ++s; - while (*s && *s != '\"') - s++; - *tokLen = s - *tokBuf; - if (*s) - s++; - *src = s; - } - else if (*s == '\n' || *s == ';') - { - *src = s+1; - return 1; - } - else if (*s == '-') - { - *tokBuf = s++; - while (*s && *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r' && - *s != ';') - s++; - *tokLen = s - *tokBuf; - *src = s; - return 3; - } - else - { - *tokBuf = s++; - while (*s && *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r' && - *s != ';') - s++; - *tokLen = s - *tokBuf; - } - *src = s; - return 2; -} - -static char *regxStrz (const char *src, int len, char *str) -{ - if (len > 63) - len = 63; - memcpy (str, src, len); - str[len] = '\0'; - return str; -} - -#if HAVE_TCL_H -static int cmd_tcl_begin (ClientData clientData, Tcl_Interp *interp, - int argc, const char **argv) -{ - struct lexSpec *spec = (struct lexSpec *) clientData; - if (argc < 2) - return TCL_ERROR; - if (!strcmp(argv[1], "record") && argc == 3) - { - const char *absynName = argv[2]; - data1_node *res; - -#if REGX_DEBUG - yaz_log (YLOG_LOG, "begin record %s", absynName); -#endif - res = data1_mk_root (spec->dh, spec->m, absynName); - - spec->d1_level = 0; - - spec->d1_stack[spec->d1_level++] = res; - - res = data1_mk_tag (spec->dh, spec->m, absynName, 0, res); - - spec->d1_stack[spec->d1_level++] = res; - - spec->d1_stack[spec->d1_level] = NULL; - } - else if (!strcmp(argv[1], "element") && argc == 3) - { - tagBegin (spec, argv[2], strlen(argv[2])); - } - else if (!strcmp (argv[1], "variant") && argc == 5) - { - variantBegin (spec, argv[2], strlen(argv[2]), - argv[3], strlen(argv[3]), - argv[4], strlen(argv[4])); - } - else if (!strcmp (argv[1], "context") && argc == 3) - { - struct lexContext *lc = spec->context; -#if REGX_DEBUG - yaz_log (YLOG_LOG, "begin context %s",argv[2]); -#endif - while (lc && strcmp (argv[2], lc->name)) - lc = lc->next; - if (lc) - { - spec->context_stack[++(spec->context_stack_top)] = lc; - } - else - yaz_log (YLOG_WARN, "unknown context %s", argv[2]); - } - else - return TCL_ERROR; - return TCL_OK; -} - -static int cmd_tcl_end (ClientData clientData, Tcl_Interp *interp, - int argc, const char **argv) -{ - struct lexSpec *spec = (struct lexSpec *) clientData; - if (argc < 2) - return TCL_ERROR; - - if (!strcmp (argv[1], "record")) - { - while (spec->d1_level) - { - tagDataRelease (spec); - (spec->d1_level)--; - } -#if REGX_DEBUG - yaz_log (YLOG_LOG, "end record"); -#endif - spec->stop_flag = 1; - } - else if (!strcmp (argv[1], "element")) - { - int min_level = 2; - const char *element = 0; - if (argc >= 3 && !strcmp(argv[2], "-record")) - { - min_level = 0; - if (argc == 4) - element = argv[3]; - } - else - if (argc == 3) - element = argv[2]; - tagEnd (spec, min_level, element, (element ? strlen(element) : 0)); - if (spec->d1_level <= 1) - { -#if REGX_DEBUG - yaz_log (YLOG_LOG, "end element end records"); -#endif - spec->stop_flag = 1; - } - } - else if (!strcmp (argv[1], "context")) - { -#if REGX_DEBUG - yaz_log (YLOG_LOG, "end context"); -#endif - if (spec->context_stack_top) - (spec->context_stack_top)--; - } - else - return TCL_ERROR; - return TCL_OK; -} - -static int cmd_tcl_data (ClientData clientData, Tcl_Interp *interp, - int argc, const char **argv) -{ - int argi = 1; - int textFlag = 0; - const char *element = 0; - const char *attribute = 0; - struct lexSpec *spec = (struct lexSpec *) clientData; - - while (argi < argc) - { - if (!strcmp("-text", argv[argi])) - { - textFlag = 1; - argi++; - } - else if (!strcmp("-element", argv[argi])) - { - argi++; - if (argi < argc) - element = argv[argi++]; - } - else if (!strcmp("-attribute", argv[argi])) - { - argi++; - if (argi < argc) - attribute = argv[argi++]; - } - else - break; - } - if (element) - tagBegin (spec, element, strlen(element)); - - while (argi < argc) - { -#if TCL_MAJOR_VERSION > 8 || (TCL_MAJOR_VERSION == 8 && TCL_MINOR_VERSION > 0) - Tcl_DString ds; - char *native = Tcl_UtfToExternalDString(0, argv[argi], -1, &ds); - execData (spec, native, strlen(native), textFlag, attribute, - attribute ? strlen(attribute) : 0); - Tcl_DStringFree (&ds); -#else - execData (spec, argv[argi], strlen(argv[argi]), textFlag, attribute, - attribute ? strlen(attribute) : 0); -#endif - argi++; - } - if (element) - tagEnd (spec, 2, NULL, 0); - return TCL_OK; -} - -static int cmd_tcl_unread (ClientData clientData, Tcl_Interp *interp, - int argc, const char **argv) -{ - struct lexSpec *spec = (struct lexSpec *) clientData; - int argi = 1; - int offset = 0; - int no; - - while (argi < argc) - { - if (!strcmp("-offset", argv[argi])) - { - argi++; - if (argi < argc) - { - offset = atoi(argv[argi]); - argi++; - } - } - else - break; - } - if (argi != argc-1) - return TCL_ERROR; - no = atoi(argv[argi]); - if (no >= spec->arg_no) - no = spec->arg_no - 1; - spec->ptr = spec->arg_start[no] + offset; - return TCL_OK; -} - -static void execTcl (struct lexSpec *spec, struct regxCode *code) -{ - int i; - int ret; - for (i = 0; i < spec->arg_no; i++) - { - char var_name[10], *var_buf; - int var_len, ch; - - sprintf (var_name, "%d", i); - var_buf = f_win_get (spec, spec->arg_start[i], spec->arg_end[i], - &var_len); - if (var_buf) - { - ch = var_buf[var_len]; - var_buf[var_len] = '\0'; - Tcl_SetVar (spec->tcl_interp, var_name, var_buf, 0); - var_buf[var_len] = ch; - } - } -#if HAVE_TCL_OBJECTS - ret = Tcl_GlobalEvalObj(spec->tcl_interp, code->tcl_obj); -#else - ret = Tcl_GlobalEval (spec->tcl_interp, code->str); -#endif - if (ret != TCL_OK) - { - const char *err = Tcl_GetVar(spec->tcl_interp, "errorInfo", 0); - yaz_log(YLOG_FATAL, "Tcl error, line=%d, \"%s\"\n%s", - spec->tcl_interp->errorLine, - spec->tcl_interp->result, - err ? err : "[NO ERRORINFO]"); - } -} -/* HAVE_TCL_H */ -#endif - -static void execCode (struct lexSpec *spec, struct regxCode *code) -{ - const char *s = code->str; - int cmd_len, r; - const char *cmd_str; - - r = execTok (spec, &s, &cmd_str, &cmd_len); - while (r) - { - char *p, ptmp[64]; - - if (r == 1) - { - r = execTok (spec, &s, &cmd_str, &cmd_len); - continue; - } - p = regxStrz (cmd_str, cmd_len, ptmp); - if (!strcmp (p, "begin")) - { - r = execTok (spec, &s, &cmd_str, &cmd_len); - if (r < 2) - { - yaz_log (YLOG_WARN, "missing keyword after 'begin'"); - continue; - } - p = regxStrz (cmd_str, cmd_len, ptmp); - if (!strcmp (p, "record")) - { - r = execTok (spec, &s, &cmd_str, &cmd_len); - if (r < 2) - continue; - if (spec->d1_level <= 1) - { - static char absynName[64]; - data1_node *res; - - if (cmd_len > 63) - cmd_len = 63; - memcpy (absynName, cmd_str, cmd_len); - absynName[cmd_len] = '\0'; -#if REGX_DEBUG - yaz_log (YLOG_LOG, "begin record %s", absynName); -#endif - res = data1_mk_root (spec->dh, spec->m, absynName); - - spec->d1_level = 0; - - spec->d1_stack[spec->d1_level++] = res; - - res = data1_mk_tag (spec->dh, spec->m, absynName, 0, res); - - spec->d1_stack[spec->d1_level++] = res; - - spec->d1_stack[spec->d1_level] = NULL; - } - r = execTok (spec, &s, &cmd_str, &cmd_len); - } - else if (!strcmp (p, "element")) - { - r = execTok (spec, &s, &cmd_str, &cmd_len); - if (r < 2) - continue; - tagBegin (spec, cmd_str, cmd_len); - r = execTok (spec, &s, &cmd_str, &cmd_len); - } - else if (!strcmp (p, "variant")) - { - int class_len; - const char *class_str = NULL; - int type_len; - const char *type_str = NULL; - int value_len; - const char *value_str = NULL; - r = execTok (spec, &s, &cmd_str, &cmd_len); - if (r < 2) - continue; - class_str = cmd_str; - class_len = cmd_len; - r = execTok (spec, &s, &cmd_str, &cmd_len); - if (r < 2) - continue; - type_str = cmd_str; - type_len = cmd_len; - - r = execTok (spec, &s, &cmd_str, &cmd_len); - if (r < 2) - continue; - value_str = cmd_str; - value_len = cmd_len; - - variantBegin (spec, class_str, class_len, - type_str, type_len, value_str, value_len); - - - r = execTok (spec, &s, &cmd_str, &cmd_len); - } - else if (!strcmp (p, "context")) - { - if (r > 1) - { - struct lexContext *lc = spec->context; - r = execTok (spec, &s, &cmd_str, &cmd_len); - p = regxStrz (cmd_str, cmd_len, ptmp); -#if REGX_DEBUG - yaz_log (YLOG_LOG, "begin context %s", p); -#endif - while (lc && strcmp (p, lc->name)) - lc = lc->next; - if (lc) - spec->context_stack[++(spec->context_stack_top)] = lc; - else - yaz_log (YLOG_WARN, "unknown context %s", p); - - } - r = execTok (spec, &s, &cmd_str, &cmd_len); - } - else - { - yaz_log (YLOG_WARN, "bad keyword '%s' after begin", p); - } - } - else if (!strcmp (p, "end")) - { - r = execTok (spec, &s, &cmd_str, &cmd_len); - if (r < 2) - { - yaz_log (YLOG_WARN, "missing keyword after 'end'"); - continue; - } - p = regxStrz (cmd_str, cmd_len, ptmp); - if (!strcmp (p, "record")) - { - while (spec->d1_level) - { - tagDataRelease (spec); - (spec->d1_level)--; - } - r = execTok (spec, &s, &cmd_str, &cmd_len); -#if REGX_DEBUG - yaz_log (YLOG_LOG, "end record"); -#endif - spec->stop_flag = 1; - } - else if (!strcmp (p, "element")) - { - int min_level = 2; - while ((r = execTok (spec, &s, &cmd_str, &cmd_len)) == 3) - { - if (cmd_len==7 && !memcmp ("-record", cmd_str, cmd_len)) - min_level = 0; - } - if (r > 2) - { - tagEnd (spec, min_level, cmd_str, cmd_len); - r = execTok (spec, &s, &cmd_str, &cmd_len); - } - else - tagEnd (spec, min_level, NULL, 0); - if (spec->d1_level <= 1) - { -#if REGX_DEBUG - yaz_log (YLOG_LOG, "end element end records"); -#endif - spec->stop_flag = 1; - } - - } - else if (!strcmp (p, "context")) - { -#if REGX_DEBUG - yaz_log (YLOG_LOG, "end context"); -#endif - if (spec->context_stack_top) - (spec->context_stack_top)--; - r = execTok (spec, &s, &cmd_str, &cmd_len); - } - else - yaz_log (YLOG_WARN, "bad keyword '%s' after end", p); - } - else if (!strcmp (p, "data")) - { - int textFlag = 0; - int element_len; - const char *element_str = NULL; - int attribute_len; - const char *attribute_str = NULL; - - while ((r = execTok (spec, &s, &cmd_str, &cmd_len)) == 3) - { - if (cmd_len==5 && !memcmp ("-text", cmd_str, cmd_len)) - textFlag = 1; - else if (cmd_len==8 && !memcmp ("-element", cmd_str, cmd_len)) - { - r = execTok (spec, &s, &element_str, &element_len); - if (r < 2) - break; - } - else if (cmd_len==10 && !memcmp ("-attribute", cmd_str, - cmd_len)) - { - r = execTok (spec, &s, &attribute_str, &attribute_len); - if (r < 2) - break; - } - else - yaz_log (YLOG_WARN, "bad data option: %.*s", - cmd_len, cmd_str); - } - if (r != 2) - { - yaz_log (YLOG_WARN, "missing data item after data"); - continue; - } - if (element_str) - tagBegin (spec, element_str, element_len); - do - { - execData (spec, cmd_str, cmd_len, textFlag, - attribute_str, attribute_len); - r = execTok (spec, &s, &cmd_str, &cmd_len); - } while (r > 1); - if (element_str) - tagEnd (spec, 2, NULL, 0); - } - else if (!strcmp (p, "unread")) - { - int no, offset; - r = execTok (spec, &s, &cmd_str, &cmd_len); - if (r==3 && cmd_len == 7 && !memcmp ("-offset", cmd_str, cmd_len)) - { - r = execTok (spec, &s, &cmd_str, &cmd_len); - if (r < 2) - { - yaz_log (YLOG_WARN, "missing number after -offset"); - continue; - } - p = regxStrz (cmd_str, cmd_len, ptmp); - offset = atoi (p); - r = execTok (spec, &s, &cmd_str, &cmd_len); - } - else - offset = 0; - if (r < 2) - { - yaz_log (YLOG_WARN, "missing index after unread command"); - continue; - } - if (cmd_len != 1 || *cmd_str < '0' || *cmd_str > '9') - { - yaz_log (YLOG_WARN, "bad index after unread command"); - continue; - } - else - { - no = *cmd_str - '0'; - if (no >= spec->arg_no) - no = spec->arg_no - 1; - spec->ptr = spec->arg_start[no] + offset; - } - r = execTok (spec, &s, &cmd_str, &cmd_len); - } - else if (!strcmp (p, "context")) - { - if (r > 1) - { - struct lexContext *lc = spec->context; - r = execTok (spec, &s, &cmd_str, &cmd_len); - p = regxStrz (cmd_str, cmd_len, ptmp); - - while (lc && strcmp (p, lc->name)) - lc = lc->next; - if (lc) - spec->context_stack[spec->context_stack_top] = lc; - else - yaz_log (YLOG_WARN, "unknown context %s", p); - - } - r = execTok (spec, &s, &cmd_str, &cmd_len); - } - else - { - yaz_log (YLOG_WARN, "unknown code command '%.*s'", cmd_len, cmd_str); - r = execTok (spec, &s, &cmd_str, &cmd_len); - continue; - } - if (r > 1) - { - yaz_log (YLOG_WARN, "ignoring token %.*s", cmd_len, cmd_str); - do { - r = execTok (spec, &s, &cmd_str, &cmd_len); - } while (r > 1); - } - } -} - - -static int execAction (struct lexSpec *spec, struct lexRuleAction *ap, - int start_ptr, int *pptr) -{ - int sptr; - int arg_start[20]; - int arg_end[20]; - int arg_no = 1; - - if (!ap) - return 1; - arg_start[0] = start_ptr; - arg_end[0] = *pptr; - spec->arg_start = arg_start; - spec->arg_end = arg_end; - - while (ap) - { - switch (ap->which) - { - case REGX_PATTERN: - if (ap->u.pattern.body) - { - arg_start[arg_no] = *pptr; - if (!tryMatch (spec, pptr, &sptr, ap->u.pattern.dfa, 0)) - { - arg_end[arg_no] = F_WIN_EOF; - arg_no++; - arg_start[arg_no] = F_WIN_EOF; - arg_end[arg_no] = F_WIN_EOF; - yaz_log(YLOG_DEBUG, "Pattern match rest of record"); - *pptr = F_WIN_EOF; - } - else - { - arg_end[arg_no] = sptr; - arg_no++; - arg_start[arg_no] = sptr; - arg_end[arg_no] = *pptr; - } - } - else - { - arg_start[arg_no] = *pptr; - if (!tryMatch (spec, pptr, &sptr, ap->u.pattern.dfa, 1)) - return 1; - if (sptr != arg_start[arg_no]) - return 1; - arg_end[arg_no] = *pptr; - } - arg_no++; - break; - case REGX_CODE: - spec->arg_no = arg_no; - spec->ptr = *pptr; -#if HAVE_TCL_H - if (spec->tcl_interp) - execTcl(spec, ap->u.code); - else - execCode (spec, ap->u.code); -#else - execCode (spec, ap->u.code); -#endif - *pptr = spec->ptr; - if (spec->stop_flag) - return 0; - break; - case REGX_END: - arg_start[arg_no] = *pptr; - arg_end[arg_no] = F_WIN_EOF; - arg_no++; - *pptr = F_WIN_EOF; - } - ap = ap->next; - } - return 1; -} - -static int execRule (struct lexSpec *spec, struct lexContext *context, - int ruleNo, int start_ptr, int *pptr) -{ -#if REGX_DEBUG - yaz_log (YLOG_LOG, "exec rule %d", ruleNo); -#endif - return execAction (spec, context->fastRule[ruleNo]->actionList, - start_ptr, pptr); -} - -int lexNode (struct lexSpec *spec, int *ptr) -{ - struct lexContext *context = spec->context_stack[spec->context_stack_top]; - struct DFA_state *state = context->dfa->states[0]; - struct DFA_tran *t; - unsigned char c; - unsigned char c_prev = '\n'; - int i; - int last_rule = 0; /* rule number of current match */ - int last_ptr = *ptr; /* last char of match */ - int start_ptr = *ptr; /* first char of match */ - int skip_ptr = *ptr; /* first char of run */ - int more = 0; - - while (1) - { - c = f_win_advance (spec, ptr); - if (*ptr == F_WIN_EOF) - { - /* end of file met */ - if (last_rule) - { - /* there was a match */ - if (skip_ptr < start_ptr) - { - /* deal with chars that didn't match */ - int size; - char *buf; - buf = f_win_get (spec, skip_ptr, start_ptr, &size); - execDataP (spec, buf, size, 0); - } - /* restore pointer */ - *ptr = last_ptr; - /* execute rule */ - if (!execRule (spec, context, last_rule, start_ptr, ptr)) - return more; - /* restore skip pointer */ - skip_ptr = *ptr; - last_rule = 0; - } - else if (skip_ptr < *ptr) - { - /* deal with chars that didn't match */ - int size; - char *buf; - buf = f_win_get (spec, skip_ptr, *ptr, &size); - execDataP (spec, buf, size, 0); - } - state = context->dfa->states[0]; - if (*ptr == F_WIN_EOF) - return more; - } - t = state->trans; - i = state->tran_no; - while (1) - if (--i < 0) - { /* no transition for character c ... */ - if (last_rule) - { - if (skip_ptr < start_ptr) - { - /* deal with chars that didn't match */ - int size; - char *buf; - buf = f_win_get (spec, skip_ptr, start_ptr, &size); - execDataP (spec, buf, size, 0); - } - /* restore pointer */ - *ptr = last_ptr; - if (!execRule (spec, context, last_rule, start_ptr, ptr)) - { - if (spec->f_win_ef && *ptr != F_WIN_EOF) - { - off_t end_offset = *ptr; -#if REGX_DEBUG - yaz_log (YLOG_LOG, "regx: endf ptr=%d", *ptr); -#endif - (*spec->f_win_ef)(spec->stream, &end_offset); - } - return more; - } - context = spec->context_stack[spec->context_stack_top]; - skip_ptr = *ptr; - last_rule = 0; - last_ptr = start_ptr = *ptr; - if (start_ptr > 0) - { - --start_ptr; - c_prev = f_win_advance (spec, &start_ptr); - } - } - else - { - c_prev = f_win_advance (spec, &start_ptr); - *ptr = start_ptr; - } - state = context->dfa->states[0]; - break; - } - else if (c >= t->ch[0] && c <= t->ch[1]) - { /* transition ... */ - state = context->dfa->states[t->to]; - if (state->rule_no) - { - if (c_prev == '\n') - { - last_rule = state->rule_no; - last_ptr = *ptr; - } - else if (state->rule_nno) - { - last_rule = state->rule_nno; - last_ptr = *ptr; - } - more = 1; - } - break; - } - else - t++; - } - return more; -} - -static data1_node *lexRoot (struct lexSpec *spec, off_t offset, - const char *context_name) -{ - struct lexContext *lt = spec->context; - int ptr = offset; - int ret; - - spec->stop_flag = 0; - spec->d1_level = 0; - spec->context_stack_top = 0; - while (lt) - { - if (!strcmp (lt->name, context_name)) - break; - lt = lt->next; - } - if (!lt) - { - yaz_log (YLOG_WARN, "cannot find context %s", context_name); - return NULL; - } - spec->context_stack[spec->context_stack_top] = lt; - spec->d1_stack[spec->d1_level] = NULL; -#if 1 - if (!lt->initFlag) - { - lt->initFlag = 1; - execAction (spec, lt->initActionList, ptr, &ptr); - } -#endif - execAction (spec, lt->beginActionList, ptr, &ptr); - - ret = lexNode (spec, &ptr); - while (spec->d1_level) - { - tagDataRelease (spec); - (spec->d1_level)--; - } - if (!ret) - return 0; - execAction (spec, lt->endActionList, ptr, &ptr); - return spec->d1_stack[0]; -} - -void grs_destroy(void *clientData) -{ - struct lexSpecs *specs = (struct lexSpecs *) clientData; - if (specs->spec) - { - lexSpecDestroy(&specs->spec); - } - xfree (specs); -} - -void *grs_init(Res res, RecType recType) -{ - struct lexSpecs *specs = (struct lexSpecs *) xmalloc (sizeof(*specs)); - specs->spec = 0; - strcpy(specs->type, ""); - return specs; -} - - -ZEBRA_RES grs_config(void *clientData, Res res, const char *args) -{ - struct lexSpecs *specs = (struct lexSpecs *) clientData; - if (strlen(args) < sizeof(specs->type)) - strcpy(specs->type, args); - return ZEBRA_OK; -} - -data1_node *grs_read_regx (struct grs_read_info *p) -{ - int res; - struct lexSpecs *specs = (struct lexSpecs *) p->clientData; - struct lexSpec **curLexSpec = &specs->spec; - off_t start_offset; - -#if REGX_DEBUG - yaz_log (YLOG_LOG, "grs_read_regx"); -#endif - if (!*curLexSpec || strcmp ((*curLexSpec)->name, specs->type)) - { - if (*curLexSpec) - lexSpecDestroy (curLexSpec); - *curLexSpec = lexSpecCreate (specs->type, p->dh); - res = readFileSpec (*curLexSpec); - if (res) - { - lexSpecDestroy (curLexSpec); - return NULL; - } - } - (*curLexSpec)->dh = p->dh; - start_offset = p->stream->tellf(p->stream); - if (start_offset == 0) - { - (*curLexSpec)->f_win_start = 0; - (*curLexSpec)->f_win_end = 0; - (*curLexSpec)->f_win_rf = p->stream->readf; - (*curLexSpec)->f_win_sf = p->stream->seekf; - (*curLexSpec)->stream = p->stream; - (*curLexSpec)->f_win_ef = p->stream->endf; - (*curLexSpec)->f_win_size = 500000; - } - (*curLexSpec)->m = p->mem; - return lexRoot (*curLexSpec, start_offset, "main"); -} - -static int extract_regx(void *clientData, struct recExtractCtrl *ctrl) -{ - return zebra_grs_extract(clientData, ctrl, grs_read_regx); -} - -static int retrieve_regx(void *clientData, struct recRetrieveCtrl *ctrl) -{ - return zebra_grs_retrieve(clientData, ctrl, grs_read_regx); -} - -static struct recType regx_type = { - 0, - "grs.regx", - grs_init, - grs_config, - grs_destroy, - extract_regx, - retrieve_regx, -}; - - -#if HAVE_TCL_H -data1_node *grs_read_tcl (struct grs_read_info *p) -{ - int res; - struct lexSpecs *specs = (struct lexSpecs *) p->clientData; - struct lexSpec **curLexSpec = &specs->spec; - off_t start_offset; - -#if REGX_DEBUG - yaz_log (YLOG_LOG, "grs_read_tcl"); -#endif - if (!*curLexSpec || strcmp ((*curLexSpec)->name, specs->type)) - { - Tcl_Interp *tcl_interp; - if (*curLexSpec) - lexSpecDestroy (curLexSpec); - *curLexSpec = lexSpecCreate (specs->type, p->dh); - Tcl_FindExecutable(""); - tcl_interp = (*curLexSpec)->tcl_interp = Tcl_CreateInterp(); - Tcl_Init(tcl_interp); - Tcl_CreateCommand (tcl_interp, "begin", cmd_tcl_begin, *curLexSpec, 0); - Tcl_CreateCommand (tcl_interp, "end", cmd_tcl_end, *curLexSpec, 0); - Tcl_CreateCommand (tcl_interp, "data", cmd_tcl_data, *curLexSpec, 0); - Tcl_CreateCommand (tcl_interp, "unread", cmd_tcl_unread, - *curLexSpec, 0); - res = readFileSpec (*curLexSpec); - if (res) - { - lexSpecDestroy (curLexSpec); - return NULL; - } - } - (*curLexSpec)->dh = p->dh; - start_offset = p->stream->tellf(p->stream); - if (start_offset == 0) - { - (*curLexSpec)->f_win_start = 0; - (*curLexSpec)->f_win_end = 0; - (*curLexSpec)->f_win_rf = p->stream->readf; - (*curLexSpec)->f_win_sf = p->stream->seekf; - (*curLexSpec)->stream = p->stream; - (*curLexSpec)->f_win_ef = p->stream->endf; - (*curLexSpec)->f_win_size = 500000; - } - (*curLexSpec)->m = p->mem; - return lexRoot (*curLexSpec, start_offset, "main"); -} - -static int extract_tcl(void *clientData, struct recExtractCtrl *ctrl) -{ - return zebra_grs_extract(clientData, ctrl, grs_read_tcl); -} - -static int retrieve_tcl(void *clientData, struct recRetrieveCtrl *ctrl) -{ - return zebra_grs_retrieve(clientData, ctrl, grs_read_tcl); -} - -static struct recType tcl_type = { - 0, - "grs.tcl", - grs_init, - grs_config, - grs_destroy, - extract_tcl, - retrieve_tcl, -}; - -#endif - -RecType -#ifdef IDZEBRA_STATIC_GRS_REGX -idzebra_filter_grs_regx -#else -idzebra_filter -#endif - -[] = { - ®x_type, -#if HAVE_TCL_H - &tcl_type, -#endif - 0, -}; -/* - * Local variables: - * c-basic-offset: 4 - * indent-tabs-mode: nil - * End: - * vim: shiftwidth=4 tabstop=8 expandtab - */ - diff --git a/index/safari.c b/index/safari.c deleted file mode 100644 index be8c567..0000000 --- a/index/safari.c +++ /dev/null @@ -1,320 +0,0 @@ -/* This file is part of the Zebra server. - Copyright (C) 1995-2008 Index Data - -Zebra is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free -Software Foundation; either version 2, or (at your option) any later -version. - -Zebra is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*/ - -#include -#include -#include -#include - -#include -#include - -struct filter_info { - int segments; -}; - -static void *filter_init(Res res, RecType recType) -{ - struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo)); - tinfo->segments = 0; - return tinfo; -} - -static void *filter_init2(Res res, RecType recType) -{ - struct filter_info *tinfo = (struct filter_info *) xmalloc(sizeof(*tinfo)); - tinfo->segments = 1; - return tinfo; -} - -static ZEBRA_RES filter_config(void *clientData, Res res, const char *args) -{ - return ZEBRA_OK; -} - -static void filter_destroy(void *clientData) -{ - struct filter_info *tinfo = clientData; - xfree (tinfo); -} - -struct fi_info { - struct recExtractCtrl *p; - char *buf; - int offset; - int max; -}; - -static struct fi_info *fi_open(struct recExtractCtrl *p) -{ - struct fi_info *fi = (struct fi_info *) xmalloc (sizeof(*fi)); - - fi->p = p; - fi->buf = (char *) xmalloc (4096); - fi->offset = 1; - fi->max = 1; - return fi; -} - -static int fi_getchar(struct fi_info *fi, char *dst) -{ - if (fi->offset >= fi->max) - { - if (fi->max <= 0) - return 0; - fi->max = fi->p->stream->readf(fi->p->stream, fi->buf, 4096); - fi->offset = 0; - if (fi->max <= 0) - return 0; - } - *dst = fi->buf[(fi->offset)++]; - return 1; -} - -static int fi_gets(struct fi_info *fi, char *dst, int max) -{ - int l = 0; - while(1) - { - char dstbyte; - if (!fi_getchar(fi, &dstbyte)) - return 0; - if (dstbyte == '\n') - break; - if (l < max) - dst[l++] = dstbyte; - } - dst[l] = '\0'; - return 1; -} - -static void fi_close (struct fi_info *fi) -{ - xfree (fi->buf); - xfree (fi); -} - -static int filter_extract(void *clientData, struct recExtractCtrl *p) -{ - struct filter_info *tinfo = clientData; - char line[512]; - RecWord recWord; - int ret = RECCTRL_EXTRACT_OK; - struct fi_info *fi = fi_open(p); - -#if 0 - yaz_log(YLOG_LOG, "filter_extract off=%ld", - (long) (*fi->p->tellf)(fi->p->fh)); -#endif - (*p->init)(p, &recWord); - - if (!fi_gets(fi, line, sizeof(line)-1)) - ret = RECCTRL_EXTRACT_EOF; - else - { - sscanf(line, "%255s", p->match_criteria); - while (fi_gets(fi, line, sizeof(line)-1)) - { - int nor = 0; - char field[40]; - const char *cp = line; - char type_cstr[2]; -#if 0 - yaz_log(YLOG_LOG, "safari line: %s", line); -#endif - type_cstr[1] = '\0'; - if (*cp >= '0' && *cp <= '9') - type_cstr[0] = '0'; /* the default is 0 (raw) */ - else - type_cstr[0] = *cp++; /* type given */ - type_cstr[1] = '\0'; - - recWord.index_type = type_cstr; - if (tinfo->segments) - { - if (sscanf(cp, ZINT_FORMAT " " ZINT_FORMAT " " ZINT_FORMAT - ZINT_FORMAT " %39s %n", - &recWord.record_id, &recWord.section_id, - &recWord.segment, - &recWord.seqno, - field, &nor) < 5) - { - yaz_log(YLOG_WARN, "Bad safari record line: %s", line); - ret = RECCTRL_EXTRACT_ERROR_GENERIC; - break; - } - } - else - { - if (sscanf(cp, ZINT_FORMAT " " ZINT_FORMAT " " ZINT_FORMAT " %39s %n", - &recWord.record_id, &recWord.section_id, &recWord.seqno, - field, &nor) < 4) - { - yaz_log(YLOG_WARN, "Bad safari record line: %s", line); - ret = RECCTRL_EXTRACT_ERROR_GENERIC; - break; - } - } - for (cp = cp + nor; *cp == ' '; cp++) - ; - recWord.index_name = field; - recWord.term_buf = cp; - recWord.term_len = strlen(cp); - (*p->tokenAdd)(&recWord); - } - } - fi_close(fi); - return ret; -} - -static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p) -{ - int r, filter_ptr = 0; - static char *filter_buf = NULL; - static int filter_size = 0; - int make_header = 1; - int make_body = 1; - const char *elementSetName = NULL; - int no_lines = 0; - - if (p->comp && p->comp->which == Z_RecordComp_simple && - p->comp->u.simple->which == Z_ElementSetNames_generic) - elementSetName = p->comp->u.simple->u.generic; - - if (elementSetName) - { - /* don't make header for the R(aw) element set name */ - if (!strcmp(elementSetName, "R")) - { - make_header = 0; - make_body = 1; - } - /* only make header for the H(eader) element set name */ - else if (!strcmp(elementSetName, "H")) - { - make_header = 1; - make_body = 0; - } - } - while (1) - { - if (filter_ptr + 4096 >= filter_size) - { - char *nb; - - filter_size = 2*filter_size + 8192; - nb = (char *) xmalloc (filter_size); - if (filter_buf) - { - memcpy (nb, filter_buf, filter_ptr); - xfree (filter_buf); - } - filter_buf = nb; - } - if (make_header && filter_ptr == 0) - { - if (p->score >= 0) - { - sprintf (filter_buf, "Rank: %d\n", p->score); - filter_ptr = strlen(filter_buf); - } - sprintf (filter_buf + filter_ptr, "Local Number: " ZINT_FORMAT "\n", - p->localno); - filter_ptr = strlen(filter_buf); - if (p->fname) - { - sprintf (filter_buf + filter_ptr, "Filename: %s\n", p->fname); - filter_ptr = strlen(filter_buf); - } - strcpy(filter_buf+filter_ptr++, "\n"); - } - if (!make_body) - break; - r = p->stream->readf(p->stream, filter_buf + filter_ptr, 4096); - if (r <= 0) - break; - filter_ptr += r; - } - filter_buf[filter_ptr] = '\0'; - if (elementSetName) - { - if (!strcmp (elementSetName, "B")) - no_lines = 4; - if (!strcmp (elementSetName, "M")) - no_lines = 20; - } - if (no_lines) - { - char *p = filter_buf; - int i = 0; - - while (++i <= no_lines && (p = strchr (p, '\n'))) - p++; - if (p) - { - p[1] = '\0'; - filter_ptr = p-filter_buf; - } - } - p->output_format = yaz_oid_recsyn_sutrs; - p->rec_buf = filter_buf; - p->rec_len = filter_ptr; - return 0; -} - -static struct recType filter_type = { - 0, - "safari", - filter_init, - filter_config, - filter_destroy, - filter_extract, - filter_retrieve -}; - -static struct recType filter_type2 = { - 0, - "safari2", - filter_init2, - filter_config, - filter_destroy, - filter_extract, - filter_retrieve -}; - -RecType -#ifdef IDZEBRA_STATIC_SAFARI -idzebra_filter_safari -#else -idzebra_filter -#endif - -[] = { - &filter_type, - &filter_type2, - 0, -}; -/* - * Local variables: - * c-basic-offset: 4 - * indent-tabs-mode: nil - * End: - * vim: shiftwidth=4 tabstop=8 expandtab - */ - diff --git a/index/sgmlread.c b/index/sgmlread.c deleted file mode 100644 index 38fcbe3..0000000 --- a/index/sgmlread.c +++ /dev/null @@ -1,143 +0,0 @@ -/* This file is part of the Zebra server. - Copyright (C) 1995-2008 Index Data - -Zebra is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free -Software Foundation; either version 2, or (at your option) any later -version. - -Zebra is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*/ - - -#include -#include - -#include - -struct sgml_getc_info { - char *buf; - int buf_size; - int size; - int off; - struct ZebraRecStream *stream; - off_t moffset; - WRBUF wrbuf; -}; - -int sgml_getc (void *clientData) -{ - struct sgml_getc_info *p = (struct sgml_getc_info *) clientData; - int res; - - if (p->off < p->size) - return p->buf[(p->off)++]; - if (p->size < p->buf_size) - return 0; - p->moffset += p->off; - p->off = 0; - p->size = 0; - res = p->stream->readf(p->stream, p->buf, p->buf_size); - if (res > 0) - { - p->size += res; - return p->buf[(p->off)++]; - } - return 0; -} - -static data1_node *grs_read_sgml (struct grs_read_info *p) -{ - struct sgml_getc_info *sgi = (struct sgml_getc_info *) p->clientData; - data1_node *node; - int res; - - sgi->moffset = p->stream->tellf(p->stream); - sgi->stream = p->stream; - sgi->off = 0; - sgi->size = 0; - res = sgi->stream->readf(sgi->stream, sgi->buf, sgi->buf_size); - if (res > 0) - sgi->size += res; - else - return 0; - node = data1_read_nodex(p->dh, p->mem, sgml_getc, sgi, sgi->wrbuf); - if (node && p->stream->endf) - { - off_t end_offset = sgi->moffset + sgi->off; - p->stream->endf(sgi->stream, &end_offset); - } - return node; -} - -static void *grs_init_sgml(Res res, RecType recType) -{ - struct sgml_getc_info *p = (struct sgml_getc_info *) xmalloc (sizeof(*p)); - p->buf_size = 512; - p->buf = xmalloc (p->buf_size); - p->wrbuf = wrbuf_alloc(); - return p; -} - -static ZEBRA_RES grs_config_sgml(void *clientData, Res res, const char *args) -{ - return ZEBRA_OK; -} - -static void grs_destroy_sgml(void *clientData) -{ - struct sgml_getc_info *p = (struct sgml_getc_info *) clientData; - - wrbuf_destroy(p->wrbuf); - xfree(p->buf); - xfree(p); -} - -static int grs_extract_sgml(void *clientData, struct recExtractCtrl *ctrl) -{ - return zebra_grs_extract(clientData, ctrl, grs_read_sgml); -} - -static int grs_retrieve_sgml(void *clientData, struct recRetrieveCtrl *ctrl) -{ - return zebra_grs_retrieve(clientData, ctrl, grs_read_sgml); -} - -static struct recType grs_type_sgml = -{ - 0, - "grs.sgml", - grs_init_sgml, - grs_config_sgml, - grs_destroy_sgml, - grs_extract_sgml, - grs_retrieve_sgml -}; - -RecType -#ifdef IDZEBRA_STATIC_GRS_SGML -idzebra_filter_grs_sgml -#else -idzebra_filter -#endif - -[] = { - &grs_type_sgml, - 0, -}; -/* - * Local variables: - * c-basic-offset: 4 - * indent-tabs-mode: nil - * End: - * vim: shiftwidth=4 tabstop=8 expandtab - */ - diff --git a/index/xmlread.c b/index/xmlread.c deleted file mode 100644 index 248342b..0000000 --- a/index/xmlread.c +++ /dev/null @@ -1,547 +0,0 @@ -/* This file is part of the Zebra server. - Copyright (C) 1995-2008 Index Data - -Zebra is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free -Software Foundation; either version 2, or (at your option) any later -version. - -Zebra is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*/ - -#if HAVE_EXPAT_H - -#include -#include -#include -#if HAVE_ICONV_H -#include -#include -#endif - -#include - -#include - -#include -#include - -#include - -#define XML_CHUNK 1024 - -struct user_info { - data1_node *d1_stack[256]; - int level; - data1_handle dh; - NMEM nmem; - int loglevel; -}; - -static void cb_start (void *user, const char *el, const char **attr) -{ - struct user_info *ui = (struct user_info*) user; - if (ui->level == 1) - data1_set_root (ui->dh, ui->d1_stack[0], ui->nmem, el); - ui->d1_stack[ui->level] = data1_mk_tag (ui->dh, ui->nmem, el, attr, - ui->d1_stack[ui->level-1]); - ui->level++; - yaz_log (ui->loglevel, "cb_start %s", el); -} - -static void cb_end (void *user, const char *el) -{ - struct user_info *ui = (struct user_info*) user; - - ui->level--; - yaz_log (ui->loglevel, "cb_end %s", el); -} - -static void cb_chardata (void *user, const char *s, int len) -{ - struct user_info *ui = (struct user_info*) user; -#if 0 - yaz_log (ui->loglevel, "cb_chardata %.*s", len, s); -#endif - ui->d1_stack[ui->level] = data1_mk_text_n (ui->dh, ui->nmem, s, len, - ui->d1_stack[ui->level -1]); -} - -static void cb_decl (void *user, const char *version, const char *encoding, - int standalone) -{ - struct user_info *ui = (struct user_info*) user; - const char *attr_list[7]; - - attr_list[0] = "version"; - attr_list[1] = version; - - attr_list[2] = "encoding"; - attr_list[3] = "UTF-8"; /* internally it's always UTF-8 */ - - attr_list[4] = "standalone"; - attr_list[5] = standalone ? "yes" : "no"; - - attr_list[6] = 0; - - data1_mk_preprocess (ui->dh, ui->nmem, "xml", attr_list, - ui->d1_stack[ui->level-1]); -#if 0 - yaz_log (YLOG_LOG, "decl version=%s encoding=%s", - version ? version : "null", - encoding ? encoding : "null"); -#endif -} - -static void cb_processing (void *user, const char *target, - const char *data) -{ - struct user_info *ui = (struct user_info*) user; - data1_node *res = - data1_mk_preprocess (ui->dh, ui->nmem, target, 0, - ui->d1_stack[ui->level-1]); - data1_mk_text_nf (ui->dh, ui->nmem, data, strlen(data), res); - - yaz_log (ui->loglevel, "decl processing target=%s data=%s", - target ? target : "null", - data ? data : "null"); -} - -static void cb_comment (void *user, const char *data) -{ - struct user_info *ui = (struct user_info*) user; - yaz_log (ui->loglevel, "decl comment data=%s", data ? data : "null"); - data1_mk_comment (ui->dh, ui->nmem, data, ui->d1_stack[ui->level-1]); -} - -static void cb_doctype_start (void *userData, const char *doctypeName, - const char *sysid, const char *pubid, - int has_internal_subset) -{ - struct user_info *ui = (struct user_info*) userData; - yaz_log (ui->loglevel, "doctype start doctype=%s sysid=%s pubid=%s", - doctypeName, sysid, pubid); -} - -static void cb_doctype_end (void *userData) -{ - struct user_info *ui = (struct user_info*) userData; - yaz_log (ui->loglevel, "doctype end"); -} - - -static void cb_entity_decl (void *userData, const char *entityName, - int is_parameter_entity, - const char *value, int value_length, - const char *base, const char *systemId, - const char *publicId, const char *notationName) -{ - struct user_info *ui = (struct user_info*) userData; - yaz_log (ui->loglevel, - "entity decl %s is_para_entry=%d value=%.*s base=%s systemId=%s" - " publicId=%s notationName=%s", - entityName, is_parameter_entity, value_length, value, - base, systemId, publicId, notationName); - -} - -static int cb_external_entity(XML_Parser pparser, - const char *context, - const char *base, - const char *systemId, - const char *publicId) -{ - struct user_info *ui = (struct user_info*) XML_GetUserData(pparser); - FILE *inf; - int done = 0; - XML_Parser parser; - - yaz_log (ui->loglevel, - "external entity context=%s base=%s systemid=%s publicid=%s", - context, base, systemId, publicId); - if (!systemId) - return 1; - - if (!(inf = fopen (systemId, "rb"))) - { - yaz_log (YLOG_WARN|YLOG_ERRNO, "fopen %s", systemId); - return 0; - } - - parser = XML_ExternalEntityParserCreate (pparser, "", 0); - while (!done) - { - int r; - void *buf = XML_GetBuffer (parser, XML_CHUNK); - if (!buf) - { - yaz_log (YLOG_WARN, "XML_GetBuffer fail"); - break; - } - r = fread (buf, 1, XML_CHUNK, inf); - if (r == 0) - { - if (ferror(inf)) - { - yaz_log (YLOG_WARN|YLOG_ERRNO, "fread %s", systemId); - break; - } - done = 1; - } - if (!XML_ParseBuffer (parser, r, done)) - { - done = 1; - yaz_log (YLOG_WARN, "%s:%d:%d:XML error: %s", - systemId, - XML_GetCurrentLineNumber(parser), - XML_GetCurrentColumnNumber(parser), - XML_ErrorString(XML_GetErrorCode(parser))); - } - } - fclose (inf); - XML_ParserFree (parser); - return done; -} - - -#if HAVE_ICONV_H -static int cb_encoding_convert (void *data, const char *s) -{ - iconv_t t = (iconv_t) data; - size_t ret; - size_t outleft = 2; - char outbuf_[2], *outbuf = outbuf_; - size_t inleft = 4; - char *inbuf = (char *) s; - unsigned short code; - -#if 1 - yaz_log(YLOG_LOG, "------------------------- cb_encoding_convert --- "); -#endif - ret = iconv (t, &inbuf, &inleft, &outbuf, &outleft); - if (ret == (size_t) (-1) && errno != E2BIG) - { - iconv (t, 0, 0, 0, 0); - return -1; - } - if (outleft != 0) - return -1; - memcpy (&code, outbuf_, sizeof(short)); - return code; -} - -static void cb_encoding_release (void *data) -{ - iconv_t t = (iconv_t) data; - iconv_close (t); -} - -static int cb_encoding_handler (void *userData, const char *name, - XML_Encoding *info) -{ - int i = 0; - int no_ok = 0; - struct user_info *ui = (struct user_info*) userData; - - iconv_t t = iconv_open ("UNICODE", name); - if (t == (iconv_t) (-1)) - return 0; - - info->data = 0; /* signal that multibyte is not in use */ - yaz_log (ui->loglevel, "Encoding handler of %s", name); - for (i = 0; i<256; i++) - { - size_t ret; - char outbuf_[5]; - char inbuf_[5]; - char *inbuf = inbuf_; - char *outbuf = outbuf_; - size_t inleft = 1; - size_t outleft = 2; - inbuf_[0] = i; - - iconv (t, 0, 0, 0, 0); /* reset iconv */ - - ret = iconv(t, &inbuf, &inleft, &outbuf, &outleft); - if (ret == (size_t) (-1)) - { - if (errno == EILSEQ) - { - yaz_log (ui->loglevel, "Encoding %d: invalid sequence", i); - info->map[i] = -1; /* invalid sequence */ - } - if (errno == EINVAL) - { /* multi byte input */ - int len = 2; - int j = 0; - info->map[i] = -1; - - while (len <= 4) - { - char sbuf[80]; - int k; - inbuf = inbuf_; - inleft = len; - outbuf = outbuf_; - outleft = 2; - - inbuf_[len-1] = j; - iconv (t, 0,0,0,0); - - assert (i >= 0 && i<255); - - *sbuf = 0; - for (k = 0; k 255) - break; - } - else if (errno == EINVAL) - { - len++; - j = 7; - } - } - else if (outleft == 0) - { - info->map[i] = -len; - info->data = t; /* signal that multibyte is in use */ - break; - } - else - { - break; - } - } - if (info->map[i] < -1) - yaz_log (ui->loglevel, "Encoding %d: multibyte input %d", - i, -info->map[i]); - else - yaz_log (ui->loglevel, "Encoding %d: multibyte input failed", - i); - } - if (errno == E2BIG) - { - info->map[i] = -1; /* no room for output */ - if (i != 0) - yaz_log (YLOG_WARN, "Encoding %d: no room for output", - i); - } - } - else if (outleft == 0) - { - unsigned short code; - memcpy (&code, outbuf_, sizeof(short)); - info->map[i] = code; - no_ok++; - } - else - { /* should never happen */ - info->map[i] = -1; - yaz_log (YLOG_DEBUG, "Encoding %d: bad state", i); - } - } - if (info->data) - { /* at least one multi byte */ - info->convert = cb_encoding_convert; - info->release = cb_encoding_release; - } - else - { - /* no multi byte - we no longer need iconv handler */ - iconv_close(t); - info->convert = 0; - info->release = 0; - } - if (!no_ok) - return 0; - return 1; -} -/* HAVE_ICONV_H */ -#endif - -static void cb_ns_start(void *userData, const char *prefix, const char *uri) -{ - struct user_info *ui = (struct user_info*) userData; - if (prefix && uri) - yaz_log(ui->loglevel, "cb_ns_start %s %s", prefix, uri); -} - -static void cb_ns_end(void *userData, const char *prefix) -{ - struct user_info *ui = (struct user_info*) userData; - if (prefix) - yaz_log(ui->loglevel, "cb_ns_end %s", prefix); -} - -data1_node *zebra_read_xml(data1_handle dh, - struct ZebraRecStream *stream, - NMEM m) -{ - XML_Parser parser; - struct user_info uinfo; - int done = 0; - data1_node *first_node; - int no_read = 0; - - uinfo.loglevel = YLOG_DEBUG; - uinfo.level = 1; - uinfo.dh = dh; - uinfo.nmem = m; - uinfo.d1_stack[0] = data1_mk_node2 (dh, m, DATA1N_root, 0); - uinfo.d1_stack[1] = 0; /* indicate no children (see end of routine) */ - - parser = XML_ParserCreate (0 /* encoding */); - - XML_SetElementHandler (parser, cb_start, cb_end); - XML_SetCharacterDataHandler (parser, cb_chardata); - XML_SetXmlDeclHandler (parser, cb_decl); - XML_SetProcessingInstructionHandler (parser, cb_processing); - XML_SetUserData (parser, &uinfo); - XML_SetCommentHandler (parser, cb_comment); - XML_SetDoctypeDeclHandler (parser, cb_doctype_start, cb_doctype_end); - XML_SetEntityDeclHandler (parser, cb_entity_decl); - XML_SetExternalEntityRefHandler (parser, cb_external_entity); - XML_SetNamespaceDeclHandler(parser, cb_ns_start, cb_ns_end); -#if HAVE_ICONV_H - XML_SetUnknownEncodingHandler (parser, cb_encoding_handler, &uinfo); -#endif - while (!done) - { - int r; - void *buf = XML_GetBuffer (parser, XML_CHUNK); - if (!buf) - { - /* error */ - yaz_log (YLOG_WARN, "XML_GetBuffer fail"); - break; - } - r = stream->readf(stream, buf, XML_CHUNK); - if (r < 0) - { - /* error */ - yaz_log (YLOG_WARN, "XML read fail"); - break; - } - else if (r == 0) - done = 1; - else - no_read += r; - if (no_read && !XML_ParseBuffer (parser, r, done)) - { - done = 1; - yaz_log (YLOG_WARN, "%d:%d:XML error: %s", - XML_GetCurrentLineNumber(parser), - XML_GetCurrentColumnNumber(parser), - XML_ErrorString(XML_GetErrorCode(parser))); - } - } - XML_ParserFree (parser); - if (no_read == 0) - return 0; - if (!uinfo.d1_stack[1] || !done) - return 0; - /* insert XML header if not present .. */ - first_node = uinfo.d1_stack[0]->child; - if (first_node->which != DATA1N_preprocess || - strcmp(first_node->u.preprocess.target, "xml")) - { - const char *attr_list[5]; - - attr_list[0] = "version"; - attr_list[1] = "1.0"; - - attr_list[2] = "encoding"; - attr_list[3] = "UTF-8"; /* encoding */ - - attr_list[4] = 0; - - data1_insert_preprocess (uinfo.dh, uinfo.nmem, "xml", attr_list, - uinfo.d1_stack[0]); - } - return uinfo.d1_stack[0]; -} - -struct xml_info { - XML_Expat_Version expat_version; -}; - -static data1_node *grs_read_xml(struct grs_read_info *p) -{ - return zebra_read_xml(p->dh, p->stream, p->mem); -} - -static void *filter_init(Res res, RecType recType) -{ - struct xml_info *p = (struct xml_info *) xmalloc (sizeof(*p)); - - p->expat_version = XML_ExpatVersionInfo(); - - return p; -} - -static void filter_destroy(void *clientData) -{ - struct xml_info *p = (struct xml_info *) clientData; - - xfree (p); -} - -static int filter_extract(void *clientData, struct recExtractCtrl *ctrl) -{ - return zebra_grs_extract(clientData, ctrl, grs_read_xml); -} - -static int filter_retrieve(void *clientData, struct recRetrieveCtrl *ctrl) -{ - return zebra_grs_retrieve(clientData, ctrl, grs_read_xml); -} - -static struct recType filter_type = { - 0, - "grs.xml", - filter_init, - 0, - filter_destroy, - filter_extract, - filter_retrieve, -}; - -RecType -#ifdef IDZEBRA_STATIC_GRS_XML -idzebra_filter_grs_xml -#else -idzebra_filter -#endif - -[] = { - &filter_type, - 0, -}; - -#endif - -/* - * Local variables: - * c-basic-offset: 4 - * indent-tabs-mode: nil - * End: - * vim: shiftwidth=4 tabstop=8 expandtab - */ - diff --git a/win/makefile b/win/makefile index caee566..40cb077 100644 --- a/win/makefile +++ b/win/makefile @@ -15,7 +15,7 @@ EXPAT_DIR=c:\Program files\Expat 2.0.1 HAVE_ICONV=1 ICONV_DIR=c:\iconv-1.9.2.win32 -# libxslt. For alvis +# libxslt. For mod_dom and mod_alvis HAVE_LIBXSLT=1 LIBXSLT_DIR=c:\libxslt-1.1.19.win32 @@ -215,7 +215,7 @@ icu: !endif !if $(HAVE_LIBXSLT) -MOD_ALVIS_OBJS= $(OBJDIR)\alvis.obj +MOD_ALVIS_OBJS= $(OBJDIR)\mod_alvis.obj MOD_ALVIS_CFLAGS=/DIDZEBRA_STATIC_ALVIS=1 MOD_DOM_OBJS= $(OBJDIR)\mod_dom.obj @@ -407,7 +407,7 @@ ZEBRALIB_OBJS= \ $(OBJDIR)\lookup.obj \ $(OBJDIR)\lookupec.obj \ $(OBJDIR)\marcomp.obj \ - $(OBJDIR)\marcread.obj \ + $(OBJDIR)\mod_grs_marc.obj \ $(OBJDIR)\merge.obj \ $(OBJDIR)\mfile.obj \ $(OBJDIR)\open.obj \ @@ -421,8 +421,8 @@ ZEBRALIB_OBJS= \ $(OBJDIR)\records.obj \ $(OBJDIR)\recindex.obj \ $(OBJDIR)\reckeys.obj \ - $(OBJDIR)\rectext.obj \ - $(OBJDIR)\regxread.obj \ + $(OBJDIR)\mod_text.obj \ + $(OBJDIR)\mod_grs_regx.obj \ $(OBJDIR)\res.obj \ $(OBJDIR)\retrieve.obj \ $(OBJDIR)\rpnscan.obj \ @@ -438,10 +438,10 @@ ZEBRALIB_OBJS= \ $(OBJDIR)\rsnull.obj \ $(OBJDIR)\rsprox.obj \ $(OBJDIR)\rstemp.obj \ - $(OBJDIR)\safari.obj \ + $(OBJDIR)\mod_safari.obj \ $(OBJDIR)\scan.obj \ $(OBJDIR)\set.obj \ - $(OBJDIR)\sgmlread.obj \ + $(OBJDIR)\mod_grs_sgml.obj \ $(OBJDIR)\snippet.obj \ $(OBJDIR)\sortidx.obj \ $(OBJDIR)\states.obj \ @@ -453,7 +453,7 @@ ZEBRALIB_OBJS= \ $(OBJDIR)\update_path.obj \ $(OBJDIR)\update_file.obj \ $(OBJDIR)\version.obj \ - $(OBJDIR)\xmlread.obj \ + $(OBJDIR)\mod_grs_xml.obj \ $(OBJDIR)\xpath.obj \ $(OBJDIR)\zaptterm.obj \ $(OBJDIR)\zebra-lock.obj \