X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fmod_dom.c;h=babee870da05c9b7138a6263bb60841624659141;hb=527dab66d9847bb7f8a931c558306a070064bf25;hp=30a77991d398082b27efdc77238524a96da4bdec;hpb=317fbe6637258762fd75aa72f75db406db9f2f9d;p=idzebra-moved-to-github.git diff --git a/index/mod_dom.c b/index/mod_dom.c index 30a7799..babee87 100644 --- a/index/mod_dom.c +++ b/index/mod_dom.c @@ -1,4 +1,4 @@ -/* $Id: mod_dom.c,v 1.22 2007-02-28 13:16:24 marc Exp $ +/* $Id: mod_dom.c,v 1.42 2007-12-13 19:59:21 adam Exp $ Copyright (C) 1995-2007 Index Data ApS @@ -43,6 +43,7 @@ #include #include +#include /* DOM filter style indexing */ #define ZEBRA_DOM_NS "http://indexdata.com/zebra-2.0" @@ -85,14 +86,14 @@ struct filter_input { int type; union { struct { + xmlTextReaderPtr reader; + int split_level; + } xmlreader; + struct { const char *input_charset; yaz_marc_t handle; yaz_iconv_t iconv; } marc; - struct { - xmlTextReaderPtr reader; - int split_level; - } xmlreader; } u; struct filter_input *next; }; @@ -108,6 +109,7 @@ struct filter_info { struct filter_retrieve *retrieve_list; struct filter_input *input_list; struct filter_store *store; + int record_info_invoked; }; @@ -183,6 +185,7 @@ static void *filter_init(Res res, RecType recType) tinfo->input_list = 0; tinfo->store = 0; tinfo->doc_config = 0; + tinfo->record_info_invoked = 0; #if YAZ_HAVE_EXSLT exsltRegisterAll(); @@ -331,6 +334,7 @@ static ZEBRA_RES parse_convert(struct filter_info *tinfo, xmlNodePtr ptr, } static ZEBRA_RES perform_convert(struct filter_info *tinfo, + struct recExtractCtrl *extctr, struct convert_s *convert, const char **params, xmlDocPtr *doc, @@ -338,12 +342,34 @@ static ZEBRA_RES perform_convert(struct filter_info *tinfo, { for (; convert; convert = convert->next) { + xmlChar *buf_out = 0; + int len_out = 0; xmlDocPtr res_doc = xsltApplyStylesheet(convert->stylesheet_xsp, *doc, params); if (last_xsp) *last_xsp = convert->stylesheet_xsp; + + if (!res_doc) + break; + + /* now saving into buffer and re-reading into DOM to avoid annoing + XSLT problem with thrown-out indentation text nodes */ + xsltSaveResultToString(&buf_out, &len_out, res_doc, + convert->stylesheet_xsp); + xmlFreeDoc(res_doc); + xmlFreeDoc(*doc); - *doc = res_doc; + + *doc = xmlParseMemory((const char *) buf_out, len_out); + + /* writing debug info out */ + if (extctr && extctr->flagShowRecords) + yaz_log(YLOG_LOG, "%s: XSLT %s\n %.*s", + tinfo->fname ? tinfo->fname : "(none)", + convert->stylesheet, + len_out, buf_out); + + xmlFree(buf_out); } return ZEBRA_OK; } @@ -605,6 +631,13 @@ static ZEBRA_RES parse_dom(struct filter_info *tinfo, const char *fname) return ZEBRA_FAIL; } } + if (!tinfo->input_list) + { + struct filter_input *p + = new_input(tinfo, DOM_INPUT_XMLREADER); + p->u.xmlreader.split_level = 0; + p->u.xmlreader.reader = 0; + } return ZEBRA_OK; } @@ -671,12 +704,12 @@ static int ioclose_ex(void *context) /* DOM filter style indexing */ static int attr_content_xml(struct _xmlAttr *attr, const char *name, - xmlChar **dst_content) + const char **dst_content) { if (0 == XML_STRCMP(attr->name, name) && attr->children && attr->children->type == XML_TEXT_NODE) { - *dst_content = (attr->children->content); + *dst_content = (const char *) (attr->children->content); return 1; } return 0; @@ -688,111 +721,134 @@ static void index_value_of(struct filter_info *tinfo, struct recExtractCtrl *extctr, RecWord* recword, xmlNodePtr node, - xmlChar * index_p) + const char *index_p) { - xmlChar *text = xmlNodeGetContent(node); - size_t text_len = strlen((const char *)text); - - /*dom_log(YLOG_DEBUG, tinfo, node, "Indexing: '%s' '%s'", index_p, text);*/ - - /* if there is no text, we do not need to proceed */ - if (text_len) - { - xmlChar *look = index_p; - xmlChar *bval; - xmlChar *eval; - - xmlChar index[256]; - xmlChar type[256]; + if (tinfo->record_info_invoked == 1) + { + xmlChar *text = xmlNodeGetContent(node); + size_t text_len = strlen((const char *)text); + + /* if there is no text, we do not need to proceed */ + if (text_len) + { + const char *look = index_p; + const char *bval; + const char *eval; + + xmlChar index[256]; + xmlChar type[256]; - /* assingning text to be indexed */ - recword->term_buf = (const char *)text; - recword->term_len = text_len; + /* assingning text to be indexed */ + recword->term_buf = (const char *)text; + recword->term_len = text_len; - /* parsing all index name/type pairs */ - /* may not start with ' ' or ':' */ - while (*look && ' ' != *look && ':' != *look) - { - /* setting name and type to zero */ - *index = '\0'; - *type = '\0'; - - /* parsing one index name */ - bval = look; - while (*look && ':' != *look && ' ' != *look) + /* parsing all index name/type pairs */ + /* may not start with ' ' or ':' */ + while (*look && ' ' != *look && ':' != *look) { - look++; - } - eval = look; - strncpy((char *)index, (const char *)bval, eval - bval); - index[eval - bval] = '\0'; - + /* setting name and type to zero */ + *index = '\0'; + *type = '\0'; - /* parsing one index type, if existing */ - if (':' == *look) - { - look++; - + /* parsing one index name */ bval = look; - while (*look && ' ' != *look) + while (*look && ':' != *look && ' ' != *look) { look++; } eval = look; - strncpy((char *)type, (const char *)bval, eval - bval); - type[eval - bval] = '\0'; - } + strncpy((char *)index, (const char *)bval, eval - bval); + index[eval - bval] = '\0'; + + + /* parsing one index type, if existing */ + if (':' == *look) + { + look++; + + bval = look; + while (*look && ' ' != *look) + { + look++; + } + eval = look; + strncpy((char *)type, (const char *)bval, eval - bval); + type[eval - bval] = '\0'; + } - /* actually indexing the text given */ - dom_log(YLOG_DEBUG, tinfo, 0, - "INDEX '%s:%s' '%s'", - index ? (const char *) index : "null", - type ? (const char *) type : "null", - text ? (const char *) text : "null"); + /* actually indexing the text given */ - recword->index_name = (const char *)index; - if (type && *type) - recword->index_type = *type; - (extctr->tokenAdd)(recword); + recword->index_name = (const char *)index; + if (type && *type) + recword->index_type = (const char *) type; - /* eat whitespaces */ - if (*look && ' ' == *look && *(look+1)) - { - look++; - } + /* writing debug out */ + if (extctr->flagShowRecords) + dom_log(YLOG_LOG, tinfo, 0, + "INDEX '%s:%s' '%s'", + index ? (const char *) index : "null", + type ? (const char *) type : "null", + text ? (const char *) text : "null"); + + (extctr->tokenAdd)(recword); + + /* eat whitespaces */ + if (*look && ' ' == *look) + { + look++; + } + } } + xmlFree(text); } - - xmlFree(text); } /* DOM filter style indexing */ static void set_record_info(struct filter_info *tinfo, struct recExtractCtrl *extctr, - xmlChar * id_p, - xmlChar * rank_p, - xmlChar * type_p) + xmlNodePtr node, + const char * id_p, + const char * rank_p, + const char * type_p) { - dom_log(YLOG_DEBUG, tinfo, 0, - "RECORD id=%s rank=%s type=%s", - id_p ? (const char *) id_p : "null", - rank_p ? (const char *) rank_p : "null", - type_p ? (const char *) type_p : "null"); + /* writing debug info out */ + if (extctr && extctr->flagShowRecords) + dom_log(YLOG_LOG, tinfo, node, + "RECORD id=%s rank=%s type=%s", + id_p ? (const char *) id_p : "(null)", + rank_p ? (const char *) rank_p : "(null)", + type_p ? (const char *) type_p : "(null)"); - if (id_p) + + if (id_p && *id_p) sscanf((const char *)id_p, "%255s", extctr->match_criteria); - if (rank_p) + if (rank_p && *rank_p) extctr->staticrank = atozint((const char *)rank_p); - /* if (!strcmp("update", type_str)) */ - /* index_node(tinfo, ctrl, ptr, recword); */ - /* else if (!strcmp("delete", type_str)) */ - /* dom_log(YLOG_WARN, tinfo, ptr, "dom filter delete: to be implemented"); */ - /* else */ - /* dom_log(YLOG_WARN, tinfo, ptr, "dom filter: unknown record type '%s'", */ - /* type_str); */ + if (type_p && *type_p) + { + enum zebra_recctrl_action_t action = action_update; + if (!strcmp(type_p, "insert")) + action = action_insert; + else if (!strcmp(type_p, "delete")) + action = action_delete; + else if (!strcmp(type_p, "replace")) + action = action_replace; + else if (!strcmp(type_p, "update")) + action = action_update; + else + dom_log(YLOG_WARN, tinfo, node, "bad @type value: %s", type_p); + extctr->action = action; + } + + if (tinfo->record_info_invoked == 1) + { + /* warn about multiple only once */ + dom_log(YLOG_WARN, tinfo, node, "multiple record elements"); + } + tinfo->record_info_invoked++; } @@ -808,14 +864,14 @@ static void process_xml_element_zebra_node(struct filter_info *tinfo, { if (0 == XML_STRCMP(node->name, "index")) { - xmlChar *index_p = 0; + const char *index_p = 0; struct _xmlAttr *attr; for (attr = node->properties; attr; attr = attr->next) { if (attr_content_xml(attr, "name", &index_p)) { - index_value_of(tinfo, extctr, recword,node, index_p); + index_value_of(tinfo, extctr, recword, node, index_p); } else { @@ -827,9 +883,9 @@ static void process_xml_element_zebra_node(struct filter_info *tinfo, } else if (0 == XML_STRCMP(node->name, "record")) { - xmlChar *id_p = 0; - xmlChar *rank_p = 0; - xmlChar *type_p = 0; + const char *id_p = 0; + const char *rank_p = 0; + const char *type_p = 0; struct _xmlAttr *attr; for (attr = node->properties; attr; attr = attr->next) @@ -846,15 +902,8 @@ static void process_xml_element_zebra_node(struct filter_info *tinfo, "bad attribute @%s, expected @id|@rank|@type", attr->name); } - - if (type_p && 0 != strcmp("update", (const char *)type_p)) - { - dom_log(YLOG_WARN, tinfo, node, - "attribute @%s, only implemented '@type='update'", - attr->name); - } } - set_record_info(tinfo, extctr, id_p, rank_p, type_p); + set_record_info(tinfo, extctr, node, id_p, rank_p, type_p); } else { @@ -866,80 +915,74 @@ static void process_xml_element_zebra_node(struct filter_info *tinfo, } } +static int attr_content_pi(const char **c_ptr, const char *name, + char *value, size_t value_max) +{ + size_t name_len = strlen(name); + const char *look = *c_ptr; + int ret = 0; + + *value = '\0'; + while (*look && ' ' == *look) + look++; + if (strlen(look) > name_len) + { + if (look[name_len] == '=' && !memcmp(look, name, name_len)) + { + size_t i = 0; + look += name_len+1; + while (*look && ' ' != *look) + { + if (i < value_max-1) + value[i++] = *look; + look++; + } + value[i] = '\0'; + ret = 1; + } + } + while (*look && ' ' == *look) + look++; + *c_ptr = look; + return ret; +} /* DOM filter style indexing */ static void process_xml_pi_node(struct filter_info *tinfo, struct recExtractCtrl *extctr, xmlNodePtr node, - xmlChar **index_pp) + const char **index_pp) { /* if right PI name, continue parsing PI */ if (0 == strcmp(zebra_pi_name, (const char *)node->name)) { xmlChar *pi_p = node->content; - xmlChar *look = pi_p; + const char *look = (const char *) node->content; - xmlChar *bval; - xmlChar *eval; - /* parsing PI record instructions */ if (0 == strncmp((const char *)look, "record", 6)) { - xmlChar id[256]; - xmlChar rank[256]; - xmlChar type[256]; - + char id[256]; + char rank[256]; + char type[256]; + *id = '\0'; *rank = '\0'; *type = '\0'; - look += 6; - - /* eat whitespace */ - while (*look && ' ' == *look && *(look+1)) - look++; - - /* parse possible id */ - if (*look && 0 == strncmp((const char *)look, "id=", 3)) - { - look += 3; - bval = look; - while (*look && ' ' != *look) - look++; - eval = look; - strncpy((char *)id, (const char *)bval, eval - bval); - id[eval - bval] = '\0'; - } - - /* eat whitespace */ - while (*look && ' ' == *look && *(look+1)) - look++; - - /* parse possible rank */ - if (*look && 0 == strncmp((const char *)look, "rank=", 5)) - { - look += 6; - bval = look; - while (*look && ' ' != *look) - look++; - eval = look; - strncpy((char *)rank, (const char *)bval, eval - bval); - rank[eval - bval] = '\0'; - } - - /* eat whitespace */ - while (*look && ' ' == *look && *(look+1)) - look++; - - if (look && '\0' != *look) - { - dom_log(YLOG_WARN, tinfo, node, - "content '%s', can not parse '%s'", - pi_p, look); - } - else - set_record_info(tinfo, extctr, id, rank, 0); - + while (*look) + if (attr_content_pi(&look, "id", id, sizeof(id))) + ; + else if (attr_content_pi(&look, "rank", rank, sizeof(rank))) + ; + else if (attr_content_pi(&look, "type", type, sizeof(type))) + { + dom_log(YLOG_WARN, tinfo, node, + "content '%s', can not parse '%s'", + pi_p, look); + break; + } + set_record_info(tinfo, extctr, node, id, rank, type); } /* parsing index instruction */ else if (0 == strncmp((const char *)look, "index", 5)) @@ -947,7 +990,7 @@ static void process_xml_pi_node(struct filter_info *tinfo, look += 5; /* eat whitespace */ - while (*look && ' ' == *look && *(look+1)) + while (*look && ' ' == *look) look++; /* export index instructions to outside */ @@ -969,7 +1012,7 @@ static void process_xml_element_node(struct filter_info *tinfo, xmlNodePtr node) { /* remember indexing instruction from PI to next element node */ - xmlChar *index_p = 0; + const char *index_p = 0; /* check if we are an element node in the special zebra namespace and either set record data or index value-of node content*/ @@ -1004,20 +1047,10 @@ static void extract_dom_doc_node(struct filter_info *tinfo, struct recExtractCtrl *extctr, xmlDocPtr doc) { - xmlChar *buf_out; - int len_out; - /* only need to do the initialization once, reuse recword for all terms */ RecWord recword; (*extctr->init)(extctr, &recword); - if (extctr->flagShowRecords) - { - xmlDocDumpMemory(doc, &buf_out, &len_out); - fwrite(buf_out, len_out, 1, stdout); - xmlFree(buf_out); - } - process_xml_element_node(tinfo, extctr, &recword, (xmlNodePtr)doc); } @@ -1036,43 +1069,71 @@ static int convert_extract_doc(struct filter_info *tinfo, xsltStylesheetPtr last_xsp = 0; xmlDocPtr store_doc = 0; + /* per default do not ingest record */ + tinfo->record_info_invoked = 0; + + /* exit if empty document given */ + if (!doc) + return RECCTRL_EXTRACT_SKIP; + + /* we actuallu have a document which needs to be processed further */ params[0] = 0; set_param_str(params, "schema", zebra_dom_ns, tinfo->odr_record); + if (p && p->flagShowRecords) + { + xmlChar *buf_out; + int len_out; +#if 0 + FILE *outf = fopen("extract.xml", "w"); + xmlDocDumpMemory(doc, &buf_out, &len_out); + fwrite(buf_out, 1, len_out, outf); +#endif + yaz_log(YLOG_LOG, "Extract Doc: %.*s", len_out, buf_out); +#if 0 + fclose(outf); +#endif + } + /* input conversion */ - perform_convert(tinfo, input->convert, params, &doc, 0); + perform_convert(tinfo, p, input->convert, params, &doc, 0); + if (tinfo->store) { /* store conversion */ store_doc = xmlCopyDoc(doc, 1); - perform_convert(tinfo, tinfo->store->convert, + perform_convert(tinfo, p, tinfo->store->convert, params, &store_doc, &last_xsp); } + /* saving either store doc or original doc in case no store doc exists */ if (last_xsp) xsltSaveResultToString(&buf_out, &len_out, store_doc ? store_doc : doc, last_xsp); else xmlDocDumpMemory(store_doc ? store_doc : doc, &buf_out, &len_out); - if (p->flagShowRecords) - fwrite(buf_out, len_out, 1, stdout); - (*p->setStoreData)(p, buf_out, len_out); + + if (p->setStoreData) + (*p->setStoreData)(p, buf_out, len_out); xmlFree(buf_out); if (store_doc) xmlFreeDoc(store_doc); /* extract conversion */ - perform_convert(tinfo, tinfo->extract->convert, params, &doc, 0); + perform_convert(tinfo, p, tinfo->extract->convert, params, &doc, 0); + /* finally, do the indexing */ - if (doc) - { + if (doc){ extract_dom_doc_node(tinfo, p, doc); - /* extract_doc_alvis(tinfo, p, doc); */ xmlFreeDoc(doc); } + + /* there was nothing to index, so there is no inserted/updated record */ + if (tinfo->record_info_invoked == 0) + return RECCTRL_EXTRACT_SKIP; return RECCTRL_EXTRACT_OK; } @@ -1091,8 +1152,9 @@ static int extract_xml_split(struct filter_info *tinfo, p /* I/O handler */, 0 /* URL */, 0 /* encoding */, - XML_PARSE_XINCLUDE| - XML_PARSE_NOENT); + XML_PARSE_XINCLUDE + | XML_PARSE_NOENT + | XML_PARSE_NONET); } if (!input->u.xmlreader.reader) return RECCTRL_EXTRACT_ERROR_GENERIC; @@ -1102,18 +1164,37 @@ static int extract_xml_split(struct filter_info *tinfo, { int type = xmlTextReaderNodeType(input->u.xmlreader.reader); int depth = xmlTextReaderDepth(input->u.xmlreader.reader); + if (type == XML_READER_TYPE_ELEMENT && input->u.xmlreader.split_level == depth) { - xmlNodePtr ptr - = xmlTextReaderExpand(input->u.xmlreader.reader); + xmlNodePtr ptr; + + /* per default do not ingest record */ + tinfo->record_info_invoked = 0; + + ptr = xmlTextReaderExpand(input->u.xmlreader.reader); if (ptr) - { + { + /* we have a new document */ + xmlNodePtr ptr2 = xmlCopyNode(ptr, 1); xmlDocPtr doc = xmlNewDoc((const xmlChar*) "1.0"); xmlDocSetRootElement(doc, ptr2); + /* writing debug info out */ + if (p->flagShowRecords) + { + xmlChar *buf_out = 0; + int len_out = 0; + xmlDocDumpMemory(doc, &buf_out, &len_out); + yaz_log(YLOG_LOG, "%s: XMLREADER level: %i\n%.*s", + tinfo->fname ? tinfo->fname : "(none)", + depth, len_out, buf_out); + xmlFree(buf_out); + } + return convert_extract_doc(tinfo, input, p, doc); } else @@ -1140,7 +1221,9 @@ static int extract_xml_full(struct filter_info *tinfo, p /* I/O handler */, 0 /* URL */, 0 /* encoding */, - XML_PARSE_XINCLUDE|XML_PARSE_NOENT); + XML_PARSE_XINCLUDE + | XML_PARSE_NOENT + | XML_PARSE_NONET); if (!doc) { return RECCTRL_EXTRACT_ERROR_GENERIC; @@ -1200,7 +1283,8 @@ static int extract_iso2709(struct filter_info *tinfo, { xmlDocPtr rdoc; xmlNode *root_ptr; - yaz_marc_write_xml(input->u.marc.handle, &root_ptr, 0, 0, 0); + yaz_marc_write_xml(input->u.marc.handle, &root_ptr, + "http://www.loc.gov/MARC21/slim", 0, 0); rdoc = xmlNewDoc((const xmlChar*) "1.0"); xmlDocSetRootElement(rdoc, root_ptr); return convert_extract_doc(tinfo, input, p, rdoc); @@ -1215,8 +1299,11 @@ static int filter_extract(void *clientData, struct recExtractCtrl *p) if (!input) return RECCTRL_EXTRACT_ERROR_GENERIC; - + odr_reset(tinfo->odr_record); + + if (p->setStoreData == 0) + return extract_xml_full(tinfo, input, p); switch(input->type) { case DOM_INPUT_XMLREADER: @@ -1272,6 +1359,7 @@ static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p) { p->diagnostic = YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_; + p->addinfo = odr_strdup(p->odr, esn); return 0; } @@ -1299,7 +1387,7 @@ static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p) doc = xmlReadIO(ioread_ret, ioclose_ret, p /* I/O handler */, 0 /* URL */, 0 /* encoding */, - XML_PARSE_XINCLUDE|XML_PARSE_NOENT); + XML_PARSE_XINCLUDE | XML_PARSE_NOENT | XML_PARSE_NONET); if (!doc) { p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS; @@ -1307,12 +1395,13 @@ static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p) } /* retrieve conversion */ - perform_convert(tinfo, retrieve->convert, params, &doc, &last_xsp); + perform_convert(tinfo, 0, retrieve->convert, params, &doc, &last_xsp); if (!doc) { p->diagnostic = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS; } - else if (p->input_format == VAL_NONE || p->input_format == VAL_TEXT_XML) + else if (!p->input_format + || !oid_oidcmp(p->input_format, yaz_oid_recsyn_xml)) { xmlChar *buf_out; int len_out; @@ -1322,13 +1411,13 @@ static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p) else xmlDocDumpMemory(doc, &buf_out, &len_out); - p->output_format = VAL_TEXT_XML; + p->output_format = yaz_oid_recsyn_xml; p->rec_len = len_out; p->rec_buf = odr_malloc(p->odr, p->rec_len); memcpy(p->rec_buf, buf_out, p->rec_len); xmlFree(buf_out); } - else if (p->output_format == VAL_SUTRS) + else if (!oid_oidcmp(p->output_format, yaz_oid_recsyn_sutrs)) { xmlChar *buf_out; int len_out; @@ -1338,7 +1427,7 @@ static int filter_retrieve (void *clientData, struct recRetrieveCtrl *p) else xmlDocDumpMemory(doc, &buf_out, &len_out); - p->output_format = VAL_SUTRS; + p->output_format = yaz_oid_recsyn_sutrs; p->rec_len = len_out; p->rec_buf = odr_malloc(p->odr, p->rec_len); memcpy(p->rec_buf, buf_out, p->rec_len);