X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Flogic.c;h=0e7780402e48378eb513b8fc4355d1791debb840;hb=15790e226ac62ed0813c89ba29fcd9b3d564fe19;hp=625adc056f8eba6c57d949c87f856b218c6d437d;hpb=56a55ffd1562165febc1a2a66d0c2a95e966d540;p=pazpar2-moved-to-github.git diff --git a/src/logic.c b/src/logic.c index 625adc0..0e77804 100644 --- a/src/logic.c +++ b/src/logic.c @@ -1,35 +1,39 @@ -/* $Id: logic.c,v 1.69 2007-10-02 12:11:14 adam Exp $ - Copyright (c) 2006-2007, Index Data. +/* This file is part of Pazpar2. + Copyright (C) 2006-2009 Index Data - This file is part of Pazpar2. +Pazpar2 is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. - Pazpar2 is free software; you can redistribute it and/or modify it under - the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2, or (at your option) any later - version. +Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. - Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - You should have received a copy of the GNU General Public License - along with Pazpar2; see the file LICENSE. If not, write to the - Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA - 02111-1307, USA. */ /** \file logic.c \brief high-level logic; mostly user sessions and settings */ +#if HAVE_CONFIG_H +#include +#endif + #include #include #include +#if HAVE_SYS_TIME_H #include +#endif +#if HAVE_UNISTD_H #include -#include -#include +#endif #include #include #include @@ -48,16 +52,11 @@ #include #include -#if HAVE_CONFIG_H -#include "cconfig.h" -#endif - #define USE_TIMING 0 #if USE_TIMING #include #endif -#include #include "pazpar2.h" #include "eventl.h" @@ -65,7 +64,6 @@ #include "termlists.h" #include "reclists.h" #include "relevance.h" -#include "config.h" #include "database.h" #include "client.h" #include "settings.h" @@ -78,25 +76,33 @@ // Note: Some things in this structure will eventually move to configuration struct parameters global_parameters = { - "", - "", - "", - 0, 0, // dump_records 0, // debug_mode 30, // operations timeout - "81", - "Index Data PazPar2", - VERSION, 60, // session timeout 100, - MAX_CHUNK, - 0, - 0, 180, // Z39.50 session timeout 15 // Connect timeout }; +static void log_xml_doc(xmlDoc *doc) +{ + FILE *lf = yaz_log_file(); + xmlChar *result = 0; + int len = 0; +#if LIBXML_VERSION >= 20600 + xmlDocDumpFormatMemory(doc, &result, &len, 1); +#else + xmlDocDumpMemory(doc, &result, &len); +#endif + if (lf && len) + { + fwrite(result, 1, len, lf); + fprintf(lf, "\n"); + } + xmlFree(result); +} + // Recursively traverse query structure to extract terms. void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) { @@ -151,123 +157,26 @@ static void add_facet(struct session *s, const char *type, const char *value) termlist_insert(s->termlists[i].termlist, value); } -xmlDoc *record_to_xml(struct session_database *sdb, Z_External *rec) +xmlDoc *record_to_xml(struct session_database *sdb, const char *rec) { struct database *db = sdb->database; xmlDoc *rdoc = 0; - const Odr_oid *oid = rec->direct_reference; - /* convert response record to XML somehow */ - if (rec->which == Z_External_octet && oid - && !oid_oidcmp(oid, yaz_oid_recsyn_xml)) - { - /* xml already */ - rdoc = xmlParseMemory((char*) rec->u.octet_aligned->buf, - rec->u.octet_aligned->len); - if (!rdoc) - { - yaz_log(YLOG_FATAL, "Non-wellformed XML received from %s", - db->url); - return 0; - } - } - else if (rec->which == Z_External_OPAC) - { - if (!sdb->yaz_marc) - { - yaz_log(YLOG_WARN, "MARC decoding not configured"); - return 0; - } - else - { - /* OPAC gets converted to XML too */ - WRBUF wrbuf_opac = wrbuf_alloc(); - /* MARCXML inside the OPAC XML. Charset is in effect because we - use the yaz_marc handle */ - yaz_marc_xml(sdb->yaz_marc, YAZ_MARC_MARCXML); - yaz_opac_decode_wrbuf(sdb->yaz_marc, rec->u.opac, wrbuf_opac); - - rdoc = xmlParseMemory((char*) wrbuf_buf(wrbuf_opac), - wrbuf_len(wrbuf_opac)); - if (!rdoc) - { - yaz_log(YLOG_WARN, "Unable to parse OPAC XML"); - /* Was used to debug bug #1348 */ -#if 0 - FILE *f = fopen("/tmp/opac.xml.txt", "wb"); - if (f) - { - fwrite(wrbuf_buf(wrbuf_opac), 1, wrbuf_len(wrbuf_opac), f); - fclose(f); - } -#endif - } - wrbuf_destroy(wrbuf_opac); - } - } - else if (oid && yaz_oid_is_iso2709(oid)) - { - /* ISO2709 gets converted to MARCXML */ - if (!sdb->yaz_marc) - { - yaz_log(YLOG_WARN, "MARC decoding not configured"); - return 0; - } - else - { - xmlNode *res; - char *buf; - int len; - - if (rec->which != Z_External_octet) - { - yaz_log(YLOG_WARN, "Unexpected external branch, probably BER %s", - db->url); - return 0; - } - buf = (char*) rec->u.octet_aligned->buf; - len = rec->u.octet_aligned->len; - if (yaz_marc_read_iso2709(sdb->yaz_marc, buf, len) < 0) - { - yaz_log(YLOG_WARN, "Failed to decode MARC %s", db->url); - return 0; - } - - if (yaz_marc_write_xml(sdb->yaz_marc, &res, - "http://www.loc.gov/MARC21/slim", 0, 0) < 0) - { - yaz_log(YLOG_WARN, "Failed to encode as XML %s", - db->url); - return 0; - } - rdoc = xmlNewDoc((xmlChar *) "1.0"); - xmlDocSetRootElement(rdoc, res); - } - } - else + rdoc = xmlParseMemory(rec, strlen(rec)); + + if (!rdoc) { - char oid_name_buf[OID_STR_MAX]; - const char *oid_name = yaz_oid_to_string_buf(oid, 0, oid_name_buf); - yaz_log(YLOG_FATAL, - "Unable to handle record of type %s from %s", - oid_name, db->url); + yaz_log(YLOG_FATAL, "Non-wellformed XML received from %s", + db->url); return 0; } - + if (global_parameters.dump_records) { - FILE *lf = yaz_log_file(); - if (lf) - { - yaz_log(YLOG_LOG, "Un-normalized record from %s", db->url); -#if LIBXML_VERSION >= 20600 - xmlDocFormatDump(lf, rdoc, 1); -#else - xmlDocDump(lf, rdoc); -#endif - fprintf(lf, "\n"); - } + yaz_log(YLOG_LOG, "Un-normalized record from %s", db->url); + log_xml_doc(rdoc); } + return rdoc; } @@ -277,7 +186,7 @@ xmlDoc *record_to_xml(struct session_database *sdb, Z_External *rec) static void insert_settings_parameters(struct session_database *sdb, struct session *se, char **parms) { - struct conf_service *service = global_parameters.server->service; + struct conf_service *service = se->service; int i; int nparms = 0; int offset = 0; @@ -288,7 +197,7 @@ static void insert_settings_parameters(struct session_database *sdb, int setting; if (md->setting == Metadata_setting_parameter && - (setting = settings_offset(md->name)) > 0) + (setting = settings_offset(service, md->name)) > 0) { const char *val = session_setting_oneval(sdb, setting); if (val && nparms < MAX_XSLT_ARGS) @@ -310,9 +219,9 @@ static void insert_settings_parameters(struct session_database *sdb, } // Add static values from session database settings if applicable -static void insert_settings_values(struct session_database *sdb, xmlDoc *doc) +static void insert_settings_values(struct session_database *sdb, xmlDoc *doc, + struct conf_service *service) { - struct conf_service *service = global_parameters.server->service; int i; for (i = 0; i < service->num_metadata; i++) @@ -321,7 +230,7 @@ static void insert_settings_values(struct session_database *sdb, xmlDoc *doc) int offset; if (md->setting == Metadata_setting_postproc && - (offset = settings_offset(md->name)) > 0) + (offset = settings_offset(service, md->name)) > 0) { const char *val = session_setting_oneval(sdb, offset); if (val) @@ -336,7 +245,7 @@ static void insert_settings_values(struct session_database *sdb, xmlDoc *doc) } xmlDoc *normalize_record(struct session_database *sdb, struct session *se, - Z_External *rec) + const char *rec) { struct database_retrievalmap *m; xmlDoc *rdoc = record_to_xml(sdb, rec); @@ -368,23 +277,13 @@ xmlDoc *normalize_record(struct session_database *sdb, struct session *se, rdoc = new; } - insert_settings_values(sdb, rdoc); + insert_settings_values(sdb, rdoc, se->service); if (global_parameters.dump_records) { - FILE *lf = yaz_log_file(); - - if (lf) - { - yaz_log(YLOG_LOG, "Normalized record from %s", - sdb->database->url); -#if LIBXML_VERSION >= 20600 - xmlDocFormatDump(lf, rdoc, 1); -#else - xmlDocDump(lf, rdoc); -#endif - fprintf(lf, "\n"); - } + yaz_log(YLOG_LOG, "Normalized record from %s", + sdb->database->url); + log_xml_doc(rdoc); } } return rdoc; @@ -399,44 +298,6 @@ const char *session_setting_oneval(struct session_database *db, int offset) return db->settings[offset]->value; } - - -// Initialize YAZ Map structures for MARC-based targets -static int prepare_yazmarc(struct session_database *sdb) -{ - const char *s; - - if (!sdb->settings) - { - yaz_log(YLOG_WARN, "No settings for %s", sdb->database->url); - return -1; - } - if ((s = session_setting_oneval(sdb, PZ_NATIVESYNTAX)) - && !strncmp(s, "iso2709", 7)) - { - char *encoding = "marc-8s", *e; - yaz_iconv_t cm; - - // See if a native encoding is specified - if ((e = strchr(s, ';'))) - encoding = e + 1; - - sdb->yaz_marc = yaz_marc_create(); - yaz_marc_subfield_str(sdb->yaz_marc, "\t"); - - cm = yaz_iconv_open("utf-8", encoding); - if (!cm) - { - yaz_log(YLOG_FATAL, - "Unable to map from %s to UTF-8 for target %s", - encoding, sdb->database->url); - return -1; - } - yaz_marc_iconv(sdb->yaz_marc, cm); - } - return 0; -} - // Prepare XSLT stylesheets for record normalization // Structures are allocated on the session_wide nmem to avoid having // to recompute this for every search. This would lead @@ -485,7 +346,8 @@ static int prepare_map(struct session *se, struct session_database *sdb) { (*m) = nmem_malloc(se->session_nmem, sizeof(**m)); (*m)->next = 0; - if (!((*m)->stylesheet = conf_load_stylesheet(stylesheets[i]))) + if (!((*m)->stylesheet = conf_load_stylesheet(se->service->config, + stylesheets[i]))) { yaz_log(YLOG_FATAL|YLOG_ERRNO, "Unable to load stylesheet: %s", stylesheets[i]); @@ -511,11 +373,6 @@ static int prepare_session_database(struct session *se, "No settings associated with %s", sdb->database->url); return -1; } - if (sdb->settings[PZ_NATIVESYNTAX] && !sdb->yaz_marc) - { - if (prepare_yazmarc(sdb) < 0) - return -1; - } if (sdb->settings[PZ_XSLT] && !sdb->map) { if (prepare_map(se, sdb) < 0) @@ -554,9 +411,12 @@ void session_alert_watch(struct session *s, int what) if (s->watchlist[what].fun) { /* our watch is no longer associated with http_channel */ + void *data; + session_watchfun fun; + http_remove_observer(s->watchlist[what].obs); - session_watchfun fun = s->watchlist[what].fun; - void *data = s->watchlist[what].data; + fun = s->watchlist[what].fun; + data = s->watchlist[what].data; /* reset watch before fun is invoked - in case fun wants to set it again */ @@ -660,11 +520,10 @@ enum pazpar2_error_code search(struct session *se, se->reclist = 0; se->num_termlists = 0; criteria = parse_filter(se->nmem, filter); - se->requestid++; live_channels = select_targets(se, criteria); if (live_channels) { - int maxrecs = live_channels * global_parameters.toget; + int maxrecs = live_channels * global_parameters.toget; // This is buggy!!! se->reclist = reclist_create(se->nmem, maxrecs); se->expected_maxrecs = maxrecs; } @@ -684,7 +543,8 @@ enum pazpar2_error_code search(struct session *se, else { no_working++; - client_prep_connection(cl); + if (client_prep_connection(cl)) + client_start_search(cl); } } @@ -701,12 +561,12 @@ enum pazpar2_error_code search(struct session *se, static void session_init_databases_fun(void *context, struct database *db) { struct session *se = (struct session *) context; + struct conf_service *service = se->service; struct session_database *new = nmem_malloc(se->session_nmem, sizeof(*new)); - int num = settings_num(); + int num = settings_num(service); int i; new->database = db; - new->yaz_marc = 0; new->map = 0; new->settings @@ -729,8 +589,6 @@ static void session_database_destroy(struct session_database *sdb) for (m = sdb->map; m; m = m->next) xsltFreeStylesheet(m->stylesheet); - if (sdb->yaz_marc) - yaz_marc_destroy(sdb->yaz_marc); } // Initialize session_database list -- this represents this session's view @@ -738,7 +596,7 @@ static void session_database_destroy(struct session_database *sdb) void session_init_databases(struct session *se) { se->databases = 0; - predef_grep_databases(se, 0, session_init_databases_fun); + predef_grep_databases(se, se->service, 0, session_init_databases_fun); } // Probably session_init_databases_fun should be refactored instead of @@ -746,7 +604,7 @@ void session_init_databases(struct session *se) static struct session_database *load_session_database(struct session *se, char *id) { - struct database *db = find_database(id, 0); + struct database *db = find_database(id, 0, se->service); session_init_databases_fun((void*) se, db); // New sdb is head of se->databases list @@ -770,8 +628,9 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, char *value) { struct session_database *sdb = find_session_database(se, dbname); + struct conf_service *service = se->service; struct setting *new = nmem_malloc(se->session_nmem, sizeof(*new)); - int offset = settings_offset_cprefix(setting); + int offset = settings_offset_cprefix(service, setting); if (offset < 0) { @@ -795,13 +654,6 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, // (happens when a search starts and client connections are prepared) switch (offset) { - case PZ_NATIVESYNTAX: - if (sdb->yaz_marc) - { - yaz_marc_destroy(sdb->yaz_marc); - sdb->yaz_marc = 0; - } - break; case PZ_XSLT: if (sdb->map) { @@ -827,19 +679,21 @@ void destroy_session(struct session *s) wrbuf_destroy(s->wrbuf); } -struct session *new_session(NMEM nmem) +struct session *new_session(NMEM nmem, struct conf_service *service) { int i; struct session *session = nmem_malloc(nmem, sizeof(*session)); yaz_log(YLOG_DEBUG, "New Pazpar2 session"); - + + session->service = service; session->relevance = 0; session->total_hits = 0; session->total_records = 0; + session->number_of_warnings_unknown_elements = 0; + session->number_of_warnings_unknown_metadata = 0; session->num_termlists = 0; session->reclist = 0; - session->requestid = -1; session->clients = 0; session->expected_maxrecs = 0; session->session_nmem = nmem; @@ -905,14 +759,24 @@ void report_nmem_stats(void) } #endif -struct record_cluster *show_single(struct session *s, const char *id) +struct record_cluster *show_single(struct session *s, const char *id, + struct record_cluster **prev_r, + struct record_cluster **next_r) { struct record_cluster *r; reclist_rewind(s->reclist); + *prev_r = 0; + *next_r = 0; while ((r = reclist_read_record(s->reclist))) + { if (!strcmp(r->recid, id)) + { + *next_r = reclist_read_record(s->reclist); return r; + } + *prev_r = r; + } return 0; } @@ -990,9 +854,7 @@ void statistics(struct session *se, struct statistics *stat) switch (client_get_state(cl)) { case Client_Connecting: stat->num_connecting++; break; - case Client_Initializing: stat->num_initializing++; break; - case Client_Searching: stat->num_searching++; break; - case Client_Presenting: stat->num_presenting++; break; + case Client_Working: stat->num_working++; break; case Client_Idle: stat->num_idle++; break; case Client_Failed: stat->num_failed++; break; case Client_Error: stat->num_error++; break; @@ -1006,52 +868,10 @@ void statistics(struct session *se, struct statistics *stat) stat->num_clients = count; } -void start_http_listener(void) -{ - char hp[128] = ""; - struct conf_server *ser = global_parameters.server; - - if (*global_parameters.listener_override) - strcpy(hp, global_parameters.listener_override); - else - { - strcpy(hp, ser->host ? ser->host : ""); - if (ser->port) - { - if (*hp) - strcat(hp, ":"); - sprintf(hp + strlen(hp), "%d", ser->port); - } - } - http_init(hp); -} - -void start_proxy(void) -{ - char hp[128] = ""; - struct conf_server *ser = global_parameters.server; - - if (*global_parameters.proxy_override) - strcpy(hp, global_parameters.proxy_override); - else if (ser->proxy_host || ser->proxy_port) - { - strcpy(hp, ser->proxy_host ? ser->proxy_host : ""); - if (ser->proxy_port) - { - if (*hp) - strcat(hp, ":"); - sprintf(hp + strlen(hp), "%d", ser->proxy_port); - } - } - else - return; - - http_set_proxyaddr(hp, ser->myurl ? ser->myurl : ""); -} - // Master list of connections we're handling events to -static IOCHAN channel_list = 0; +static IOCHAN channel_list = 0; /* thread pr */ + void pazpar2_add_channel(IOCHAN chan) { chan->next = channel_list; @@ -1075,11 +895,16 @@ static struct record_metadata *record_metadata_init( rec_md->data.text.disp = nmem_strdup(nmem, p); rec_md->data.text.sort = 0; } - else if (type == Metadata_type_year) + else if (type == Metadata_type_year || type == Metadata_type_date) { int first, last; - if (extract7bit_years((char *) value, &first, &last) < 0) + int longdate = 0; + + if (type == Metadata_type_date) + longdate = 1; + if (extract7bit_dates((char *) value, &first, &last, longdate) < 0) return 0; + rec_md->data.number.min = first; rec_md->data.number.max = last; } @@ -1088,7 +913,107 @@ static struct record_metadata *record_metadata_init( return rec_md; } -struct record *ingest_record(struct client *cl, Z_External *rec, +static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, + struct conf_service *service, NMEM nmem) +{ + char *mergekey_norm = 0; + xmlNode *root = xmlDocGetRootElement(doc); + WRBUF norm_wr = wrbuf_alloc(); + xmlNode *n; + + /* create mergekey based on mergekey attribute from XSL (if any) */ + xmlChar *mergekey = xmlGetProp(root, (xmlChar *) "mergekey"); + if (mergekey) + { + const char *norm_str; + pp2_relevance_token_t prt = + pp2_relevance_tokenize( + service->mergekey_pct, + (const char *) mergekey); + + while ((norm_str = pp2_relevance_token_next(prt))) + { + if (*norm_str) + { + if (wrbuf_len(norm_wr)) + wrbuf_puts(norm_wr, " "); + wrbuf_puts(norm_wr, norm_str); + } + } + pp2_relevance_token_destroy(prt); + xmlFree(mergekey); + } + /* append (if any) mergekey=yes metadata values */ + for (n = root->children; n; n = n->next) + { + if (n->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) n->name, "metadata")) + { + struct conf_metadata *ser_md = 0; + int md_field_id = -1; + + xmlChar *type = xmlGetProp(n, (xmlChar *) "type"); + + if (!type) + continue; + + md_field_id + = conf_service_metadata_field_id(service, + (const char *) type); + if (md_field_id >= 0) + { + ser_md = &service->metadata[md_field_id]; + if (ser_md->mergekey == Metadata_mergekey_yes) + { + xmlChar *value = xmlNodeListGetString(doc, n->children, 1); + if (value) + { + const char *norm_str; + pp2_relevance_token_t prt = + pp2_relevance_tokenize( + service->mergekey_pct, + (const char *) value); + + while ((norm_str = pp2_relevance_token_next(prt))) + { + if (*norm_str) + { + if (wrbuf_len(norm_wr)) + wrbuf_puts(norm_wr, " "); + wrbuf_puts(norm_wr, norm_str); + } + } + xmlFree(value); + pp2_relevance_token_destroy(prt); + } + } + } + xmlFree(type); + } + } + + /* generate unique key if none is not generated already or is empty */ + if (wrbuf_len(norm_wr) == 0) + { + wrbuf_printf(norm_wr, "%s-%d", + client_get_database(cl)->database->url, record_no); + } + if (wrbuf_len(norm_wr) > 0) + mergekey_norm = nmem_strdup(nmem, wrbuf_cstr(norm_wr)); + wrbuf_destroy(norm_wr); + return mergekey_norm; +} + + + +/** \brief ingest XML record + \param cl client holds the result set for record + \param rec record buffer (0 terminated) + \param record_no record position (1, 2, ..) + \returns resulting record or NULL on failure +*/ +struct record *ingest_record(struct client *cl, const char *rec, int record_no) { xmlDoc *xdoc = normalize_record(client_get_database(cl), @@ -1097,54 +1022,29 @@ struct record *ingest_record(struct client *cl, Z_External *rec, struct record *record; struct record_cluster *cluster; struct session *se = client_get_session(cl); - xmlChar *mergekey, *mergekey_norm; + const char *mergekey_norm; xmlChar *type = 0; xmlChar *value = 0; - struct conf_service *service = global_parameters.server->service; - const char *norm_str = 0; - pp2_relevance_token_t prt = 0; - WRBUF norm_wr = 0; + struct conf_service *service = se->service; if (!xdoc) return 0; root = xmlDocGetRootElement(xdoc); - if (!(mergekey = xmlGetProp(root, (xmlChar *) "mergekey"))) + + mergekey_norm = get_mergekey(xdoc, cl, record_no, service, se->nmem); + if (!mergekey_norm) { - yaz_log(YLOG_WARN, "No mergekey found in record"); + yaz_log(YLOG_WARN, "Got no mergekey"); xmlFreeDoc(xdoc); return 0; } - record = record_create(se->nmem, service->num_metadata, service->num_sortkeys, cl, record_no); - prt = pp2_relevance_tokenize( - global_parameters.server->mergekey_pct, (const char *) mergekey); - - - norm_wr = wrbuf_alloc(); - - while ((norm_str = pp2_relevance_token_next(prt))) - { - if (*norm_str) - { - if (wrbuf_len(norm_wr)) - wrbuf_puts(norm_wr, " "); - wrbuf_puts(norm_wr, norm_str); - } - } - - mergekey_norm = (xmlChar *)nmem_strdup(se->nmem, wrbuf_cstr(norm_wr)); - wrbuf_destroy(norm_wr); - - pp2_relevance_token_destroy(prt); - - xmlFree(mergekey); - cluster = reclist_insert(se->reclist, - global_parameters.server->service, + service, record, (char *) mergekey_norm, &se->total_merged); if (global_parameters.dump_records) @@ -1157,119 +1057,131 @@ struct record *ingest_record(struct client *cl, Z_External *rec, return 0; } relevance_newrec(se->relevance, cluster); - - - // now parsing XML record and adding data to cluster or record metadata - for (n = root->children; n; n = n->next) - { - if (type) - xmlFree(type); - if (value) - xmlFree(value); - type = value = 0; - - if (n->type != XML_ELEMENT_NODE) - continue; - if (!strcmp((const char *) n->name, "metadata")) - { - struct conf_metadata *ser_md = 0; - struct conf_sortkey *ser_sk = 0; - struct record_metadata **wheretoput = 0; - struct record_metadata *rec_md = 0; - int md_field_id = -1; - int sk_field_id = -1; - - type = xmlGetProp(n, (xmlChar *) "type"); - value = xmlNodeListGetString(xdoc, n->children, 1); - - if (!type || !value || !*value) - continue; - - md_field_id - = conf_service_metadata_field_id(service, (const char *) type); - if (md_field_id < 0) - { - yaz_log(YLOG_WARN, - "Ignoring unknown metadata element: %s", type); - continue; - } - - ser_md = &service->metadata[md_field_id]; - - if (ser_md->sortkey_offset >= 0){ - sk_field_id = ser_md->sortkey_offset; - ser_sk = &service->sortkeys[sk_field_id]; - } - - // non-merged metadata - rec_md = record_metadata_init(se->nmem, (char *) value, - ser_md->type); - if (!rec_md) - { - yaz_log(YLOG_WARN, "bad metadata data '%s' for element '%s'", - value, type); - continue; - } - rec_md->next = record->metadata[md_field_id]; - record->metadata[md_field_id] = rec_md; - - // merged metadata - rec_md = record_metadata_init(se->nmem, (char *) value, - ser_md->type); - wheretoput = &cluster->metadata[md_field_id]; - - // and polulate with data: - // assign cluster or record based on merge action - if (ser_md->merge == Metadata_merge_unique) - { - struct record_metadata *mnode; - for (mnode = *wheretoput; mnode; mnode = mnode->next) - if (!strcmp((const char *) mnode->data.text.disp, - rec_md->data.text.disp)) - break; - if (!mnode) - { - rec_md->next = *wheretoput; - *wheretoput = rec_md; - } - } - else if (ser_md->merge == Metadata_merge_longest) - { - if (!*wheretoput - || strlen(rec_md->data.text.disp) - > strlen((*wheretoput)->data.text.disp)) - { - *wheretoput = rec_md; - if (ser_sk) - { - const char *sort_str = 0; - int skip_article = - ser_sk->type == Metadata_sortkey_skiparticle; - - if (!cluster->sortkeys[sk_field_id]) - cluster->sortkeys[sk_field_id] = - nmem_malloc(se->nmem, - sizeof(union data_types)); + + // now parsing XML record and adding data to cluster or record metadata + for (n = root->children; n; n = n->next) + { + pp2_relevance_token_t prt; + if (type) + xmlFree(type); + if (value) + xmlFree(value); + type = value = 0; + + if (n->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) n->name, "metadata")) + { + struct conf_metadata *ser_md = 0; + struct conf_sortkey *ser_sk = 0; + struct record_metadata **wheretoput = 0; + struct record_metadata *rec_md = 0; + int md_field_id = -1; + int sk_field_id = -1; + + type = xmlGetProp(n, (xmlChar *) "type"); + value = xmlNodeListGetString(xdoc, n->children, 1); + + if (!type || !value || !*value) + continue; + + md_field_id + = conf_service_metadata_field_id(service, (const char *) type); + if (md_field_id < 0) + { + if (se->number_of_warnings_unknown_metadata == 0) + { + yaz_log(YLOG_WARN, + "Ignoring unknown metadata element: %s", type); + } + se->number_of_warnings_unknown_metadata++; + continue; + } + + ser_md = &service->metadata[md_field_id]; + + if (ser_md->sortkey_offset >= 0){ + sk_field_id = ser_md->sortkey_offset; + ser_sk = &service->sortkeys[sk_field_id]; + } + + // non-merged metadata + rec_md = record_metadata_init(se->nmem, (char *) value, + ser_md->type); + if (!rec_md) + { + yaz_log(YLOG_WARN, "bad metadata data '%s' for element '%s'", + value, type); + continue; + } + wheretoput = &record->metadata[md_field_id]; + while (*wheretoput) + wheretoput = &(*wheretoput)->next; + *wheretoput = rec_md; + + // merged metadata + rec_md = record_metadata_init(se->nmem, (char *) value, + ser_md->type); + wheretoput = &cluster->metadata[md_field_id]; + + // and polulate with data: + // assign cluster or record based on merge action + if (ser_md->merge == Metadata_merge_unique) + { + struct record_metadata *mnode; + for (mnode = *wheretoput; mnode; mnode = mnode->next) + if (!strcmp((const char *) mnode->data.text.disp, + rec_md->data.text.disp)) + break; + if (!mnode) + { + rec_md->next = *wheretoput; + *wheretoput = rec_md; + } + } + else if (ser_md->merge == Metadata_merge_longest) + { + if (!*wheretoput + || strlen(rec_md->data.text.disp) + > strlen((*wheretoput)->data.text.disp)) + { + *wheretoput = rec_md; + if (ser_sk) + { + const char *sort_str = 0; + int skip_article = + ser_sk->type == Metadata_sortkey_skiparticle; + + if (!cluster->sortkeys[sk_field_id]) + cluster->sortkeys[sk_field_id] = + nmem_malloc(se->nmem, + sizeof(union data_types)); - prt = pp2_relevance_tokenize( - global_parameters.server->sort_pct, - rec_md->data.text.disp); + prt = pp2_relevance_tokenize( + service->sort_pct, + rec_md->data.text.disp); - pp2_relevance_token_next(prt); + pp2_relevance_token_next(prt); - sort_str = pp2_get_sort(prt, skip_article); + sort_str = pp2_get_sort(prt, skip_article); - cluster->sortkeys[sk_field_id]->text.disp = - rec_md->data.text.disp; - cluster->sortkeys[sk_field_id]->text.sort = - nmem_strdup(se->nmem, sort_str); + cluster->sortkeys[sk_field_id]->text.disp = + rec_md->data.text.disp; + if (!sort_str) + { + sort_str = rec_md->data.text.disp; + yaz_log(YLOG_WARN, + "Could not make sortkey. Bug #1858"); + } + cluster->sortkeys[sk_field_id]->text.sort = + nmem_strdup(se->nmem, sort_str); #if 0 - yaz_log(YLOG_LOG, "text disp=%s", - cluster->sortkeys[sk_field_id]->text.disp); - yaz_log(YLOG_LOG, "text sort=%s", - cluster->sortkeys[sk_field_id]->text.sort); + yaz_log(YLOG_LOG, "text disp=%s", + cluster->sortkeys[sk_field_id]->text.disp); + yaz_log(YLOG_LOG, "text sort=%s", + cluster->sortkeys[sk_field_id]->text.sort); #endif - pp2_relevance_token_destroy(prt); + pp2_relevance_token_destroy(prt); } } } @@ -1296,15 +1208,6 @@ struct record *ingest_record(struct client *cl, Z_External *rec, if (this_max > (*wheretoput)->data.number.max) (*wheretoput)->data.number.max = this_max; } -#ifdef GAGA - if (ser_sk) - { - union data_types *sdata - = cluster->sortkeys[sk_field_id]; - yaz_log(YLOG_LOG, "SK range: %d-%d", - sdata->number.min, sdata->number.max); - } -#endif } @@ -1337,8 +1240,12 @@ struct record *ingest_record(struct client *cl, Z_External *rec, type = value = 0; } else - yaz_log(YLOG_WARN, - "Unexpected element %s in internal record", n->name); + { + if (se->number_of_warnings_unknown_elements == 0) + yaz_log(YLOG_WARN, + "Unexpected element in internal record: %s", n->name); + se->number_of_warnings_unknown_elements++; + } } if (type) xmlFree(type); @@ -1358,7 +1265,9 @@ struct record *ingest_record(struct client *cl, Z_External *rec, /* * Local variables: * c-basic-offset: 4 + * c-file-style: "Stroustrup" * indent-tabs-mode: nil * End: * vim: shiftwidth=4 tabstop=8 expandtab */ +