X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Flogic.c;h=1d34beecdf5c4b5fd74dd0fcc67b2686f7c61658;hb=34bcbd2020326fe7ad6afe43db64e54f887322ec;hp=eae862b13e2e43707e4befb1d5647ffbd9a90b1e;hpb=a085694d89c72f473733b58d08ddc2f5b37390e2;p=pazpar2-moved-to-github.git diff --git a/src/logic.c b/src/logic.c index eae862b..1d34bee 100644 --- a/src/logic.c +++ b/src/logic.c @@ -1,7 +1,5 @@ -/* $Id: logic.c,v 1.54 2007-07-16 17:01:46 adam Exp $ - Copyright (c) 2006-2007, Index Data. - -This file is part of Pazpar2. +/* This file is part of Pazpar2. + Copyright (C) 2006-2009 Index Data Pazpar2 is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -14,22 +12,28 @@ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with Pazpar2; see the file LICENSE. If not, write to the -Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA -02111-1307, USA. - */ +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ /** \file logic.c \brief high-level logic; mostly user sessions and settings */ +#if HAVE_CONFIG_H +#include +#endif + #include #include #include +#if HAVE_SYS_TIME_H #include +#endif +#if HAVE_UNISTD_H #include -#include -#include +#endif #include #include #include @@ -46,25 +50,20 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include #include #include - -#if HAVE_CONFIG_H -#include "cconfig.h" -#endif +#include #define USE_TIMING 0 #if USE_TIMING #include #endif -#include - +#include "parameters.h" #include "pazpar2.h" #include "eventl.h" #include "http.h" #include "termlists.h" #include "reclists.h" #include "relevance.h" -#include "config.h" #include "database.h" #include "client.h" #include "settings.h" @@ -77,24 +76,27 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA // Note: Some things in this structure will eventually move to configuration struct parameters global_parameters = { - "", - "", - "", - 0, - 0, /* dump_records */ - 0, /* debug_mode */ - 30, - "81", - "Index Data PazPar2", - VERSION, - 600, // 10 minutes - 60, - 100, - MAX_CHUNK, - 0, - 0 + 0, // dump_records + 0 // debug_mode }; +static void log_xml_doc(xmlDoc *doc) +{ + FILE *lf = yaz_log_file(); + xmlChar *result = 0; + int len = 0; +#if LIBXML_VERSION >= 20600 + xmlDocDumpFormatMemory(doc, &result, &len, 1); +#else + xmlDocDumpMemory(doc, &result, &len); +#endif + if (lf && len) + { + fwrite(result, 1, len, lf); + fprintf(lf, "\n"); + } + xmlFree(result); +} // Recursively traverse query structure to extract terms. void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) @@ -105,20 +107,20 @@ void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) switch (n->kind) { - case CCL_RPN_AND: - case CCL_RPN_OR: - case CCL_RPN_NOT: - case CCL_RPN_PROX: - pull_terms(nmem, n->u.p[0], termlist, num); - pull_terms(nmem, n->u.p[1], termlist, num); - break; - case CCL_RPN_TERM: - nmem_strsplit(nmem, " ", n->u.t.term, &words, &numwords); - for (i = 0; i < numwords; i++) - termlist[(*num)++] = words[i]; - break; - default: // NOOP - break; + case CCL_RPN_AND: + case CCL_RPN_OR: + case CCL_RPN_NOT: + case CCL_RPN_PROX: + pull_terms(nmem, n->u.p[0], termlist, num); + pull_terms(nmem, n->u.p[1], termlist, num); + break; + case CCL_RPN_TERM: + nmem_strsplit(nmem, " ", n->u.t.term, &words, &numwords); + for (i = 0; i < numwords; i++) + termlist[(*num)++] = words[i]; + break; + default: // NOOP + break; } } @@ -143,216 +145,163 @@ static void add_facet(struct session *s, const char *type, const char *value) s->termlists[i].name = nmem_strdup(s->nmem, type); s->termlists[i].termlist - = termlist_create(s->nmem, s->expected_maxrecs, - TERMLIST_HIGH_SCORE); + = termlist_create(s->nmem, TERMLIST_HIGH_SCORE); s->num_termlists = i + 1; } termlist_insert(s->termlists[i].termlist, value); } -xmlDoc *record_to_xml(struct session_database *sdb, Z_External *rec) +static xmlDoc *record_to_xml(struct session_database *sdb, const char *rec) { struct database *db = sdb->database; xmlDoc *rdoc = 0; - const Odr_oid *oid = rec->direct_reference; - /* convert response record to XML somehow */ - if (rec->which == Z_External_octet && oid - && !oid_oidcmp(oid, yaz_oid_recsyn_xml)) + rdoc = xmlParseMemory(rec, strlen(rec)); + + if (!rdoc) { - /* xml already */ - rdoc = xmlParseMemory((char*) rec->u.octet_aligned->buf, - rec->u.octet_aligned->len); - if (!rdoc) - { - yaz_log(YLOG_FATAL, "Non-wellformed XML received from %s", - db->url); - return 0; - } + yaz_log(YLOG_FATAL, "Non-wellformed XML received from %s", + db->url); + return 0; } - else if (rec->which == Z_External_OPAC) + + if (global_parameters.dump_records) { - if (!sdb->yaz_marc) - { - yaz_log(YLOG_WARN, "MARC decoding not configured"); - return 0; - } - else - { - /* OPAC gets converted to XML too */ - WRBUF wrbuf_opac = wrbuf_alloc(); - /* MARCXML inside the OPAC XML. Charset is in effect because we - use the yaz_marc handle */ - yaz_marc_xml(sdb->yaz_marc, YAZ_MARC_MARCXML); - yaz_opac_decode_wrbuf(sdb->yaz_marc, rec->u.opac, wrbuf_opac); - - rdoc = xmlParseMemory((char*) wrbuf_buf(wrbuf_opac), - wrbuf_len(wrbuf_opac)); - if (!rdoc) - yaz_log(YLOG_WARN, "Unable to parse OPAC XML"); - wrbuf_destroy(wrbuf_opac); - } + yaz_log(YLOG_LOG, "Un-normalized record from %s", db->url); + log_xml_doc(rdoc); } - else if (oid && yaz_oid_is_iso2709(oid)) + + return rdoc; +} + +#define MAX_XSLT_ARGS 16 + +// Add static values from session database settings if applicable +static void insert_settings_parameters(struct session_database *sdb, + struct session *se, char **parms) +{ + struct conf_service *service = se->service; + int i; + int nparms = 0; + int offset = 0; + + for (i = 0; i < service->num_metadata; i++) { - /* ISO2709 gets converted to MARCXML */ - if (!sdb->yaz_marc) - { - yaz_log(YLOG_WARN, "MARC decoding not configured"); - return 0; - } - else + struct conf_metadata *md = &service->metadata[i]; + int setting; + + if (md->setting == Metadata_setting_parameter && + (setting = settings_lookup_offset(service, md->name)) >= 0) { - xmlNode *res; - char *buf; - int len; - - if (rec->which != Z_External_octet) + const char *val = session_setting_oneval(sdb, setting); + if (val && nparms < MAX_XSLT_ARGS) { - yaz_log(YLOG_WARN, "Unexpected external branch, probably BER %s", - db->url); - return 0; + char *buf; + int len = strlen(val); + buf = nmem_malloc(se->nmem, len + 3); + buf[0] = '\''; + strcpy(buf + 1, val); + buf[len+1] = '\''; + buf[len+2] = '\0'; + parms[offset++] = md->name; + parms[offset++] = buf; + nparms++; } - buf = (char*) rec->u.octet_aligned->buf; - len = rec->u.octet_aligned->len; - if (yaz_marc_read_iso2709(sdb->yaz_marc, buf, len) < 0) - { - yaz_log(YLOG_WARN, "Failed to decode MARC %s", db->url); - return 0; - } - - yaz_marc_write_using_libxml2(sdb->yaz_marc, 1); - if (yaz_marc_write_xml(sdb->yaz_marc, &res, - "http://www.loc.gov/MARC21/slim", 0, 0) < 0) - { - yaz_log(YLOG_WARN, "Failed to encode as XML %s", - db->url); - return 0; - } - rdoc = xmlNewDoc((xmlChar *) "1.0"); - xmlDocSetRootElement(rdoc, res); } } - else - { - char oid_name_buf[OID_STR_MAX]; - const char *oid_name = yaz_oid_to_string_buf(oid, 0, oid_name_buf); - yaz_log(YLOG_FATAL, - "Unable to handle record of type %s from %s", - oid_name, db->url); - return 0; - } - - if (global_parameters.dump_records) + parms[offset] = 0; +} + +// Add static values from session database settings if applicable +static void insert_settings_values(struct session_database *sdb, xmlDoc *doc, + struct conf_service *service) +{ + int i; + + for (i = 0; i < service->num_metadata; i++) { - FILE *lf = yaz_log_file(); - if (lf) + struct conf_metadata *md = &service->metadata[i]; + int offset; + + if (md->setting == Metadata_setting_postproc && + (offset = settings_lookup_offset(service, md->name)) >= 0) { - yaz_log(YLOG_LOG, "Un-normalized record from %s", db->url); -#if LIBXML_VERSION >= 20600 - xmlDocFormatDump(lf, rdoc, 1); -#else - xmlDocDump(lf, rdoc); -#endif - fprintf(lf, "\n"); + const char *val = session_setting_oneval(sdb, offset); + if (val) + { + xmlNode *r = xmlDocGetRootElement(doc); + xmlNode *n = xmlNewTextChild(r, 0, (xmlChar *) "metadata", + (xmlChar *) val); + xmlSetProp(n, (xmlChar *) "type", (xmlChar *) md->name); + } } } - return rdoc; } -xmlDoc *normalize_record(struct session_database *sdb, Z_External *rec) +static xmlDoc *normalize_record(struct session_database *sdb, + struct session *se, + const char *rec) { - struct database_retrievalmap *m; xmlDoc *rdoc = record_to_xml(sdb, rec); + if (rdoc) { - for (m = sdb->map; m; m = m->next){ - xmlDoc *new = 0; - - { - xmlNodePtr root = 0; - new = xsltApplyStylesheet(m->stylesheet, rdoc, 0); - root= xmlDocGetRootElement(new); - if (!new || !root || !(root->children)) - { - yaz_log(YLOG_WARN, "XSLT transformation failed from %s", - sdb->database->url); - xmlFreeDoc(new); - xmlFreeDoc(rdoc); - return 0; - } - } - - xmlFreeDoc(rdoc); - rdoc = new; + char *parms[MAX_XSLT_ARGS*2+1]; + + insert_settings_parameters(sdb, se, parms); + + if (normalize_record_transform(sdb->map, &rdoc, (const char **)parms)) + { + yaz_log(YLOG_WARN, "Normalize failed from %s", sdb->database->url); } - if (global_parameters.dump_records) + else { - FILE *lf = yaz_log_file(); + insert_settings_values(sdb, rdoc, se->service); - if (lf) + if (global_parameters.dump_records) { yaz_log(YLOG_LOG, "Normalized record from %s", sdb->database->url); -#if LIBXML_VERSION >= 20600 - xmlDocFormatDump(lf, rdoc, 1); -#else - xmlDocDump(lf, rdoc); -#endif - fprintf(lf, "\n"); + log_xml_doc(rdoc); } } } return rdoc; } +void session_settings_dump(struct session *se, + struct session_database *db, + WRBUF w) +{ + if (db->settings) + { + int i, num = db->num_settings; + for (i = 0; i < num; i++) + { + struct setting *s = db->settings[i]; + for (;s ; s = s->next) + { + wrbuf_puts(w, "name); + wrbuf_puts(w, "\" value=\""); + wrbuf_xmlputs(w, s->value); + wrbuf_puts(w, "\"/>"); + } + if (db->settings[i]) + wrbuf_puts(w, "\n"); + } + } +} + // Retrieve first defined value for 'name' for given database. // Will be extended to take into account user associated with session -char *session_setting_oneval(struct session_database *db, int offset) +const char *session_setting_oneval(struct session_database *db, int offset) { if (!db->settings[offset]) return ""; return db->settings[offset]->value; } - - -// Initialize YAZ Map structures for MARC-based targets -static int prepare_yazmarc(struct session_database *sdb) -{ - char *s; - - if (!sdb->settings) - { - yaz_log(YLOG_WARN, "No settings for %s", sdb->database->url); - return -1; - } - if ((s = session_setting_oneval(sdb, PZ_NATIVESYNTAX)) - && !strncmp(s, "iso2709", 7)) - { - char *encoding = "marc-8s", *e; - yaz_iconv_t cm; - - // See if a native encoding is specified - if ((e = strchr(s, ';'))) - encoding = e + 1; - - sdb->yaz_marc = yaz_marc_create(); - yaz_marc_subfield_str(sdb->yaz_marc, "\t"); - - cm = yaz_iconv_open("utf-8", encoding); - if (!cm) - { - yaz_log(YLOG_FATAL, - "Unable to map from %s to UTF-8 for target %s", - encoding, sdb->database->url); - return -1; - } - yaz_marc_iconv(sdb->yaz_marc, cm); - } - return 0; -} - // Prepare XSLT stylesheets for record normalization // Structures are allocated on the session_wide nmem to avoid having // to recompute this for every search. This would lead @@ -360,7 +309,7 @@ static int prepare_yazmarc(struct session_database *sdb) // setting. However, this is not a realistic use scenario. static int prepare_map(struct session *se, struct session_database *sdb) { - char *s; + const char *s; if (!sdb->settings) { @@ -369,27 +318,35 @@ static int prepare_map(struct session *se, struct session_database *sdb) } if ((s = session_setting_oneval(sdb, PZ_XSLT))) { - char **stylesheets; - struct database_retrievalmap **m = &sdb->map; - int num, i; + char auto_stylesheet[256]; - nmem_strsplit(se->session_nmem, ",", s, &stylesheets, &num); - for (i = 0; i < num; i++) + if (!strcmp(s, "auto")) { - (*m) = nmem_malloc(se->session_nmem, sizeof(**m)); - (*m)->next = 0; - if (!((*m)->stylesheet = conf_load_stylesheet(stylesheets[i]))) + const char *request_syntax = session_setting_oneval( + sdb, PZ_REQUESTSYNTAX); + if (request_syntax) + { + char *cp; + yaz_snprintf(auto_stylesheet, sizeof(auto_stylesheet), + "%s.xsl", request_syntax); + for (cp = auto_stylesheet; *cp; cp++) + { + /* deliberately only consider ASCII */ + if (*cp > 32 && *cp < 127) + *cp = tolower(*cp); + } + s = auto_stylesheet; + } + else { - yaz_log(YLOG_FATAL|YLOG_ERRNO, "Unable to load stylesheet: %s", - stylesheets[i]); - return -1; + yaz_log(YLOG_WARN, "No pz:requestsyntax for auto stylesheet"); } - m = &(*m)->next; } + sdb->map = normalize_cache_get(se->normalize_cache, + se->service, s); + if (!sdb->map) + return -1; } - if (!sdb->map) - yaz_log(YLOG_WARN, "No Normalization stylesheet for target %s", - sdb->database->url); return 0; } @@ -404,11 +361,6 @@ static int prepare_session_database(struct session *se, "No settings associated with %s", sdb->database->url); return -1; } - if (sdb->settings[PZ_NATIVESYNTAX] && !sdb->yaz_marc) - { - if (prepare_yazmarc(sdb) < 0) - return -1; - } if (sdb->settings[PZ_XSLT] && !sdb->map) { if (prepare_map(se, sdb) < 0) @@ -418,7 +370,8 @@ static int prepare_session_database(struct session *se, } // called if watch should be removed because http_channel is to be destroyed -static void session_watch_cancel(void *data, struct http_channel *c) +static void session_watch_cancel(void *data, struct http_channel *c, + void *data2) { struct session_watchentry *ent = data; @@ -443,13 +396,24 @@ int session_set_watch(struct session *s, int what, void session_alert_watch(struct session *s, int what) { - if (!s->watchlist[what].fun) - return; - http_remove_observer(s->watchlist[what].obs); - (*s->watchlist[what].fun)(s->watchlist[what].data); - s->watchlist[what].fun = 0; - s->watchlist[what].data = 0; - s->watchlist[what].obs = 0; + if (s->watchlist[what].fun) + { + /* our watch is no longer associated with http_channel */ + void *data; + session_watchfun fun; + + http_remove_observer(s->watchlist[what].obs); + fun = s->watchlist[what].fun; + data = s->watchlist[what].data; + + /* reset watch before fun is invoked - in case fun wants to set + it again */ + s->watchlist[what].fun = 0; + s->watchlist[what].data = 0; + s->watchlist[what].obs = 0; + + fun(data); + } } //callback for grep_databases @@ -464,12 +428,12 @@ static void select_targets_callback(void *context, struct session_database *db) // Associates a set of clients with a session; // Note: Session-databases represent databases with per-session // setting overrides -int select_targets(struct session *se, struct database_criterion *crit) +static int select_targets(struct session *se, const char *filter) { while (se->clients) client_destroy(se->clients); - return session_grep_databases(se, crit, select_targets_callback); + return session_grep_databases(se, filter, select_targets_callback); } int session_active_clients(struct session *s) @@ -484,100 +448,62 @@ int session_active_clients(struct session *s) return res; } -// parses crit1=val1,crit2=val2|val3,... -static struct database_criterion *parse_filter(NMEM m, const char *buf) -{ - struct database_criterion *res = 0; - char **values; - int num; - int i; - - if (!buf || !*buf) - return 0; - nmem_strsplit(m, ",", buf, &values, &num); - for (i = 0; i < num; i++) - { - char **subvalues; - int subnum; - int subi; - struct database_criterion *new = nmem_malloc(m, sizeof(*new)); - char *eq = strchr(values[i], '='); - if (!eq) - { - yaz_log(YLOG_WARN, "Missing equal-sign in filter"); - return 0; - } - *(eq++) = '\0'; - new->name = values[i]; - nmem_strsplit(m, "|", eq, &subvalues, &subnum); - new->values = 0; - for (subi = 0; subi < subnum; subi++) - { - struct database_criterion_value *newv - = nmem_malloc(m, sizeof(*newv)); - newv->value = subvalues[subi]; - newv->next = new->values; - new->values = newv; - } - new->next = res; - res = new; - } - return res; -} enum pazpar2_error_code search(struct session *se, - char *query, char *filter, + const char *query, + const char *startrecs, const char *maxrecs, + const char *filter, const char **addinfo) { int live_channels = 0; int no_working = 0; int no_failed = 0; struct client *cl; - struct database_criterion *criteria; yaz_log(YLOG_DEBUG, "Search"); *addinfo = 0; nmem_reset(se->nmem); - criteria = parse_filter(se->nmem, filter); - se->requestid++; - live_channels = select_targets(se, criteria); + se->relevance = 0; + se->total_records = se->total_hits = se->total_merged = 0; + se->reclist = 0; + se->num_termlists = 0; + live_channels = select_targets(se, filter); if (live_channels) { - int maxrecs = live_channels * global_parameters.toget; - se->num_termlists = 0; - se->reclist = reclist_create(se->nmem, maxrecs); - // This will be initialized in send_search() - se->total_records = se->total_hits = se->total_merged = 0; - se->expected_maxrecs = maxrecs; + se->reclist = reclist_create(se->nmem); } else return PAZPAR2_NO_TARGETS; - se->relevance = 0; - for (cl = se->clients; cl; cl = client_next_in_session(cl)) { + if (maxrecs) + client_set_maxrecs(cl, atoi(maxrecs)); + if (startrecs) + client_set_startrecs(cl, atoi(startrecs)); if (prepare_session_database(se, client_get_database(cl)) < 0) - { - *addinfo = client_get_database(cl)->database->url; - return PAZPAR2_CONFIG_TARGET; - } + continue; // Parse query for target if (client_parse_query(cl, query) < 0) no_failed++; else { no_working++; - client_prep_connection(cl); + if (client_prep_connection(cl, se->service->z3950_operation_timeout, + se->service->z3950_session_timeout)) + client_start_search(cl); } } - - // If no queries could be mapped, we signal an error if (no_working == 0) { - *addinfo = "query"; - return PAZPAR2_MALFORMED_PARAMETER_VALUE; + if (no_failed > 0) + { + *addinfo = "query"; + return PAZPAR2_MALFORMED_PARAMETER_VALUE; + } + else + return PAZPAR2_NO_TARGETS; } return PAZPAR2_NO_ERROR; } @@ -587,31 +513,17 @@ static void session_init_databases_fun(void *context, struct database *db) { struct session *se = (struct session *) context; struct session_database *new = nmem_malloc(se->session_nmem, sizeof(*new)); - int num = settings_num(); int i; new->database = db; - new->yaz_marc = 0; -#ifdef HAVE_ICU - if (global_parameters.server && global_parameters.server->icu_chn) - new->pct = pp2_charset_create(global_parameters.server->icu_chn); - else - new->pct = pp2_charset_create(0); -#else // HAVE_ICU - new->pct = pp2_charset_create(0); -#endif // HAVE_ICU - new->map = 0; - new->settings - = nmem_malloc(se->session_nmem, sizeof(struct settings *) * num); - memset(new->settings, 0, sizeof(struct settings*) * num); - - if (db->settings) - { - for (i = 0; i < num; i++) - new->settings[i] = db->settings[i]; - } + assert(db->settings); + new->settings = nmem_malloc(se->session_nmem, + sizeof(struct settings *) * db->num_settings); + new->num_settings = db->num_settings; + for (i = 0; i < db->num_settings; i++) + new->settings[i] = db->settings[i]; new->next = se->databases; se->databases = new; } @@ -619,14 +531,7 @@ static void session_init_databases_fun(void *context, struct database *db) // Doesn't free memory associated with sdb -- nmem takes care of that static void session_database_destroy(struct session_database *sdb) { - struct database_retrievalmap *m; - - for (m = sdb->map; m; m = m->next) - xsltFreeStylesheet(m->stylesheet); - if (sdb->yaz_marc) - yaz_marc_destroy(sdb->yaz_marc); - if (sdb->pct) - pp2_charset_destroy(sdb->pct); + sdb->map = 0; } // Initialize session_database list -- this represents this session's view @@ -634,7 +539,7 @@ static void session_database_destroy(struct session_database *sdb) void session_init_databases(struct session *se) { se->databases = 0; - predef_grep_databases(se, 0, session_init_databases_fun); + predef_grep_databases(se, se->service, session_init_databases_fun); } // Probably session_init_databases_fun should be refactored instead of @@ -642,7 +547,9 @@ void session_init_databases(struct session *se) static struct session_database *load_session_database(struct session *se, char *id) { - struct database *db = find_database(id, 0); + struct database *db = find_database(id, se->service); + + resolve_database(db); session_init_databases_fun((void*) se, db); // New sdb is head of se->databases list @@ -666,20 +573,12 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, char *value) { struct session_database *sdb = find_session_database(se, dbname); + struct conf_service *service = se->service; struct setting *new = nmem_malloc(se->session_nmem, sizeof(*new)); - int offset = settings_offset_cprefix(setting); + int offset = settings_create_offset(service, setting); - if (offset < 0) - { - yaz_log(YLOG_WARN, "Unknown setting %s", setting); - return; - } - // Jakub: This breaks the filter setting. - /*if (offset == PZ_ID) - { - yaz_log(YLOG_WARN, "No need to set pz:id setting. Ignoring"); - return; - }*/ + expand_settings_array(&sdb->settings, &sdb->num_settings, offset, + se->session_nmem); new->precedence = 0; new->target = dbname; new->name = setting; @@ -691,23 +590,12 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, // (happens when a search starts and client connections are prepared) switch (offset) { - case PZ_NATIVESYNTAX: - if (sdb->yaz_marc) - { - yaz_marc_destroy(sdb->yaz_marc); - sdb->yaz_marc = 0; - } - break; - case PZ_XSLT: - if (sdb->map) - { - struct database_retrievalmap *m; - // We don't worry about the map structure -- it's in nmem - for (m = sdb->map; m; m = m->next) - xsltFreeStylesheet(m->stylesheet); - sdb->map = 0; - } - break; + case PZ_XSLT: + if (sdb->map) + { + sdb->map = 0; + } + break; } } @@ -719,25 +607,28 @@ void destroy_session(struct session *s) client_destroy(s->clients); for (sdb = s->databases; sdb; sdb = sdb->next) session_database_destroy(sdb); + normalize_cache_destroy(s->normalize_cache); nmem_destroy(s->nmem); + service_destroy(s->service); wrbuf_destroy(s->wrbuf); } -struct session *new_session(NMEM nmem) +struct session *new_session(NMEM nmem, struct conf_service *service) { int i; struct session *session = nmem_malloc(nmem, sizeof(*session)); yaz_log(YLOG_DEBUG, "New Pazpar2 session"); - + + session->service = service; session->relevance = 0; session->total_hits = 0; session->total_records = 0; + session->number_of_warnings_unknown_elements = 0; + session->number_of_warnings_unknown_metadata = 0; session->num_termlists = 0; session->reclist = 0; - session->requestid = -1; session->clients = 0; - session->expected_maxrecs = 0; session->session_nmem = nmem; session->nmem = nmem_create(); session->wrbuf = wrbuf_alloc(); @@ -747,19 +638,27 @@ struct session *new_session(NMEM nmem) session->watchlist[i].data = 0; session->watchlist[i].fun = 0; } + session->normalize_cache = normalize_cache_create(); return session; } -struct hitsbytarget *hitsbytarget(struct session *se, int *count) +struct hitsbytarget *hitsbytarget(struct session *se, int *count, NMEM nmem) { - static struct hitsbytarget res[1000]; // FIXME MM + struct hitsbytarget *res = 0; struct client *cl; + size_t sz = 0; + + for (cl = se->clients; cl; cl = client_next_in_session(cl)) + sz++; + res = nmem_malloc(nmem, sizeof(*res) * sz); *count = 0; for (cl = se->clients; cl; cl = client_next_in_session(cl)) { - char *name = session_setting_oneval(client_get_database(cl), PZ_NAME); + WRBUF w = wrbuf_alloc(); + const char *name = session_setting_oneval(client_get_database(cl), + PZ_NAME); res[*count].id = client_get_database(cl)->database->url; res[*count].name = *name ? name : "Unknown"; @@ -768,9 +667,10 @@ struct hitsbytarget *hitsbytarget(struct session *se, int *count) res[*count].diagnostic = client_get_diagnostic(cl); res[*count].state = client_get_state_str(cl); res[*count].connected = client_get_connection(cl) ? 1 : 0; + session_settings_dump(se, client_get_database(cl), w); + res[*count].settings_xml = w; (*count)++; } - return res; } @@ -797,23 +697,33 @@ void report_nmem_stats(void) } #endif -struct record_cluster *show_single(struct session *s, const char *id) +struct record_cluster *show_single(struct session *s, const char *id, + struct record_cluster **prev_r, + struct record_cluster **next_r) { struct record_cluster *r; reclist_rewind(s->reclist); + *prev_r = 0; + *next_r = 0; while ((r = reclist_read_record(s->reclist))) + { if (!strcmp(r->recid, id)) + { + *next_r = reclist_read_record(s->reclist); return r; + } + *prev_r = r; + } return 0; } struct record_cluster **show(struct session *s, struct reclist_sortparms *sp, - int start, int *num, int *total, int *sumhits, + int start, int *num, int *total, Odr_int *sumhits, NMEM nmem_show) { struct record_cluster **recs = nmem_malloc(nmem_show, *num - * sizeof(struct record_cluster *)); + * sizeof(struct record_cluster *)); struct reclist_sortparms *spp; int i; #if USE_TIMING @@ -837,7 +747,7 @@ struct record_cluster **show(struct session *s, struct reclist_sortparms *sp, } reclist_sort(s->reclist, sp); - *total = s->reclist->num_records; + *total = reclist_get_num_records(s->reclist); *sumhits = s->total_hits; for (i = 0; i < start; i++) @@ -881,14 +791,12 @@ void statistics(struct session *se, struct statistics *stat) stat->num_no_connection++; switch (client_get_state(cl)) { - case Client_Connecting: stat->num_connecting++; break; - case Client_Initializing: stat->num_initializing++; break; - case Client_Searching: stat->num_searching++; break; - case Client_Presenting: stat->num_presenting++; break; - case Client_Idle: stat->num_idle++; break; - case Client_Failed: stat->num_failed++; break; - case Client_Error: stat->num_error++; break; - default: break; + case Client_Connecting: stat->num_connecting++; break; + case Client_Working: stat->num_working++; break; + case Client_Idle: stat->num_idle++; break; + case Client_Failed: stat->num_failed++; break; + case Client_Error: stat->num_error++; break; + default: break; } count++; } @@ -898,52 +806,10 @@ void statistics(struct session *se, struct statistics *stat) stat->num_clients = count; } -void start_http_listener(void) -{ - char hp[128] = ""; - struct conf_server *ser = global_parameters.server; - - if (*global_parameters.listener_override) - strcpy(hp, global_parameters.listener_override); - else - { - strcpy(hp, ser->host ? ser->host : ""); - if (ser->port) - { - if (*hp) - strcat(hp, ":"); - sprintf(hp + strlen(hp), "%d", ser->port); - } - } - http_init(hp); -} - -void start_proxy(void) -{ - char hp[128] = ""; - struct conf_server *ser = global_parameters.server; - - if (*global_parameters.proxy_override) - strcpy(hp, global_parameters.proxy_override); - else if (ser->proxy_host || ser->proxy_port) - { - strcpy(hp, ser->proxy_host ? ser->proxy_host : ""); - if (ser->proxy_port) - { - if (*hp) - strcat(hp, ":"); - sprintf(hp + strlen(hp), "%d", ser->proxy_port); - } - } - else - return; - - http_set_proxyaddr(hp, ser->myurl ? ser->myurl : ""); -} - // Master list of connections we're handling events to -static IOCHAN channel_list = 0; +static IOCHAN channel_list = 0; /* thread pr */ + void pazpar2_add_channel(IOCHAN chan) { chan->next = channel_list; @@ -956,21 +822,50 @@ void pazpar2_event_loop() } static struct record_metadata *record_metadata_init( - NMEM nmem, char *value, enum conf_metadata_type type) + NMEM nmem, const char *value, enum conf_metadata_type type, + struct _xmlAttr *attr) { struct record_metadata *rec_md = record_metadata_create(nmem); + struct record_metadata_attr **attrp = &rec_md->attributes; + + for (; attr; attr = attr->next) + { + if (attr->children && attr->children->content) + { + if (strcmp((const char *) attr->name, "type")) + { /* skip the "type" attribute.. Its value is already part of + the element in output (md-%s) and so repeating it here + is redundant */ + *attrp = nmem_malloc(nmem, sizeof(**attrp)); + (*attrp)->name = + nmem_strdup(nmem, (const char *) attr->name); + (*attrp)->value = + nmem_strdup(nmem, (const char *) attr->children->content); + attrp = &(*attrp)->next; + } + } + } + *attrp = 0; + if (type == Metadata_type_generic) { - char * p = value; + char *p = nmem_strdup(nmem, value); + p = normalize7bit_generic(p, " ,/.:(["); - rec_md->data.text = nmem_strdup(nmem, p); + rec_md->data.text.disp = p; + rec_md->data.text.sort = 0; } - else if (type == Metadata_type_year) + else if (type == Metadata_type_year || type == Metadata_type_date) { int first, last; - if (extract7bit_years((char *) value, &first, &last) < 0) + int longdate = 0; + + if (type == Metadata_type_date) + longdate = 1; + if (extract7bit_dates((char *) value, &first, &last, longdate) < 0) return 0; + rec_md->data.number.min = first; rec_md->data.number.max = last; } @@ -979,63 +874,238 @@ static struct record_metadata *record_metadata_init( return rec_md; } -struct record *ingest_record(struct client *cl, Z_External *rec, +static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, + struct conf_service *service, WRBUF norm_wr) +{ + xmlNode *n; + int no_found = 0; + for (n = root->children; n; n = n->next) + { + if (n->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) n->name, "metadata")) + { + xmlChar *type = xmlGetProp(n, (xmlChar *) "type"); + if (!strcmp(name, (const char *) type)) + { + xmlChar *value = xmlNodeListGetString(doc, n->children, 1); + if (value) + { + const char *norm_str; + pp2_relevance_token_t prt = + pp2_relevance_tokenize( + service->mergekey_pct, + (const char *) value, 0); + + if (wrbuf_len(norm_wr) > 0) + wrbuf_puts(norm_wr, " "); + wrbuf_puts(norm_wr, name); + while ((norm_str = + pp2_relevance_token_next(prt))) + { + if (*norm_str) + { + wrbuf_puts(norm_wr, " "); + wrbuf_puts(norm_wr, norm_str); + } + } + xmlFree(value); + pp2_relevance_token_destroy(prt); + no_found++; + } + } + xmlFree(type); + } + } + return no_found; +} + +static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, + struct conf_service *service, NMEM nmem) +{ + char *mergekey_norm = 0; + xmlNode *root = xmlDocGetRootElement(doc); + WRBUF norm_wr = wrbuf_alloc(); + + /* consider mergekey from XSL first */ + xmlChar *mergekey = xmlGetProp(root, (xmlChar *) "mergekey"); + if (mergekey) + { + const char *norm_str; + pp2_relevance_token_t prt = + pp2_relevance_tokenize( + service->mergekey_pct, + (const char *) mergekey, 0); + + while ((norm_str = pp2_relevance_token_next(prt))) + { + if (*norm_str) + { + if (wrbuf_len(norm_wr)) + wrbuf_puts(norm_wr, " "); + wrbuf_puts(norm_wr, norm_str); + } + } + pp2_relevance_token_destroy(prt); + xmlFree(mergekey); + } + else + { + /* no mergekey defined in XSL. Look for mergekey metadata instead */ + int field_id; + for (field_id = 0; field_id < service->num_metadata; field_id++) + { + struct conf_metadata *ser_md = &service->metadata[field_id]; + if (ser_md->mergekey != Metadata_mergekey_no) + { + int r = get_mergekey_from_doc(doc, root, ser_md->name, + service, norm_wr); + if (r == 0 && ser_md->mergekey == Metadata_mergekey_required) + { + /* no mergekey on this one and it is required.. + Generate unique key instead */ + wrbuf_rewind(norm_wr); + break; + } + } + } + } + + /* generate unique key if none is not generated already or is empty */ + if (wrbuf_len(norm_wr) == 0) + { + wrbuf_printf(norm_wr, "%s-%d", + client_get_database(cl)->database->url, record_no); + } + if (wrbuf_len(norm_wr) > 0) + mergekey_norm = nmem_strdup(nmem, wrbuf_cstr(norm_wr)); + wrbuf_destroy(norm_wr); + return mergekey_norm; +} + +/** \brief see if metadata for pz:recordfilter exists + \param root xml root element of normalized record + \param sdb session database for client + \retval 0 if there is no metadata for pz:recordfilter + \retval 1 if there is metadata for pz:recordfilter + + If there is no pz:recordfilter defined, this function returns 1 + as well. +*/ + +static int check_record_filter(xmlNode *root, struct session_database *sdb) +{ + int match = 0; + xmlNode *n; + const char *s; + s = session_setting_oneval(sdb, PZ_RECORDFILTER); + + if (!s || !*s) + return 1; + + for (n = root->children; n; n = n->next) + { + if (n->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) n->name, "metadata")) + { + xmlChar *type = xmlGetProp(n, (xmlChar *) "type"); + if (type) + { + size_t len; + const char *eq = strchr(s, '~'); + if (eq) + len = eq - s; + else + len = strlen(s); + if (len == strlen((const char *)type) && + !memcmp((const char *) type, s, len)) + { + xmlChar *value = xmlNodeGetContent(n); + if (value && *value) + { + if (!eq || strstr((const char *) value, eq+1)) + match = 1; + } + xmlFree(value); + } + xmlFree(type); + } + } + } + return match; +} + + +/** \brief ingest XML record + \param cl client holds the result set for record + \param rec record buffer (0 terminated) + \param record_no record position (1, 2, ..) + \returns resulting record or NULL on failure +*/ +struct record *ingest_record(struct client *cl, const char *rec, int record_no) { - xmlDoc *xdoc = normalize_record(client_get_database(cl), rec); + struct session_database *sdb = client_get_database(cl); + struct session *se = client_get_session(cl); + xmlDoc *xdoc = normalize_record(sdb, se, rec); xmlNode *root, *n; struct record *record; struct record_cluster *cluster; - struct session *se = client_get_session(cl); - xmlChar *mergekey, *mergekey_norm; + const char *mergekey_norm; xmlChar *type = 0; xmlChar *value = 0; - struct conf_service *service = global_parameters.server->service; + struct conf_service *service = se->service; if (!xdoc) return 0; root = xmlDocGetRootElement(xdoc); - if (!(mergekey = xmlGetProp(root, (xmlChar *) "mergekey"))) + + if (!check_record_filter(root, sdb)) { - yaz_log(YLOG_WARN, "No mergekey found in record"); + yaz_log(YLOG_WARN, "Filtered out record no %d from %s", record_no, + sdb->database->url); xmlFreeDoc(xdoc); return 0; } + mergekey_norm = get_mergekey(xdoc, cl, record_no, service, se->nmem); + if (!mergekey_norm) + { + yaz_log(YLOG_WARN, "Got no mergekey"); + xmlFreeDoc(xdoc); + return 0; + } record = record_create(se->nmem, service->num_metadata, service->num_sortkeys, cl, record_no); - mergekey_norm = (xmlChar *) nmem_strdup(se->nmem, (char*) mergekey); - xmlFree(mergekey); - normalize7bit_mergekey((char *) mergekey_norm, 0); - cluster = reclist_insert(se->reclist, - global_parameters.server->service, + service, record, (char *) mergekey_norm, &se->total_merged); - if (global_parameters.dump_records) - yaz_log(YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid, - client_get_database(cl)->database->url, record_no); if (!cluster) { /* no room for record */ xmlFreeDoc(xdoc); return 0; } + if (global_parameters.dump_records) + yaz_log(YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid, + sdb->database->url, record_no); relevance_newrec(se->relevance, cluster); - - + // now parsing XML record and adding data to cluster or record metadata for (n = root->children; n; n = n->next) { + pp2_relevance_token_t prt; if (type) xmlFree(type); if (value) xmlFree(value); type = value = 0; - + if (n->type != XML_ELEMENT_NODE) continue; if (!strcmp((const char *) n->name, "metadata")) @@ -1046,44 +1116,50 @@ struct record *ingest_record(struct client *cl, Z_External *rec, struct record_metadata *rec_md = 0; int md_field_id = -1; int sk_field_id = -1; - + type = xmlGetProp(n, (xmlChar *) "type"); value = xmlNodeListGetString(xdoc, n->children, 1); - - if (!type || !value) + + if (!type || !value || !*value) continue; - + md_field_id = conf_service_metadata_field_id(service, (const char *) type); if (md_field_id < 0) { - yaz_log(YLOG_WARN, - "Ignoring unknown metadata element: %s", type); + if (se->number_of_warnings_unknown_metadata == 0) + { + yaz_log(YLOG_WARN, + "Ignoring unknown metadata element: %s", type); + } + se->number_of_warnings_unknown_metadata++; continue; } - + ser_md = &service->metadata[md_field_id]; - + if (ser_md->sortkey_offset >= 0){ sk_field_id = ser_md->sortkey_offset; ser_sk = &service->sortkeys[sk_field_id]; } // non-merged metadata - rec_md = record_metadata_init(se->nmem, (char *) value, - ser_md->type); + rec_md = record_metadata_init(se->nmem, (const char *) value, + ser_md->type, n->properties); if (!rec_md) { yaz_log(YLOG_WARN, "bad metadata data '%s' for element '%s'", value, type); continue; } - rec_md->next = record->metadata[md_field_id]; - record->metadata[md_field_id] = rec_md; + wheretoput = &record->metadata[md_field_id]; + while (*wheretoput) + wheretoput = &(*wheretoput)->next; + *wheretoput = rec_md; // merged metadata - rec_md = record_metadata_init(se->nmem, (char *) value, - ser_md->type); + rec_md = record_metadata_init(se->nmem, (const char *) value, + ser_md->type, 0); wheretoput = &cluster->metadata[md_field_id]; // and polulate with data: @@ -1092,8 +1168,8 @@ struct record *ingest_record(struct client *cl, Z_External *rec, { struct record_metadata *mnode; for (mnode = *wheretoput; mnode; mnode = mnode->next) - if (!strcmp((const char *) mnode->data.text, - rec_md->data.text)) + if (!strcmp((const char *) mnode->data.text.disp, + rec_md->data.text.disp)) break; if (!mnode) { @@ -1104,20 +1180,46 @@ struct record *ingest_record(struct client *cl, Z_External *rec, else if (ser_md->merge == Metadata_merge_longest) { if (!*wheretoput - || strlen(rec_md->data.text) - > strlen((*wheretoput)->data.text)) + || strlen(rec_md->data.text.disp) + > strlen((*wheretoput)->data.text.disp)) { *wheretoput = rec_md; if (ser_sk) { - char *s = nmem_strdup(se->nmem, rec_md->data.text); + const char *sort_str = 0; + int skip_article = + ser_sk->type == Metadata_sortkey_skiparticle; + if (!cluster->sortkeys[sk_field_id]) cluster->sortkeys[sk_field_id] = nmem_malloc(se->nmem, sizeof(union data_types)); - normalize7bit_mergekey(s, - (ser_sk->type == Metadata_sortkey_skiparticle)); - cluster->sortkeys[sk_field_id]->text = s; + + prt = pp2_relevance_tokenize( + service->sort_pct, + rec_md->data.text.disp, skip_article); + + pp2_relevance_token_next(prt); + + sort_str = pp2_get_sort(prt); + + cluster->sortkeys[sk_field_id]->text.disp = + rec_md->data.text.disp; + if (!sort_str) + { + sort_str = rec_md->data.text.disp; + yaz_log(YLOG_WARN, + "Could not make sortkey. Bug #1858"); + } + cluster->sortkeys[sk_field_id]->text.sort = + nmem_strdup(se->nmem, sort_str); +#if 0 + yaz_log(YLOG_LOG, "text disp=%s", + cluster->sortkeys[sk_field_id]->text.disp); + yaz_log(YLOG_LOG, "text sort=%s", + cluster->sortkeys[sk_field_id]->text.sort); +#endif + pp2_relevance_token_destroy(prt); } } } @@ -1144,22 +1246,14 @@ struct record *ingest_record(struct client *cl, Z_External *rec, if (this_max > (*wheretoput)->data.number.max) (*wheretoput)->data.number.max = this_max; } -#ifdef GAGA - if (ser_sk) - { - union data_types *sdata - = cluster->sortkeys[sk_field_id]; - yaz_log(YLOG_LOG, "SK range: %d-%d", - sdata->number.min, sdata->number.max); - } -#endif } // ranking of _all_ fields enabled ... if (ser_md->rank) relevance_countwords(se->relevance, cluster, - (char *) value, ser_md->rank); + (char *) value, ser_md->rank, + ser_md->name); // construct facets ... if (ser_md->termlist) @@ -1185,8 +1279,12 @@ struct record *ingest_record(struct client *cl, Z_External *rec, type = value = 0; } else - yaz_log(YLOG_WARN, - "Unexpected element %s in internal record", n->name); + { + if (se->number_of_warnings_unknown_elements == 0) + yaz_log(YLOG_WARN, + "Unexpected element in internal record: %s", n->name); + se->number_of_warnings_unknown_elements++; + } } if (type) xmlFree(type); @@ -1206,7 +1304,9 @@ struct record *ingest_record(struct client *cl, Z_External *rec, /* * Local variables: * c-basic-offset: 4 + * c-file-style: "Stroustrup" * indent-tabs-mode: nil * End: * vim: shiftwidth=4 tabstop=8 expandtab */ +