X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Fsession.c;h=5d34aa47a3502e3809e3caf0c99b9cefe32aec8b;hb=817e3ec506c4095bc4fcc1923cee36153ef4ee43;hp=ea0924993bba8f0f59338b08dbf1d59e83d8e6da;hpb=b667495a906ab6c565fbf37149a9825a540736a4;p=pazpar2-moved-to-github.git diff --git a/src/session.c b/src/session.c index ea09249..d965d75 100644 --- a/src/session.c +++ b/src/session.c @@ -1,5 +1,5 @@ /* This file is part of Pazpar2. - Copyright (C) 2006-2010 Index Data + Copyright (C) Index Data Pazpar2 is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -41,6 +41,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #include #include +#include #include #include @@ -55,9 +56,8 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #include #include -#include -#define USE_TIMING 0 +#define USE_TIMING 1 #if USE_TIMING #include #endif @@ -75,15 +75,18 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include "settings.h" #include "normalize7bit.h" -#define TERMLIST_HIGH_SCORE 25 +#include #define MAX_CHUNK 15 +#define MAX(a,b) ((a)>(b)?(a):(b)) + // Note: Some things in this structure will eventually move to configuration -struct parameters global_parameters = +struct parameters global_parameters = { 0, // dump_records - 0 // debug_mode + 0, // debug_mode + 0, // predictable sessions }; struct client_list { @@ -91,6 +94,29 @@ struct client_list { struct client_list *next; }; +/* session counting (1) , disable client counting (0) */ +static YAZ_MUTEX g_session_mutex = 0; +static int no_sessions = 0; + +static int session_use(int delta) +{ + int sessions; + if (!g_session_mutex) + yaz_mutex_create(&g_session_mutex); + yaz_mutex_enter(g_session_mutex); + no_sessions += delta; + sessions = no_sessions; + yaz_mutex_leave(g_session_mutex); + yaz_log(YLOG_DEBUG, "%s sessions=%d", delta == 0 ? "" : + (delta > 0 ? "INC" : "DEC"), no_sessions); + return sessions; +} + +int sessions_count(void) +{ + return session_use(0); +} + static void log_xml_doc(xmlDoc *doc) { FILE *lf = yaz_log_file(); @@ -109,71 +135,106 @@ static void log_xml_doc(xmlDoc *doc) xmlFree(result); } -static void session_enter(struct session *s) +static void session_enter(struct session *s, const char *caller) { + if (caller) + session_log(s, YLOG_DEBUG, "Session lock by %s", caller); yaz_mutex_enter(s->session_mutex); } -static void session_leave(struct session *s) +static void session_leave(struct session *s, const char *caller) { yaz_mutex_leave(s->session_mutex); + if (caller) + session_log(s, YLOG_DEBUG, "Session unlock by %s", caller); } -// Recursively traverse query structure to extract terms. -void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) +static void session_normalize_facet(struct session *s, + const char *type, const char *value, + WRBUF display_wrbuf, WRBUF facet_wrbuf) { - char **words; - int numwords; + struct conf_service *service = s->service; + pp2_charset_token_t prt; + const char *facet_component; int i; + const char *icu_chain_id = 0; - switch (n->kind) + for (i = 0; i < service->num_metadata; i++) + if (!strcmp((service->metadata + i)->name, type)) + icu_chain_id = (service->metadata + i)->facetrule; + if (!icu_chain_id) + icu_chain_id = "facet"; + prt = pp2_charset_token_create(service->charsets, icu_chain_id); + if (!prt) { - case CCL_RPN_AND: - case CCL_RPN_OR: - case CCL_RPN_NOT: - case CCL_RPN_PROX: - pull_terms(nmem, n->u.p[0], termlist, num); - pull_terms(nmem, n->u.p[1], termlist, num); - break; - case CCL_RPN_TERM: - nmem_strsplit(nmem, " ", n->u.t.term, &words, &numwords); - for (i = 0; i < numwords; i++) - termlist[(*num)++] = words[i]; - break; - default: // NOOP - break; + session_log(s, YLOG_FATAL, + "Unknown ICU chain '%s' for facet of type '%s'", + icu_chain_id, type); + wrbuf_destroy(facet_wrbuf); + wrbuf_destroy(display_wrbuf); + return; + } + pp2_charset_token_first(prt, value, 0); + while ((facet_component = pp2_charset_token_next(prt))) + { + const char *display_component; + if (*facet_component) + { + if (wrbuf_len(facet_wrbuf)) + wrbuf_puts(facet_wrbuf, " "); + wrbuf_puts(facet_wrbuf, facet_component); + } + display_component = pp2_get_display(prt); + if (display_component) + { + if (wrbuf_len(display_wrbuf)) + wrbuf_puts(display_wrbuf, " "); + wrbuf_puts(display_wrbuf, display_component); + } } + pp2_charset_token_destroy(prt); } - void add_facet(struct session *s, const char *type, const char *value, int count) { - int i; + WRBUF facet_wrbuf = wrbuf_alloc(); + WRBUF display_wrbuf = wrbuf_alloc(); - if (!*value) - return; - for (i = 0; i < s->num_termlists; i++) - if (!strcmp(s->termlists[i].name, type)) - break; - if (i == s->num_termlists) + session_normalize_facet(s, type, value, display_wrbuf, facet_wrbuf); + + if (wrbuf_len(facet_wrbuf)) { - if (i == SESSION_MAX_TERMLISTS) + int i; + for (i = 0; i < s->num_termlists; i++) + if (!strcmp(s->termlists[i].name, type)) + break; + if (i == s->num_termlists) { - yaz_log(YLOG_FATAL, "Too many termlists"); - return; + if (i == SESSION_MAX_TERMLISTS) + { + session_log(s, YLOG_FATAL, "Too many termlists"); + wrbuf_destroy(facet_wrbuf); + wrbuf_destroy(display_wrbuf); + return; + } + + s->termlists[i].name = nmem_strdup(s->nmem, type); + s->termlists[i].termlist = termlist_create(s->nmem); + s->num_termlists = i + 1; } - s->termlists[i].name = nmem_strdup(s->nmem, type); - s->termlists[i].termlist - = termlist_create(s->nmem, TERMLIST_HIGH_SCORE); - s->num_termlists = i + 1; +#if 0 + session_log(s, YLOG_LOG, "Facets for %s: %s norm:%s (%d)", type, value, wrbuf_cstr(facet_wrbuf), count); +#endif + termlist_insert(s->termlists[i].termlist, wrbuf_cstr(display_wrbuf), + wrbuf_cstr(facet_wrbuf), count); } - yaz_log(YLOG_DEBUG, "Session: facets for %s: %s (%d)", type, value, count); - - termlist_insert(s->termlists[i].termlist, value, count); + wrbuf_destroy(facet_wrbuf); + wrbuf_destroy(display_wrbuf); } -static xmlDoc *record_to_xml(struct session_database *sdb, const char *rec) +static xmlDoc *record_to_xml(struct session *se, + struct session_database *sdb, const char *rec) { struct database *db = sdb->database; xmlDoc *rdoc = 0; @@ -182,14 +243,13 @@ static xmlDoc *record_to_xml(struct session_database *sdb, const char *rec) if (!rdoc) { - yaz_log(YLOG_FATAL, "Non-wellformed XML received from %s", - db->url); + session_log(se, YLOG_WARN, "Non-wellformed XML"); return 0; } if (global_parameters.dump_records) { - yaz_log(YLOG_LOG, "Un-normalized record from %s", db->url); + session_log(se, YLOG_LOG, "Un-normalized record from %s", db->id); log_xml_doc(rdoc); } @@ -237,7 +297,8 @@ static void insert_settings_parameters(struct session_database *sdb, // Add static values from session database settings if applicable static void insert_settings_values(struct session_database *sdb, xmlDoc *doc, - struct conf_service *service) + xmlNode *root, + struct conf_service *service) { int i; @@ -252,8 +313,7 @@ static void insert_settings_values(struct session_database *sdb, xmlDoc *doc, const char *val = session_setting_oneval(sdb, offset); if (val) { - xmlNode *r = xmlDocGetRootElement(doc); - xmlNode *n = xmlNewTextChild(r, 0, (xmlChar *) "metadata", + xmlNode *n = xmlNewTextChild(root, 0, (xmlChar *) "metadata", (xmlChar *) val); xmlSetProp(n, (xmlChar *) "type", (xmlChar *) md->name); } @@ -261,32 +321,22 @@ static void insert_settings_values(struct session_database *sdb, xmlDoc *doc, } } -static xmlDoc *normalize_record(struct session_database *sdb, +static xmlDoc *normalize_record(struct session *se, + struct session_database *sdb, struct conf_service *service, const char *rec, NMEM nmem) { - xmlDoc *rdoc = record_to_xml(sdb, rec); + xmlDoc *rdoc = record_to_xml(se, sdb, rec); if (rdoc) { char *parms[MAX_XSLT_ARGS*2+1]; - + insert_settings_parameters(sdb, service, parms, nmem); - + if (normalize_record_transform(sdb->map, &rdoc, (const char **)parms)) { - yaz_log(YLOG_WARN, "Normalize failed from %s", sdb->database->url); - } - else - { - insert_settings_values(sdb, rdoc, service); - - if (global_parameters.dump_records) - { - yaz_log(YLOG_LOG, "Normalized record from %s", - sdb->database->url); - log_xml_doc(rdoc); - } + session_log(se, YLOG_WARN, "Normalize failed"); } } return rdoc; @@ -332,62 +382,43 @@ const char *session_setting_oneval(struct session_database *db, int offset) // setting. However, this is not a realistic use scenario. static int prepare_map(struct session *se, struct session_database *sdb) { - const char *s; - - if (!sdb->settings) - { - yaz_log(YLOG_WARN, "No settings on %s", sdb->database->url); - return -1; - } - if ((s = session_setting_oneval(sdb, PZ_XSLT))) + if (sdb->settings && !sdb->map) { - char auto_stylesheet[256]; + const char *s; - if (!strcmp(s, "auto")) + if (sdb->settings[PZ_XSLT] && + (s = session_setting_oneval(sdb, PZ_XSLT))) { - const char *request_syntax = session_setting_oneval( - sdb, PZ_REQUESTSYNTAX); - if (request_syntax) + char auto_stylesheet[256]; + + if (!strcmp(s, "auto")) { - char *cp; - yaz_snprintf(auto_stylesheet, sizeof(auto_stylesheet), - "%s.xsl", request_syntax); - for (cp = auto_stylesheet; *cp; cp++) + const char *request_syntax = session_setting_oneval( + sdb, PZ_REQUESTSYNTAX); + if (request_syntax) { - /* deliberately only consider ASCII */ - if (*cp > 32 && *cp < 127) - *cp = tolower(*cp); + char *cp; + yaz_snprintf(auto_stylesheet, sizeof(auto_stylesheet), + "%s.xsl", request_syntax); + for (cp = auto_stylesheet; *cp; cp++) + { + /* deliberately only consider ASCII */ + if (*cp > 32 && *cp < 127) + *cp = tolower(*cp); + } + s = auto_stylesheet; + } + else + { + session_log(se, YLOG_WARN, + "No pz:requestsyntax for auto stylesheet"); } - s = auto_stylesheet; - } - else - { - yaz_log(YLOG_WARN, "No pz:requestsyntax for auto stylesheet"); } + sdb->map = normalize_cache_get(se->normalize_cache, + se->service, s); + if (!sdb->map) + return -1; } - sdb->map = normalize_cache_get(se->normalize_cache, - se->service->server->config, s); - if (!sdb->map) - return -1; - } - return 0; -} - -// This analyzes settings and recomputes any supporting data structures -// if necessary. -static int prepare_session_database(struct session *se, - struct session_database *sdb) -{ - if (!sdb->settings) - { - yaz_log(YLOG_WARN, - "No settings associated with %s", sdb->database->url); - return -1; - } - if (sdb->settings[PZ_XSLT] && !sdb->map) - { - if (prepare_map(se, sdb) < 0) - return -1; } return 0; } @@ -404,31 +435,31 @@ static void session_watch_cancel(void *data, struct http_channel *c, } // set watch. Returns 0=OK, -1 if watch is already set -int session_set_watch(struct session *s, int what, +int session_set_watch(struct session *s, int what, session_watchfun fun, void *data, struct http_channel *chan) { int ret; - session_enter(s); + session_enter(s, "session_set_watch"); if (s->watchlist[what].fun) ret = -1; else { - + s->watchlist[what].fun = fun; s->watchlist[what].data = data; s->watchlist[what].obs = http_add_observer(chan, &s->watchlist[what], session_watch_cancel); ret = 0; } - session_leave(s); - return 0; + session_leave(s, "session_set_watch"); + return ret; } void session_alert_watch(struct session *s, int what) { assert(s); - session_enter(s); + session_enter(s, "session_alert_watch"); if (s->watchlist[what].fun) { /* our watch is no longer associated with http_channel */ @@ -445,36 +476,80 @@ void session_alert_watch(struct session *s, int what) s->watchlist[what].data = 0; s->watchlist[what].obs = 0; - session_leave(s); + session_leave(s, "session_alert_watch"); + session_log(s, YLOG_DEBUG, + "Alert Watch: %d calling function: %p", what, fun); fun(data); } else - session_leave(s); + session_leave(s,"session_alert_watch"); } //callback for grep_databases -static void select_targets_callback(void *context, struct session_database *db) +static void select_targets_callback(struct session *se, + struct session_database *db) { - struct session *se = (struct session*) context; - struct client *cl = client_create(); + struct client *cl; struct client_list *l; - client_set_database(cl, db); + for (l = se->clients_cached; l; l = l->next) + if (client_get_database(l->client) == db) + break; + + if (l) + cl = l->client; + else + { + cl = client_create(db->database->id); + client_set_database(cl, db); + + l = xmalloc(sizeof(*l)); + l->client = cl; + l->next = se->clients_cached; + se->clients_cached = l; + } + /* set session always. If may be 0 if client is not active */ client_set_session(cl, se); + l = xmalloc(sizeof(*l)); l->client = cl; - l->next = se->clients; - se->clients = l; + l->next = se->clients_active; + se->clients_active = l; +} + +static void session_reset_active_clients(struct session *se, + struct client_list *new_list) +{ + struct client_list *l; + + session_enter(se, "session_reset_active_clients"); + l = se->clients_active; + se->clients_active = new_list; + session_leave(se, "session_reset_active_clients"); + + while (l) + { + struct client_list *l_next = l->next; + + client_lock(l->client); + client_set_session(l->client, 0); /* mark client inactive */ + client_unlock(l->client); + + xfree(l); + l = l_next; + } } -static void session_remove_clients(struct session *se) +static void session_remove_cached_clients(struct session *se) { struct client_list *l; - session_enter(se); - l = se->clients; - se->clients = 0; - session_leave(se); + session_reset_active_clients(se, 0); + + session_enter(se, "session_remove_cached_clients"); + l = se->clients_cached; + se->clients_cached = 0; + session_leave(se, "session_remove_cached_clients"); while (l) { @@ -490,7 +565,7 @@ static void session_remove_clients(struct session *se) } // Associates a set of clients with a session; -// Note: Session-databases represent databases with per-session +// Note: Session-databases represent databases with per-session // setting overrides static int select_targets(struct session *se, const char *filter) { @@ -502,94 +577,266 @@ int session_active_clients(struct session *s) struct client_list *l; int res = 0; - for (l = s->clients; l; l = l->next) + for (l = s->clients_active; l; l = l->next) if (client_is_active(l->client)) res++; return res; } -int session_preferred_clients_ready(struct session *s) +int session_is_preferred_clients_ready(struct session *s) { struct client_list *l; int res = 0; - for (l = s->clients; l; l = l->next) + for (l = s->clients_active; l; l = l->next) if (client_is_active_preferred(l->client)) res++; - + session_log(s, YLOG_DEBUG, "Has %d active preferred clients.", res); return res == 0; } +static void session_clear_set(struct session *se, struct reclist_sortparms *sp) +{ + reclist_destroy(se->reclist); + if (nmem_total(se->nmem)) + session_log(se, YLOG_DEBUG, "NMEN operation usage %zd", + nmem_total(se->nmem)); + nmem_reset(se->nmem); + se->total_records = se->total_merged = 0; + se->num_termlists = 0; + + relevance_clear(se->relevance); + + /* reset list of sorted results and clear to relevance search */ + se->sorted_results = nmem_malloc(se->nmem, sizeof(*se->sorted_results)); + se->sorted_results->name = nmem_strdup(se->nmem, sp->name); + se->sorted_results->increasing = sp->increasing; + se->sorted_results->type = sp->type; + se->sorted_results->next = 0; + + session_log(se, YLOG_DEBUG, "clear_set session_sort: field=%s increasing=%d type=%d configured", + sp->name, sp->increasing, sp->type); + + se->reclist = reclist_create(se->nmem); +} + +static void session_sort_unlocked(struct session *se, + struct reclist_sortparms *sp, + const char *mergekey, + const char *rank) +{ + struct client_list *l; + const char *field = sp->name; + int increasing = sp->increasing; + int type = sp->type; + int clients_research = 0; + + session_log(se, YLOG_DEBUG, "session_sort field=%s increasing=%d type=%d", + field, increasing, type); + + if (rank && (!se->rank || strcmp(se->rank, rank))) + { + /* new rank must research/reingest anyway */ + assert(rank); + xfree(se->rank); + se->rank = *rank ? xstrdup(rank) : 0; + clients_research = 1; + session_log(se, YLOG_DEBUG, "session_sort: new rank = %s", + rank); + } + if (mergekey && (!se->mergekey || strcmp(se->mergekey, mergekey))) + { + /* new mergekey must research/reingest anyway */ + assert(mergekey); + xfree(se->mergekey); + se->mergekey = *mergekey ? xstrdup(mergekey) : 0; + clients_research = 1; + session_log(se, YLOG_DEBUG, "session_sort: new mergekey = %s", + mergekey); + } + if (clients_research == 0) + { + struct reclist_sortparms *sr; + for (sr = se->sorted_results; sr; sr = sr->next) + if (!reclist_sortparms_cmp(sr, sp)) + break; + if (sr) + { + session_log(se, YLOG_DEBUG, "session_sort: field=%s increasing=%d type=%d already fetched", + field, increasing, type); + return; + } + } + session_log(se, YLOG_DEBUG, "session_sort: field=%s increasing=%d type=%d must fetch", + field, increasing, type); + + // We need to reset reclist on every sort that changes the records, not just for position + // So if just one client requires new searching, we need to clear set. + // Ask each of the client if sorting requires re-search due to native sort + // If it does it will require us to + for (l = se->clients_active; l; l = l->next) + { + struct client *cl = l->client; + // Assume no re-search is required. + client_parse_init(cl, 1); + clients_research += client_parse_sort(cl, sp); + } + if (clients_research) + { + session_log(se, YLOG_DEBUG, + "session_sort: reset results due to %d clients researching", + clients_research); + session_clear_set(se, sp); + } + else + { + // A new sorting based on same record set + struct reclist_sortparms *sr = nmem_malloc(se->nmem, sizeof(*sr)); + sr->name = nmem_strdup(se->nmem, field); + sr->increasing = increasing; + sr->type = type; + sr->next = se->sorted_results; + se->sorted_results = sr; + session_log(se, YLOG_DEBUG, "session_sort: no research/ingesting done"); + return ; + } + session_log(se, YLOG_DEBUG, "Re- search/ingesting for clients due to change in sort order"); + + for (l = se->clients_active; l; l = l->next) + { + struct client *cl = l->client; + if (client_get_state(cl) == Client_Connecting || + client_get_state(cl) == Client_Idle || + client_get_state(cl) == Client_Working) { + client_start_search(cl); + } + else + { + session_log(se, YLOG_DEBUG, + "session_sort: %s: No re-start/ingest in show. " + "Wrong client state: %d", + client_get_id(cl), client_get_state(cl)); + } + } +} + +void session_sort(struct session *se, struct reclist_sortparms *sp, + const char *mergekey, const char *rank) +{ + //session_enter(se, "session_sort"); + session_sort_unlocked(se, sp, mergekey, rank); + //session_leave(se, "session_sort"); +} -enum pazpar2_error_code search(struct session *se, - const char *query, - const char *startrecs, const char *maxrecs, - const char *filter, - const char **addinfo) +enum pazpar2_error_code session_search(struct session *se, + const char *query, + const char *startrecs, + const char *maxrecs, + const char *filter, + const char *limit, + const char **addinfo, + const char **addinfo2, + struct reclist_sortparms *sp, + const char *mergekey, + const char *rank) { int live_channels = 0; int no_working = 0; - int no_failed = 0; - struct client_list *l; - struct timeval tval; + int no_failed_query = 0; + int no_failed_limit = 0; + struct client_list *l, *l0; - yaz_log(YLOG_DEBUG, "Search"); + session_alert_watch(se, SESSION_WATCH_SHOW); + session_alert_watch(se, SESSION_WATCH_BYTARGET); + session_alert_watch(se, SESSION_WATCH_TERMLIST); + session_alert_watch(se, SESSION_WATCH_SHOW_PREF); + + session_log(se, YLOG_DEBUG, "Search"); *addinfo = 0; - session_remove_clients(se); - - session_enter(se); - reclist_destroy(se->reclist); - se->reclist = 0; + if (se->settings_modified) { + session_remove_cached_clients(se); + } + else + session_reset_active_clients(se, 0); + + session_enter(se, "session_search"); + se->settings_modified = 0; + + if (mergekey) + { + xfree(se->mergekey); + se->mergekey = *mergekey ? xstrdup(mergekey) : 0; + } + if (rank) + { + xfree(se->rank); + se->rank = *rank ? xstrdup(rank) : 0; + } + + session_clear_set(se, sp); relevance_destroy(&se->relevance); - nmem_reset(se->nmem); - se->total_records = se->total_hits = se->total_merged = 0; - se->num_termlists = 0; + live_channels = select_targets(se, filter); if (!live_channels) { - session_leave(se); + session_leave(se, "session_search"); return PAZPAR2_NO_TARGETS; } - se->reclist = reclist_create(se->nmem); - yaz_gettimeofday(&tval); - - tval.tv_sec += 5; + facet_limits_destroy(se->facet_limits); + se->facet_limits = facet_limits_create(limit); + if (!se->facet_limits) + { + *addinfo = "limit"; + session_leave(se, "session_search"); + return PAZPAR2_MALFORMED_PARAMETER_VALUE; + } - for (l = se->clients; l; l = l->next) + l0 = se->clients_active; + se->clients_active = 0; + session_leave(se, "session_search"); + + for (l = l0; l; l = l->next) { + int parse_ret; struct client *cl = l->client; + client_parse_init(cl, 1); + if (prepare_map(se, client_get_database(cl)) < 0) + continue; - if (maxrecs) - client_set_maxrecs(cl, atoi(maxrecs)); - if (startrecs) - client_set_startrecs(cl, atoi(startrecs)); - if (prepare_session_database(se, client_get_database(cl)) < 0) - ; - else if (client_parse_query(cl, query) < 0) - no_failed++; + parse_ret = client_parse_query(cl, query, se->facet_limits, addinfo2); + if (parse_ret == -1) + no_failed_query++; + else if (parse_ret == -2) + no_failed_limit++; + else if (parse_ret < 0) + no_working++; /* other error, such as bad CCL map */ else { + client_parse_range(cl, startrecs, maxrecs); + client_parse_sort(cl, sp); + client_start_search(cl); no_working++; - if (client_prep_connection(cl, se->service->z3950_operation_timeout, - se->service->z3950_session_timeout, - se->service->server->iochan_man, - &tval)) - client_start_search(cl); } } - session_leave(se); + session_reset_active_clients(se, l0); + if (no_working == 0) { - if (no_failed > 0) + if (no_failed_query > 0) { *addinfo = "query"; return PAZPAR2_MALFORMED_PARAMETER_VALUE; } + else if (no_failed_limit > 0) + { + *addinfo = "limit"; + return PAZPAR2_MALFORMED_PARAMETER_VALUE; + } else return PAZPAR2_NO_TARGETS; } @@ -604,7 +851,7 @@ static void session_init_databases_fun(void *context, struct database *db) int i; new->database = db; - + new->map = 0; assert(db->settings); new->settings = nmem_malloc(se->session_nmem, @@ -635,13 +882,10 @@ void session_init_databases(struct session *se) // Probably session_init_databases_fun should be refactored instead of // called here. -static struct session_database *load_session_database(struct session *se, +static struct session_database *load_session_database(struct session *se, char *id) { - struct database *db = new_database(id, se->session_nmem); - - resolve_database(se->service, db); - + struct database *db = new_database_inherit_settings(id, se->session_nmem, se->service->settings); session_init_databases_fun((void*) se, db); // New sdb is head of se->databases list @@ -649,13 +893,13 @@ static struct session_database *load_session_database(struct session *se, } // Find an existing session database. If not found, load it -static struct session_database *find_session_database(struct session *se, +static struct session_database *find_session_database(struct session *se, char *id) { struct session_database *sdb; for (sdb = se->databases; sdb; sdb = sdb->next) - if (!strcmp(sdb->database->url, id)) + if (!strcmp(sdb->database->id, id)) return sdb; return load_session_database(se, id); } @@ -678,6 +922,8 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, new->next = sdb->settings[offset]; sdb->settings[offset] = new; + se->settings_modified = 1; + // Force later recompute of settings-driven data structures // (happens when a search starts and client connections are prepared) switch (offset) @@ -691,45 +937,71 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, } } -void destroy_session(struct session *se) +void session_destroy(struct session *se) { struct session_database *sdb; - - yaz_log(YLOG_DEBUG, "%p Pazpar2 session destroy", se); - session_remove_clients(se); + session_log(se, YLOG_LOG, "destroy"); + session_use(-1); + session_remove_cached_clients(se); for (sdb = se->databases; sdb; sdb = sdb->next) session_database_destroy(sdb); normalize_cache_destroy(se->normalize_cache); relevance_destroy(&se->relevance); reclist_destroy(se->reclist); + xfree(se->mergekey); + xfree(se->rank); + if (nmem_total(se->nmem)) + session_log(se, YLOG_DEBUG, "NMEN operation usage %zd", nmem_total(se->nmem)); + if (nmem_total(se->session_nmem)) + session_log(se, YLOG_DEBUG, "NMEN session usage %zd", nmem_total(se->session_nmem)); + facet_limits_destroy(se->facet_limits); nmem_destroy(se->nmem); service_destroy(se->service); yaz_mutex_destroy(&se->session_mutex); - wrbuf_destroy(se->wrbuf); } +size_t session_get_memory_status(struct session *session) { + size_t session_nmem; + if (session == 0) + return 0; + session_enter(session, "session_get_memory_status"); + session_nmem = nmem_total(session->nmem); + session_leave(session, "session_get_memory_status"); + return session_nmem; +} + + struct session *new_session(NMEM nmem, struct conf_service *service, - const char *name) + unsigned session_id) { int i; struct session *session = nmem_malloc(nmem, sizeof(*session)); - yaz_log(YLOG_DEBUG, "%p New Pazpar2 session", session); + char tmp_str[50]; + + sprintf(tmp_str, "session#%u", session_id); + session->session_id = session_id; + session_log(session, YLOG_DEBUG, "New"); session->service = service; session->relevance = 0; - session->total_hits = 0; session->total_records = 0; session->number_of_warnings_unknown_elements = 0; session->number_of_warnings_unknown_metadata = 0; session->num_termlists = 0; - session->reclist = 0; - session->clients = 0; + session->reclist = reclist_create(nmem); + session->clients_active = 0; + session->clients_cached = 0; + session->settings_modified = 0; session->session_nmem = nmem; session->nmem = nmem_create(); - session->wrbuf = wrbuf_alloc(); session->databases = 0; + session->sorted_results = 0; + session->facet_limits = 0; + session->mergekey = 0; + session->rank = 0; + for (i = 0; i <= SESSION_WATCH_MAX; i++) { session->watchlist[i].data = 0; @@ -737,59 +1009,209 @@ struct session *new_session(NMEM nmem, struct conf_service *service, } session->normalize_cache = normalize_cache_create(); session->session_mutex = 0; - pazpar2_mutex_create(&session->session_mutex, name); + pazpar2_mutex_create(&session->session_mutex, tmp_str); + session_log(session, YLOG_LOG, "create"); + session_use(1); return session; } -struct hitsbytarget *hitsbytarget(struct session *se, int *count, NMEM nmem) +const char * client_get_suggestions_xml(struct client *cl, WRBUF wrbuf); + +static struct hitsbytarget *hitsbytarget_nb(struct session *se, + int *count, NMEM nmem) { struct hitsbytarget *res = 0; struct client_list *l; size_t sz = 0; - session_enter(se); - for (l = se->clients; l; l = l->next) + for (l = se->clients_active; l; l = l->next) sz++; res = nmem_malloc(nmem, sizeof(*res) * sz); *count = 0; - for (l = se->clients; l; l = l->next) + for (l = se->clients_active; l; l = l->next) { struct client *cl = l->client; WRBUF w = wrbuf_alloc(); const char *name = session_setting_oneval(client_get_database(cl), PZ_NAME); - res[*count].id = client_get_database(cl)->database->url; + res[*count].id = client_get_id(cl); res[*count].name = *name ? name : "Unknown"; res[*count].hits = client_get_hits(cl); + res[*count].approximation = client_get_approximation(cl); res[*count].records = client_get_num_records(cl); - res[*count].diagnostic = client_get_diagnostic(cl); + res[*count].filtered = client_get_num_records_filtered(cl); + res[*count].diagnostic = + client_get_diagnostic(cl, &res[*count].message, + &res[*count].addinfo); res[*count].state = client_get_state_str(cl); res[*count].connected = client_get_connection(cl) ? 1 : 0; session_settings_dump(se, client_get_database(cl), w); - res[*count].settings_xml = w; + res[*count].settings_xml = nmem_strdup(nmem, wrbuf_cstr(w)); + wrbuf_rewind(w); + wrbuf_puts(w, ""); + res[*count].suggestions_xml = nmem_strdup(nmem, client_get_suggestions_xml(cl, w)); + wrbuf_destroy(w); (*count)++; } - session_leave(se); return res; } -struct termlist_score **termlist(struct session *se, const char *name, int *num) +struct hitsbytarget *get_hitsbytarget(struct session *se, int *count, NMEM nmem) { - int i; - struct termlist_score **tl = 0; + struct hitsbytarget *p; + session_enter(se, "get_hitsbytarget"); + p = hitsbytarget_nb(se, count, nmem); + session_leave(se, "get_hitsbytarget"); + return p; +} + +// Compares two hitsbytarget nodes by hitcount +static int cmp_ht(const void *p1, const void *p2) +{ + const struct hitsbytarget *h1 = p1; + const struct hitsbytarget *h2 = p2; + return h2->hits - h1->hits; +} + +// Compares two hitsbytarget nodes by hitcount +static int cmp_ht_approx(const void *p1, const void *p2) +{ + const struct hitsbytarget *h1 = p1; + const struct hitsbytarget *h2 = p2; + return h2->approximation - h1->approximation; +} + +static int targets_termlist_nb(WRBUF wrbuf, struct session *se, int num, + NMEM nmem, int version) +{ + struct hitsbytarget *ht; + int count, i; + + ht = hitsbytarget_nb(se, &count, nmem); + if (version >= 2) + qsort(ht, count, sizeof(struct hitsbytarget), cmp_ht_approx); + else + qsort(ht, count, sizeof(struct hitsbytarget), cmp_ht); + for (i = 0; i < count && i < num && ht[i].hits > 0; i++) + { + + // do only print terms which have display names + + wrbuf_puts(wrbuf, "\n"); + + wrbuf_puts(wrbuf, ""); + wrbuf_xmlputs(wrbuf, ht[i].id); + wrbuf_puts(wrbuf, "\n"); + + wrbuf_puts(wrbuf, ""); + if (!ht[i].name || !ht[i].name[0]) + wrbuf_xmlputs(wrbuf, "NO TARGET NAME"); + else + wrbuf_xmlputs(wrbuf, ht[i].name); + wrbuf_puts(wrbuf, "\n"); + + wrbuf_printf(wrbuf, "" ODR_INT_PRINTF "\n", + ht[i].hits); + + if (version >= 2) { + // Should not print if we know it isn't a approximation. + wrbuf_printf(wrbuf, "" ODR_INT_PRINTF "\n", ht[i].approximation); + wrbuf_printf(wrbuf, "%d\n", ht[i].records - ht[i].filtered); + wrbuf_printf(wrbuf, "%d\n", ht[i].filtered); + } + + wrbuf_puts(wrbuf, ""); + wrbuf_xmlputs(wrbuf, ht[i].state); + wrbuf_puts(wrbuf, "\n"); + + wrbuf_printf(wrbuf, "%d\n", + ht[i].diagnostic); + wrbuf_puts(wrbuf, "\n"); + } + return count; +} + +void perform_termlist(struct http_channel *c, struct session *se, + const char *name, int num, int version) +{ + int i, j; + NMEM nmem_tmp = nmem_create(); + char **names; + int num_names = 0; + + if (!name) + name = "*"; - session_enter(se); - for (i = 0; i < se->num_termlists; i++) - if (!strcmp((const char *) se->termlists[i].name, name)) + nmem_strsplit(nmem_tmp, ",", name, &names, &num_names); + + session_enter(se, "perform_termlist"); + + for (j = 0; j < num_names; j++) + { + const char *tname; + int must_generate_empty = 1; /* bug 5350 */ + + for (i = 0; i < se->num_termlists; i++) { - tl = termlist_highscore(se->termlists[i].termlist, num); - break; + tname = se->termlists[i].name; + if (!strcmp(names[j], tname) || !strcmp(names[j], "*")) + { + struct termlist_score **p = 0; + int len; + + wrbuf_puts(c->wrbuf, "wrbuf, tname); + wrbuf_puts(c->wrbuf, "\">\n"); + must_generate_empty = 0; + + p = termlist_highscore(se->termlists[i].termlist, &len, + nmem_tmp); + if (p) + { + int i; + for (i = 0; i < len && i < num; i++) + { + // prevent sending empty term elements + if (!p[i]->display_term || !p[i]->display_term[0]) + continue; + + wrbuf_puts(c->wrbuf, ""); + wrbuf_puts(c->wrbuf, ""); + wrbuf_xmlputs(c->wrbuf, p[i]->display_term); + wrbuf_puts(c->wrbuf, ""); + + wrbuf_printf(c->wrbuf, + "%d", + p[i]->frequency); + wrbuf_puts(c->wrbuf, "\n"); + } + } + wrbuf_puts(c->wrbuf, "\n"); + } + } + tname = "xtargets"; + if (!strcmp(names[j], tname) || !strcmp(names[j], "*")) + { + wrbuf_puts(c->wrbuf, "wrbuf, tname); + wrbuf_puts(c->wrbuf, "\">\n"); + + targets_termlist_nb(c->wrbuf, se, num, c->nmem, version); + wrbuf_puts(c->wrbuf, "\n"); + must_generate_empty = 0; } - session_leave(se); - return tl; + if (must_generate_empty) + { + wrbuf_puts(c->wrbuf, "wrbuf, names[j]); + wrbuf_puts(c->wrbuf, "\"/>\n"); + } + } + session_leave(se, "perform_termlist"); + nmem_destroy(nmem_tmp); } #ifdef MISSING_HEADERS @@ -800,7 +1222,7 @@ void report_nmem_stats(void) nmem_get_memory_in_use(&in_use); nmem_get_memory_free(&is_free); - yaz_log(YLOG_LOG, "nmem stat: use=%ld free=%ld", + yaz_log(YLOG_LOG, "nmem stat: use=%ld free=%ld", (long) in_use, (long) is_free); } #endif @@ -809,12 +1231,14 @@ struct record_cluster *show_single_start(struct session *se, const char *id, struct record_cluster **prev_r, struct record_cluster **next_r) { - struct record_cluster *r; + struct record_cluster *r = 0; - session_enter(se); - reclist_enter(se->reclist); + session_enter(se, "show_single_start"); *prev_r = 0; *next_r = 0; + reclist_limit(se->reclist, se, 1); + + reclist_enter(se->reclist); while ((r = reclist_read_record(se->reclist))) { if (!strcmp(r->recid, id)) @@ -826,81 +1250,153 @@ struct record_cluster *show_single_start(struct session *se, const char *id, } reclist_leave(se->reclist); if (!r) - session_leave(se); + session_leave(se, "show_single_start"); return r; } void show_single_stop(struct session *se, struct record_cluster *rec) { - session_leave(se); + session_leave(se, "show_single_stop"); +} + + +int session_fetch_more(struct session *se) +{ + struct client_list *l; + int ret = 0; + + for (l = se->clients_active; l; l = l->next) + { + struct client *cl = l->client; + if (client_get_state(cl) == Client_Idle) + { + if (client_fetch_more(cl)) + { + session_log(se, YLOG_LOG, "%s: more to fetch", + client_get_id(cl)); + ret = 1; + } + else + { + session_log(se, YLOG_LOG, "%s: no more to fetch", + client_get_id(cl)); + } + } + else + { + session_log(se, YLOG_LOG, "%s: no fetch due to state=%s", + client_get_id(cl), client_get_state_str(cl)); + } + + } + return ret; } struct record_cluster **show_range_start(struct session *se, - struct reclist_sortparms *sp, - int start, int *num, int *total, Odr_int *sumhits) + struct reclist_sortparms *sp, + int start, int *num, int *total, + Odr_int *sumhits, Odr_int *approx_hits, + void (*show_records_ready)(void *data), + struct http_channel *chan) { - struct record_cluster **recs; + struct record_cluster **recs = 0; struct reclist_sortparms *spp; + struct client_list *l; int i; -#if USE_TIMING +#if USE_TIMING yaz_timing_t t = yaz_timing_create(); #endif - session_enter(se); - recs = nmem_malloc(se->nmem, *num * sizeof(struct record_cluster *)); - if (!se->relevance) - { - *num = 0; - *total = 0; - *sumhits = 0; - recs = 0; - } - else + session_enter(se, "show_range_start"); + *sumhits = 0; + *approx_hits = 0; + *total = 0; + reclist_limit(se->reclist, se, 0); + if (se->relevance) { for (spp = sp; spp; spp = spp->next) - if (spp->type == Metadata_sortkey_relevance) + if (spp->type == Metadata_type_relevance) { relevance_prepare_read(se->relevance, se->reclist); break; } - reclist_sort(se->reclist, sp); - - reclist_enter(se->reclist); - *total = reclist_get_num_records(se->reclist); - *sumhits = se->total_hits; - - for (i = 0; i < start; i++) - if (!reclist_read_record(se->reclist)) - { - *num = 0; - recs = 0; - break; - } - - for (i = 0; i < *num; i++) - { - struct record_cluster *r = reclist_read_record(se->reclist); - if (!r) - { - *num = i; - break; - } + for (l = se->clients_active; l; l = l->next) { + *sumhits += client_get_hits(l->client); + *approx_hits += client_get_approximation(l->client); + } + } + reclist_sort(se->reclist, sp); + + reclist_enter(se->reclist); + *total = reclist_get_num_records(se->reclist); + + for (l = se->clients_active; l; l = l->next) + client_update_show_stat(l->client, 0); + + for (i = 0; i < start; i++) + { + struct record_cluster *r = reclist_read_record(se->reclist); + if (!r) + { + *num = 0; + break; + } + else + { + struct record *rec = r->records; + for (;rec; rec = rec->next) + client_update_show_stat(rec->client, 1); + } + } + recs = nmem_malloc(se->nmem, (*num > 0 ? *num : 1) * sizeof(*recs)); + for (i = 0; i < *num; i++) + { + struct record_cluster *r = reclist_read_record(se->reclist); + if (!r) + { + *num = i; + break; + } + else + { + struct record *rec = r->records; + for (;rec; rec = rec->next) + client_update_show_stat(rec->client, 1); recs[i] = r; } - reclist_leave(se->reclist); } + reclist_leave(se->reclist); #if USE_TIMING yaz_timing_stop(t); - yaz_log(YLOG_LOG, "show %6.5f %3.2f %3.2f", + session_log(se, YLOG_LOG, "show %6.5f %3.2f %3.2f", yaz_timing_get_real(t), yaz_timing_get_user(t), yaz_timing_get_sys(t)); yaz_timing_destroy(&t); #endif + + if (!session_fetch_more(se)) + session_log(se, YLOG_LOG, "can not fetch more"); + else + { + show_range_stop(se, recs); + session_log(se, YLOG_LOG, "fetching more in progress"); + if (session_set_watch(se, SESSION_WATCH_SHOW, + show_records_ready, chan, chan)) + { + session_log(se, YLOG_WARN, "Ignoring show block"); + session_enter(se, "show_range_start"); + } + else + { + session_log(se, YLOG_LOG, "session watch OK"); + return 0; + } + } return recs; } void show_range_stop(struct session *se, struct record_cluster **recs) { - session_leave(se); + session_leave(se, "show_range_stop"); } void statistics(struct session *se, struct statistics *stat) @@ -909,11 +1405,13 @@ void statistics(struct session *se, struct statistics *stat) int count = 0; memset(stat, 0, sizeof(*stat)); - for (l = se->clients; l; l = l->next) + stat->num_hits = 0; + for (l = se->clients_active; l; l = l->next) { struct client *cl = l->client; if (!client_get_connection(cl)) stat->num_no_connection++; + stat->num_hits += client_get_hits(cl); switch (client_get_state(cl)) { case Client_Connecting: stat->num_connecting++; break; @@ -925,7 +1423,6 @@ void statistics(struct session *se, struct statistics *stat) } count++; } - stat->num_hits = se->total_hits; stat->num_records = se->total_records; stat->num_clients = count; @@ -937,15 +1434,16 @@ static struct record_metadata *record_metadata_init( { struct record_metadata *rec_md = record_metadata_create(nmem); struct record_metadata_attr **attrp = &rec_md->attributes; - + for (; attr; attr = attr->next) { if (attr->children && attr->children->content) { - if (strcmp((const char *) attr->name, "type")) - { /* skip the "type" attribute.. Its value is already part of - the element in output (md-%s) and so repeating it here - is redundant */ + if (strcmp((const char *) attr->name, "type") + && strcmp((const char *) attr->name, "empty")) + { /* skip the "type" + "empty" attribute.. + The "Type" is already part of the element in output + (md-%s) and so repeating it here is redundant */ *attrp = nmem_malloc(nmem, sizeof(**attrp)); (*attrp)->name = nmem_strdup(nmem, (const char *) attr->name); @@ -957,16 +1455,17 @@ static struct record_metadata *record_metadata_init( } *attrp = 0; - if (type == Metadata_type_generic) + switch (type) { - char *p = nmem_strdup(nmem, value); - - p = normalize7bit_generic(p, " ,/.:(["); - - rec_md->data.text.disp = p; + case Metadata_type_generic: + case Metadata_type_skiparticle: + rec_md->data.text.disp = + normalize7bit_generic(nmem_strdup(nmem, value), " ,/.:(["); rec_md->data.text.sort = 0; - } - else if (type == Metadata_type_year || type == Metadata_type_date) + rec_md->data.text.snippet = 0; + break; + case Metadata_type_year: + case Metadata_type_date: { int first, last; int longdate = 0; @@ -979,11 +1478,37 @@ static struct record_metadata *record_metadata_init( rec_md->data.number.min = first; rec_md->data.number.max = last; } - else + break; + case Metadata_type_float: + rec_md->data.fnumber = atof(value); + break; + case Metadata_type_relevance: + case Metadata_type_position: return 0; + } return rec_md; } +static void mergekey_norm_wr(pp2_charset_fact_t charsets, + WRBUF norm_wr, const char *value) +{ + const char *norm_str; + pp2_charset_token_t prt = + pp2_charset_token_create(charsets, "mergekey"); + + pp2_charset_token_first(prt, value, 0); + while ((norm_str = pp2_charset_token_next(prt))) + { + if (*norm_str) + { + if (wrbuf_len(norm_wr)) + wrbuf_puts(norm_wr, " "); + wrbuf_puts(norm_wr, norm_str); + } + } + pp2_charset_token_destroy(prt); +} + static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, struct conf_service *service, WRBUF norm_wr) { @@ -1002,29 +1527,17 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, else if (!strcmp(name, (const char *) type)) { xmlChar *value = xmlNodeListGetString(doc, n->children, 1); - if (value) + if (value && *value) { - const char *norm_str; - pp2_relevance_token_t prt = - pp2_relevance_tokenize(service->mergekey_pct); - - pp2_relevance_first(prt, (const char *) value, 0); if (wrbuf_len(norm_wr) > 0) wrbuf_puts(norm_wr, " "); wrbuf_puts(norm_wr, name); - while ((norm_str = - pp2_relevance_token_next(prt))) - { - if (*norm_str) - { - wrbuf_puts(norm_wr, " "); - wrbuf_puts(norm_wr, norm_str); - } - } - xmlFree(value); - pp2_relevance_token_destroy(prt); + mergekey_norm_wr(service->charsets, norm_wr, + (const char *) value); no_found++; } + if (value) + xmlFree(value); } xmlFree(type); } @@ -1032,32 +1545,28 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, return no_found; } -static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, - struct conf_service *service, NMEM nmem) +static const char *get_mergekey(xmlDoc *doc, xmlNode *root, + struct client *cl, int record_no, + struct conf_service *service, NMEM nmem, + const char *session_mergekey) { char *mergekey_norm = 0; - xmlNode *root = xmlDocGetRootElement(doc); WRBUF norm_wr = wrbuf_alloc(); + xmlChar *mergekey; - /* consider mergekey from XSL first */ - xmlChar *mergekey = xmlGetProp(root, (xmlChar *) "mergekey"); - if (mergekey) + if (session_mergekey) { - const char *norm_str; - pp2_relevance_token_t prt = - pp2_relevance_tokenize(service->mergekey_pct); + int i, num = 0; + char **values = 0; + nmem_strsplit_escape2(nmem, ",", session_mergekey, &values, + &num, 1, '\\', 1); - pp2_relevance_first(prt, (const char *) mergekey, 0); - while ((norm_str = pp2_relevance_token_next(prt))) - { - if (*norm_str) - { - if (wrbuf_len(norm_wr)) - wrbuf_puts(norm_wr, " "); - wrbuf_puts(norm_wr, norm_str); - } - } - pp2_relevance_token_destroy(prt); + for (i = 0; i < num; i++) + get_mergekey_from_doc(doc, root, values[i], service, norm_wr); + } + else if ((mergekey = xmlGetProp(root, (xmlChar *) "mergekey"))) + { + mergekey_norm_wr(service->charsets, norm_wr, (const char *) mergekey); xmlFree(mergekey); } else @@ -1073,7 +1582,7 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, service, norm_wr); if (r == 0 && ser_md->mergekey == Metadata_mergekey_required) { - /* no mergekey on this one and it is required.. + /* no mergekey on this one and it is required.. Generate unique key instead */ wrbuf_rewind(norm_wr); break; @@ -1085,8 +1594,13 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, /* generate unique key if none is not generated already or is empty */ if (wrbuf_len(norm_wr) == 0) { - wrbuf_printf(norm_wr, "%s-%d", - client_get_database(cl)->database->url, record_no); + wrbuf_printf(norm_wr, "position: %s-%d", + client_get_id(cl), record_no); + } + else + { + const char *lead = "content: "; + wrbuf_insert(norm_wr, 0, lead, strlen(lead)); } if (wrbuf_len(norm_wr) > 0) mergekey_norm = nmem_strdup(nmem, wrbuf_cstr(norm_wr)); @@ -1094,7 +1608,7 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, return mergekey_norm; } -/** \brief see if metadata for pz:recordfilter exists +/** \brief see if metadata for pz:recordfilter exists \param root xml root element of normalized record \param sdb session database for client \retval 0 if there is no metadata for pz:recordfilter @@ -1103,7 +1617,7 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, If there is no pz:recordfilter defined, this function returns 1 as well. */ - + static int check_record_filter(xmlNode *root, struct session_database *sdb) { int match = 0; @@ -1124,9 +1638,15 @@ static int check_record_filter(xmlNode *root, struct session_database *sdb) if (type) { size_t len; - const char *eq = strchr(s, '~'); - if (eq) - len = eq - s; + int substring; + const char *eq; + + if ((eq = strchr(s, '='))) + substring = 0; + else if ((eq = strchr(s, '~'))) + substring = 1; + if (eq) + len = eq - s; else len = strlen(s); if (len == strlen((const char *)type) && @@ -1135,7 +1655,9 @@ static int check_record_filter(xmlNode *root, struct session_database *sdb) xmlChar *value = xmlNodeGetContent(n); if (value && *value) { - if (!eq || strstr((const char *) value, eq+1)) + if (!eq || + (substring && strstr((const char *) value, eq+1)) || + (!substring && !strcmp((const char *) value, eq + 1))) match = 1; } xmlFree(value); @@ -1147,12 +1669,37 @@ static int check_record_filter(xmlNode *root, struct session_database *sdb) return match; } - static int ingest_to_cluster(struct client *cl, xmlDoc *xdoc, xmlNode *root, int record_no, - const char *mergekey_norm); + struct record_metadata_attr *mergekey); + +static int ingest_sub_record(struct client *cl, xmlDoc *xdoc, xmlNode *root, + int record_no, NMEM nmem, + struct session_database *sdb, + struct record_metadata_attr *mergekeys) +{ + int ret = 0; + struct session *se = client_get_session(cl); + struct conf_service *service = se->service; + + insert_settings_values(sdb, xdoc, root, service); + + if (!check_record_filter(root, sdb)) + { + session_log(se, YLOG_LOG, + "Filtered out record no %d from %s", + record_no, sdb->database->id); + return 0; + } + session_enter(se, "ingest_sub_record"); + if (client_get_session(cl) == se && se->relevance) + ret = ingest_to_cluster(cl, xdoc, root, record_no, mergekeys); + session_leave(se, "ingest_sub_record"); + + return ret; +} /** \brief ingest XML record \param cl client holds the result set for record @@ -1161,206 +1708,553 @@ static int ingest_to_cluster(struct client *cl, \param nmem working NMEM \retval 0 OK \retval -1 failure + \retval -2 Filtered */ int ingest_record(struct client *cl, const char *rec, int record_no, NMEM nmem) { struct session *se = client_get_session(cl); - int ret = 0; struct session_database *sdb = client_get_database(cl); struct conf_service *service = se->service; - xmlDoc *xdoc = normalize_record(sdb, service, rec, nmem); + xmlDoc *xdoc = normalize_record(se, sdb, service, rec, nmem); + int r = 0; xmlNode *root; - const char *mergekey_norm; - + if (!xdoc) return -1; - + + if (global_parameters.dump_records) + { + session_log(se, YLOG_LOG, "Normalized record from %s", + sdb->database->id); + log_xml_doc(xdoc); + } + root = xmlDocGetRootElement(xdoc); - - if (!check_record_filter(root, sdb)) + + if (!strcmp((const char *) root->name, "cluster")) { - yaz_log(YLOG_WARN, "Filtered out record no %d from %s", record_no, - sdb->database->url); - xmlFreeDoc(xdoc); - return -1; + int no_merge_keys = 0; + int no_merge_dups = 0; + xmlNode *sroot; + struct record_metadata_attr *mk = 0; + + for (sroot = root->children; sroot; sroot = sroot->next) + if (sroot->type == XML_ELEMENT_NODE && + !strcmp((const char *) sroot->name, "record")) + { + struct record_metadata_attr **mkp; + const char *mergekey_norm = + get_mergekey(xdoc, sroot, cl, record_no, service, nmem, + se->mergekey); + if (!mergekey_norm) + { + r = -1; + break; + } + for (mkp = &mk; *mkp; mkp = &(*mkp)->next) + if (!strcmp((*mkp)->value, mergekey_norm)) + break; + if (!*mkp) + { + *mkp = (struct record_metadata_attr*) + nmem_malloc(nmem, sizeof(**mkp)); + (*mkp)->name = 0; + (*mkp)->value = nmem_strdup(nmem, mergekey_norm); + (*mkp)->next = 0; + no_merge_keys++; + } + else + no_merge_dups++; + } + if (no_merge_keys > 1 || no_merge_dups > 0) + { + yaz_log(YLOG_LOG, "Got %d mergekeys, %d dups for position %d", + no_merge_keys, no_merge_dups, record_no); + } + for (sroot = root->children; !r && sroot; sroot = sroot->next) + if (sroot->type == XML_ELEMENT_NODE && + !strcmp((const char *) sroot->name, "record")) + { + r = ingest_sub_record(cl, xdoc, sroot, record_no, nmem, sdb, + mk); + } } - - mergekey_norm = get_mergekey(xdoc, cl, record_no, service, nmem); - if (!mergekey_norm) + else if (!strcmp((const char *) root->name, "record")) { - yaz_log(YLOG_WARN, "Got no mergekey"); - xmlFreeDoc(xdoc); - return -1; + const char *mergekey_norm = + get_mergekey(xdoc, root, cl, record_no, service, nmem, + se->mergekey); + if (mergekey_norm) + { + struct record_metadata_attr *mk = (struct record_metadata_attr*) + nmem_malloc(nmem, sizeof(*mk)); + mk->name = 0; + mk->value = nmem_strdup(nmem, mergekey_norm); + mk->next = 0; + + r = ingest_sub_record(cl, xdoc, root, record_no, nmem, sdb, mk); + } + } + else + { + session_log(se, YLOG_WARN, "Bad pz root element: %s", + (const char *) root->name); + r = -1; } - session_enter(se); - if (client_get_session(cl) == se) - ret = ingest_to_cluster(cl, xdoc, root, record_no, mergekey_norm); - session_leave(se); - xmlFreeDoc(xdoc); + return r; +} + + +// struct conf_metadata *ser_md = &service->metadata[md_field_id]; +// struct record_metadata *rec_md = record->metadata[md_field_id]; +static int match_metadata_local(struct conf_service *service, + struct conf_metadata *ser_md, + struct record_metadata *rec_md0, + char **values, int num_v) +{ + int i; + struct record_metadata *rec_md = rec_md0; + WRBUF val_wr = 0; + WRBUF text_wr = wrbuf_alloc(); + for (i = 0; i < num_v; ) + { + if (rec_md) + { + if (ser_md->type == Metadata_type_year + || ser_md->type == Metadata_type_date) + { + int y = atoi(values[i]); + if (y >= rec_md->data.number.min + && y <= rec_md->data.number.max) + break; + } + else + { + if (!val_wr) + { + val_wr = wrbuf_alloc(); + mergekey_norm_wr(service->charsets, val_wr, values[i]); + } + wrbuf_rewind(text_wr); + mergekey_norm_wr(service->charsets, text_wr, + rec_md->data.text.disp); + if (!strcmp(wrbuf_cstr(val_wr), wrbuf_cstr(text_wr))) + break; + } + rec_md = rec_md->next; + } + else + { + rec_md = rec_md0; + wrbuf_destroy(val_wr); + val_wr = 0; + i++; + } + } + wrbuf_destroy(val_wr); + wrbuf_destroy(text_wr); + return i < num_v ? 1 : 0; +} + +int session_check_cluster_limit(struct session *se, struct record_cluster *rec) +{ + int i; + struct conf_service *service = se->service; + int ret = 1; + const char *name; + const char *value; + NMEM nmem_tmp = nmem_create(); + + for (i = 0; (name = facet_limits_get(se->facet_limits, i, &value)); i++) + { + int j; + for (j = 0; j < service->num_metadata; j++) + { + struct conf_metadata *md = service->metadata + j; + if (!strcmp(md->name, name) && md->limitcluster) + { + char **values = 0; + int num = 0; + int md_field_id = + conf_service_metadata_field_id(service, + md->limitcluster); + + if (md_field_id < 0) + { + ret = 0; + break; + } + + nmem_strsplit_escape2(nmem_tmp, "|", value, &values, + &num, 1, '\\', 1); + + if (!match_metadata_local(service, + &service->metadata[md_field_id], + rec->metadata[md_field_id], + values, num)) + { + ret = 0; + break; + } + } + } + } + nmem_destroy(nmem_tmp); return ret; } +// Skip record on non-zero +static int check_limit_local(struct client *cl, + struct record *record, + int record_no) +{ + int skip_record = 0; + struct session *se = client_get_session(cl); + struct conf_service *service = se->service; + NMEM nmem_tmp = nmem_create(); + struct session_database *sdb = client_get_database(cl); + int l = 0; + while (!skip_record) + { + int md_field_id; + char **values = 0; + int num_v = 0; + const char *name = + client_get_facet_limit_local(cl, sdb, &l, nmem_tmp, + &num_v, &values); + if (!name) + break; + + if (!strcmp(name, "*")) + { + for (md_field_id = 0; md_field_id < service->num_metadata; + md_field_id++) + { + if (match_metadata_local( + service, + &service->metadata[md_field_id], + record->metadata[md_field_id], + values, num_v)) + break; + } + if (md_field_id == service->num_metadata) + skip_record = 1; + } + else + { + md_field_id = conf_service_metadata_field_id(service, name); + if (md_field_id < 0) + { + skip_record = 1; + break; + } + if (!match_metadata_local( + service, + &service->metadata[md_field_id], + record->metadata[md_field_id], + values, num_v)) + { + skip_record = 1; + } + } + } + nmem_destroy(nmem_tmp); + return skip_record; +} + static int ingest_to_cluster(struct client *cl, xmlDoc *xdoc, xmlNode *root, int record_no, - const char *mergekey_norm) + struct record_metadata_attr *merge_keys) { xmlNode *n; xmlChar *type = 0; xmlChar *value = 0; - struct session_database *sdb = client_get_database(cl); struct session *se = client_get_session(cl); struct conf_service *service = se->service; - struct record *record = record_create(se->nmem, + int term_factor = 1; + struct record_cluster *cluster; + struct record_metadata **metadata0; + struct session_database *sdb = client_get_database(cl); + NMEM ingest_nmem = 0; + char **rank_values = 0; + int rank_num = 0; + struct record *record = record_create(se->nmem, service->num_metadata, service->num_sortkeys, cl, record_no); - struct record_cluster *cluster = reclist_insert(se->reclist, - service, - record, - mergekey_norm, - &se->total_merged); - if (!cluster) - return -1; - if (global_parameters.dump_records) - yaz_log(YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid, - sdb->database->url, record_no); - relevance_newrec(se->relevance, cluster); - - // now parsing XML record and adding data to cluster or record metadata + for (n = root->children; n; n = n->next) { - pp2_relevance_token_t prt; if (type) xmlFree(type); if (value) xmlFree(value); type = value = 0; - + if (n->type != XML_ELEMENT_NODE) continue; if (!strcmp((const char *) n->name, "metadata")) { struct conf_metadata *ser_md = 0; - struct conf_sortkey *ser_sk = 0; struct record_metadata **wheretoput = 0; struct record_metadata *rec_md = 0; int md_field_id = -1; - int sk_field_id = -1; - + type = xmlGetProp(n, (xmlChar *) "type"); value = xmlNodeListGetString(xdoc, n->children, 1); - - if (!type || !value || !*value) + if (!type) continue; - - md_field_id + if (!value || !*value) + { + xmlChar *empty = xmlGetProp(n, (xmlChar *) "empty"); + if (!empty) + continue; + if (value) + xmlFree(value); + value = empty; + } + md_field_id = conf_service_metadata_field_id(service, (const char *) type); if (md_field_id < 0) { if (se->number_of_warnings_unknown_metadata == 0) { - yaz_log(YLOG_WARN, + session_log(se, YLOG_WARN, "Ignoring unknown metadata element: %s", type); } se->number_of_warnings_unknown_metadata++; continue; } - + ser_md = &service->metadata[md_field_id]; - - if (ser_md->sortkey_offset >= 0){ - sk_field_id = ser_md->sortkey_offset; - ser_sk = &service->sortkeys[sk_field_id]; - } // non-merged metadata rec_md = record_metadata_init(se->nmem, (const char *) value, ser_md->type, n->properties); if (!rec_md) { - yaz_log(YLOG_WARN, "bad metadata data '%s' for element '%s'", - value, type); + session_log(se, YLOG_WARN, "bad metadata data '%s' " + "for element '%s'", value, type); continue; } + + if (ser_md->type == Metadata_type_generic) + { + WRBUF w = wrbuf_alloc(); + if (relevance_snippet(se->relevance, + (char*) value, ser_md->name, w)) + rec_md->data.text.snippet = nmem_strdup(se->nmem, + wrbuf_cstr(w)); + wrbuf_destroy(w); + } + + wheretoput = &record->metadata[md_field_id]; while (*wheretoput) wheretoput = &(*wheretoput)->next; *wheretoput = rec_md; + } + } + + if (check_limit_local(cl, record, record_no)) + { + session_log(se, YLOG_LOG, "Facet filtered out record no %d from %s", + record_no, sdb->database->id); + if (type) + xmlFree(type); + if (value) + xmlFree(value); + return -2; + } + cluster = reclist_insert(se->reclist, se->relevance, service, record, + merge_keys, &se->total_merged); + if (!cluster) + { + if (type) + xmlFree(type); + if (value) + xmlFree(value); + return 0; // complete match with existing record + } + + { + const char *use_term_factor_str = + session_setting_oneval(sdb, PZ_TERMLIST_TERM_FACTOR); + if (use_term_factor_str && use_term_factor_str[0] == '1') + { + int maxrecs = client_get_maxrecs(cl); + int hits = (int) client_get_hits(cl); + term_factor = MAX(hits, maxrecs) / MAX(1, maxrecs); + assert(term_factor >= 1); + session_log(se, YLOG_DEBUG, "Using term factor: %d (%d / %d)", + term_factor, MAX(hits, maxrecs), MAX(1, maxrecs)); + } + } + + if (global_parameters.dump_records) + session_log(se, YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid, + sdb->database->id, record_no); + + // original metadata, to check if first existence of a field + metadata0 = xmalloc(sizeof(*metadata0) * service->num_metadata); + memcpy(metadata0, cluster->metadata, + sizeof(*metadata0) * service->num_metadata); + + ingest_nmem = nmem_create(); + if (se->rank) + { + yaz_log(YLOG_LOG, "local in sort : %s", se->rank); + nmem_strsplit_escape2(ingest_nmem, ",", se->rank, &rank_values, + &rank_num, 1, '\\', 1); + } + + // now parsing XML record and adding data to cluster or record metadata + for (n = root->children; n; n = n->next) + { + pp2_charset_token_t prt; + if (type) + xmlFree(type); + if (value) + xmlFree(value); + type = value = 0; + + if (n->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) n->name, "metadata")) + { + struct conf_metadata *ser_md = 0; + struct conf_sortkey *ser_sk = 0; + struct record_metadata **wheretoput = 0; + struct record_metadata *rec_md = 0; + int md_field_id = -1; + int sk_field_id = -1; + const char *rank = 0; + xmlChar *xml_rank = 0; + + type = xmlGetProp(n, (xmlChar *) "type"); + value = xmlNodeListGetString(xdoc, n->children, 1); + + if (!type || !value || !*value) + continue; + + md_field_id + = conf_service_metadata_field_id(service, (const char *) type); + if (md_field_id < 0) + continue; + + ser_md = &service->metadata[md_field_id]; + + if (ser_md->sortkey_offset >= 0) + { + sk_field_id = ser_md->sortkey_offset; + ser_sk = &service->sortkeys[sk_field_id]; + } // merged metadata rec_md = record_metadata_init(se->nmem, (const char *) value, ser_md->type, 0); - wheretoput = &cluster->metadata[md_field_id]; - // and polulate with data: - // assign cluster or record based on merge action - if (ser_md->merge == Metadata_merge_unique) + // see if the field was not in cluster already (from beginning) + + if (!rec_md) + continue; + + if (rank_num) { - struct record_metadata *mnode; - for (mnode = *wheretoput; mnode; mnode = mnode->next) - if (!strcmp((const char *) mnode->data.text.disp, - rec_md->data.text.disp)) + int i; + for (i = 0; i < rank_num; i++) + { + const char *val = rank_values[i]; + const char *cp = strchr(val, '='); + if (!cp) + continue; + if ((cp - val) == strlen((const char *) type) + && !memcmp(val, type, cp - val)) + { + rank = cp + 1; break; - if (!mnode) + } + } + } + else + { + xml_rank = xmlGetProp(n, (xmlChar *) "rank"); + rank = xml_rank ? (const char *) xml_rank : ser_md->rank; + } + + wheretoput = &cluster->metadata[md_field_id]; + + if (ser_md->merge == Metadata_merge_first) + { + if (!metadata0[md_field_id]) { - rec_md->next = *wheretoput; + while (*wheretoput) + wheretoput = &(*wheretoput)->next; *wheretoput = rec_md; } } + else if (ser_md->merge == Metadata_merge_unique) + { + while (*wheretoput) + { + if (!strcmp((const char *) (*wheretoput)->data.text.disp, + rec_md->data.text.disp)) + break; + wheretoput = &(*wheretoput)->next; + } + if (!*wheretoput) + *wheretoput = rec_md; + } else if (ser_md->merge == Metadata_merge_longest) { - if (!*wheretoput - || strlen(rec_md->data.text.disp) + if (!*wheretoput + || strlen(rec_md->data.text.disp) > strlen((*wheretoput)->data.text.disp)) { *wheretoput = rec_md; if (ser_sk) { const char *sort_str = 0; - int skip_article = - ser_sk->type == Metadata_sortkey_skiparticle; + int skip_article = + ser_sk->type == Metadata_type_skiparticle; if (!cluster->sortkeys[sk_field_id]) - cluster->sortkeys[sk_field_id] = - nmem_malloc(se->nmem, + cluster->sortkeys[sk_field_id] = + nmem_malloc(se->nmem, sizeof(union data_types)); - - prt = pp2_relevance_tokenize(service->sort_pct); - pp2_relevance_first(prt, rec_md->data.text.disp, - skip_article); + prt = + pp2_charset_token_create(service->charsets, "sort"); + + pp2_charset_token_first(prt, rec_md->data.text.disp, + skip_article); + + pp2_charset_token_next(prt); - pp2_relevance_token_next(prt); - sort_str = pp2_get_sort(prt); - - cluster->sortkeys[sk_field_id]->text.disp = + + cluster->sortkeys[sk_field_id]->text.disp = rec_md->data.text.disp; if (!sort_str) { sort_str = rec_md->data.text.disp; - yaz_log(YLOG_WARN, + session_log(se, YLOG_WARN, "Could not make sortkey. Bug #1858"); } - cluster->sortkeys[sk_field_id]->text.sort = + cluster->sortkeys[sk_field_id]->text.sort = nmem_strdup(se->nmem, sort_str); -#if 0 - yaz_log(YLOG_LOG, "text disp=%s", - cluster->sortkeys[sk_field_id]->text.disp); - yaz_log(YLOG_LOG, "text sort=%s", - cluster->sortkeys[sk_field_id]->text.sort); -#endif - pp2_relevance_token_destroy(prt); + pp2_charset_token_destroy(prt); } } } else if (ser_md->merge == Metadata_merge_all) { - rec_md->next = *wheretoput; + while (*wheretoput) + wheretoput = &(*wheretoput)->next; *wheretoput = rec_md; } else if (ser_md->merge == Metadata_merge_range) @@ -1369,7 +2263,7 @@ static int ingest_to_cluster(struct client *cl, { *wheretoput = rec_md; if (ser_sk) - cluster->sortkeys[sk_field_id] + cluster->sortkeys[sk_field_id] = &rec_md->data; } else @@ -1383,33 +2277,34 @@ static int ingest_to_cluster(struct client *cl, } } - - // ranking of _all_ fields enabled ... - if (ser_md->rank) - relevance_countwords(se->relevance, cluster, - (char *) value, ser_md->rank, - ser_md->name); - + // ranking of _all_ fields enabled ... + if (rank) + { + relevance_countwords(se->relevance, cluster, + (char *) value, rank, ser_md->name); + } // construct facets ... unless the client already has reported them if (ser_md->termlist && !client_has_facet(cl, (char *) type)) { - if (ser_md->type == Metadata_type_year) { char year[64]; sprintf(year, "%d", rec_md->data.number.max); - add_facet(se, (char *) type, year, 1); + + add_facet(se, (char *) type, year, term_factor); if (rec_md->data.number.max != rec_md->data.number.min) { sprintf(year, "%d", rec_md->data.number.min); - add_facet(se, (char *) type, year, 1); + add_facet(se, (char *) type, year, term_factor); } } else - add_facet(se, (char *) type, (char *) value, 1); + add_facet(se, (char *) type, (char *) value, term_factor); } // cleaning up + if (xml_rank) + xmlFree(xml_rank); xmlFree(type); xmlFree(value); type = value = 0; @@ -1417,7 +2312,7 @@ static int ingest_to_cluster(struct client *cl, else { if (se->number_of_warnings_unknown_elements == 0) - yaz_log(YLOG_WARN, + session_log(se, YLOG_WARN, "Unexpected element in internal record: %s", n->name); se->number_of_warnings_unknown_elements++; } @@ -1427,12 +2322,26 @@ static int ingest_to_cluster(struct client *cl, if (value) xmlFree(value); + nmem_destroy(ingest_nmem); + xfree(metadata0); relevance_donerecord(se->relevance, cluster); se->total_records++; return 0; } +void session_log(struct session *s, int level, const char *fmt, ...) +{ + char buf[1024]; + va_list ap; + va_start(ap, fmt); + + yaz_vsnprintf(buf, sizeof(buf)-30, fmt, ap); + yaz_log(level, "Session %u: %s", s ? s->session_id : 0, buf); + + va_end(ap); +} + /* * Local variables: * c-basic-offset: 4