X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Fsession.c;h=3dcf40f28d3bc19138705edc666fcf94fbe28fdb;hb=85b1f355522cc620452552d76fd517f089c98ab2;hp=4049360b248ea23eb2debe616039ffaedd8c9fe1;hpb=3371e5ba3a8d62b6edbb338984ac333bec6df01b;p=pazpar2-moved-to-github.git diff --git a/src/session.c b/src/session.c index 4049360..3dcf40f 100644 --- a/src/session.c +++ b/src/session.c @@ -1,5 +1,5 @@ /* This file is part of Pazpar2. - Copyright (C) 2006-2012 Index Data + Copyright (C) Index Data Pazpar2 is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -56,7 +56,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #include #include -#include +#include #define USE_TIMING 0 #if USE_TIMING @@ -78,14 +78,12 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include -#define TERMLIST_HIGH_SCORE 25 - #define MAX_CHUNK 15 #define MAX(a,b) ((a)>(b)?(a):(b)) // Note: Some things in this structure will eventually move to configuration -struct parameters global_parameters = +struct parameters global_parameters = { 0, // dump_records 0, // debug_mode @@ -100,7 +98,6 @@ struct client_list { /* session counting (1) , disable client counting (0) */ static YAZ_MUTEX g_session_mutex = 0; static int no_sessions = 0; -static int no_session_total = 0; static int session_use(int delta) { @@ -109,11 +106,10 @@ static int session_use(int delta) yaz_mutex_create(&g_session_mutex); yaz_mutex_enter(g_session_mutex); no_sessions += delta; - if (delta > 0) - no_session_total += delta; sessions = no_sessions; yaz_mutex_leave(g_session_mutex); - yaz_log(YLOG_DEBUG, "%s sessions=%d", delta == 0 ? "" : (delta > 0 ? "INC" : "DEC"), no_sessions); + yaz_log(YLOG_DEBUG, "%s sessions=%d", delta == 0 ? "" : + (delta > 0 ? "INC" : "DEC"), no_sessions); return sessions; } @@ -122,17 +118,6 @@ int sessions_count(void) return session_use(0); } -int session_count_total(void) -{ - int total = 0; - if (!g_session_mutex) - return 0; - yaz_mutex_enter(g_session_mutex); - total = no_session_total; - yaz_mutex_leave(g_session_mutex); - return total; -} - static void log_xml_doc(xmlDoc *doc) { FILE *lf = yaz_log_file(); @@ -151,40 +136,33 @@ static void log_xml_doc(xmlDoc *doc) xmlFree(result); } -static void session_enter(struct session *s) +static void session_enter(struct session *s, const char *caller) { + if (caller) + session_log(s, YLOG_DEBUG, "Session lock by %s", caller); yaz_mutex_enter(s->session_mutex); } -static void session_leave(struct session *s) +static void session_leave(struct session *s, const char *caller) { yaz_mutex_leave(s->session_mutex); + if (caller) + session_log(s, YLOG_DEBUG, "Session unlock by %s", caller); } -static void session_normalize_facet(struct session *s, const char *type, - const char *value, - WRBUF display_wrbuf, - WRBUF facet_wrbuf) +static int run_icu(struct session *s, const char *icu_chain_id, + const char *value, + WRBUF norm_wr, WRBUF disp_wr) { - struct conf_service *service = s->service; - pp2_charset_token_t prt; const char *facet_component; - int i; - const char *icu_chain_id = 0; - - for (i = 0; i < service->num_metadata; i++) - if (!strcmp((service->metadata + i)->name, type)) - icu_chain_id = (service->metadata + i)->facetrule; - if (!icu_chain_id) - icu_chain_id = "facet"; - prt = pp2_charset_token_create(service->charsets, icu_chain_id); + struct conf_service *service = s->service; + pp2_charset_token_t prt = + pp2_charset_token_create(service->charsets, icu_chain_id); if (!prt) { - yaz_log(YLOG_FATAL, "Unknown ICU chain '%s' for facet of type '%s'", - icu_chain_id, type); - wrbuf_destroy(facet_wrbuf); - wrbuf_destroy(display_wrbuf); - return; + session_log(s, YLOG_FATAL, + "Unknown ICU chain '%s'", icu_chain_id); + return 0; } pp2_charset_token_first(prt, value, 0); while ((facet_component = pp2_charset_token_next(prt))) @@ -192,19 +170,37 @@ static void session_normalize_facet(struct session *s, const char *type, const char *display_component; if (*facet_component) { - if (wrbuf_len(facet_wrbuf)) - wrbuf_puts(facet_wrbuf, " "); - wrbuf_puts(facet_wrbuf, facet_component); + if (wrbuf_len(norm_wr)) + wrbuf_puts(norm_wr, " "); + wrbuf_puts(norm_wr, facet_component); } display_component = pp2_get_display(prt); if (display_component) { - if (wrbuf_len(display_wrbuf)) - wrbuf_puts(display_wrbuf, " "); - wrbuf_puts(display_wrbuf, display_component); + if (wrbuf_len(disp_wr)) + wrbuf_puts(disp_wr, " "); + wrbuf_puts(disp_wr, display_component); } } pp2_charset_token_destroy(prt); + return 1; +} + +static void session_normalize_facet(struct session *s, + const char *type, const char *value, + WRBUF display_wrbuf, WRBUF facet_wrbuf) +{ + struct conf_service *service = s->service; + int i; + const char *icu_chain_id = 0; + + for (i = 0; i < service->num_metadata; i++) + if (!strcmp((service->metadata + i)->name, type)) + icu_chain_id = (service->metadata + i)->facetrule; + if (!icu_chain_id) + icu_chain_id = "facet"; + + run_icu(s, icu_chain_id, value, facet_wrbuf, display_wrbuf); } void add_facet(struct session *s, const char *type, const char *value, int count) @@ -213,33 +209,21 @@ void add_facet(struct session *s, const char *type, const char *value, int count WRBUF display_wrbuf = wrbuf_alloc(); session_normalize_facet(s, type, value, display_wrbuf, facet_wrbuf); - + if (wrbuf_len(facet_wrbuf)) { - int i; - for (i = 0; i < s->num_termlists; i++) - if (!strcmp(s->termlists[i].name, type)) + struct named_termlist **tp = &s->termlists; + for (; (*tp); tp = &(*tp)->next) + if (!strcmp((*tp)->name, type)) break; - if (i == s->num_termlists) + if (!*tp) { - if (i == SESSION_MAX_TERMLISTS) - { - session_log(s, YLOG_FATAL, "Too many termlists"); - wrbuf_destroy(facet_wrbuf); - wrbuf_destroy(display_wrbuf); - return; - } - - s->termlists[i].name = nmem_strdup(s->nmem, type); - s->termlists[i].termlist - = termlist_create(s->nmem, TERMLIST_HIGH_SCORE); - s->num_termlists = i + 1; + *tp = nmem_malloc(s->nmem, sizeof(**tp)); + (*tp)->name = nmem_strdup(s->nmem, type); + (*tp)->termlist = termlist_create(s->nmem); + (*tp)->next = 0; } - -#if 0 - session_log(s, YLOG_DEBUG, "Facets for %s: %s norm:%s (%d)", type, value, wrbuf_cstr(facet_wrbuf), count); -#endif - termlist_insert(s->termlists[i].termlist, wrbuf_cstr(display_wrbuf), + termlist_insert((*tp)->termlist, wrbuf_cstr(display_wrbuf), wrbuf_cstr(facet_wrbuf), count); } wrbuf_destroy(facet_wrbuf); @@ -256,8 +240,7 @@ static xmlDoc *record_to_xml(struct session *se, if (!rdoc) { - session_log(se, YLOG_FATAL, "Non-wellformed XML received from %s", - db->id); + session_log(se, YLOG_WARN, "Non-wellformed XML"); return 0; } @@ -311,7 +294,8 @@ static void insert_settings_parameters(struct session_database *sdb, // Add static values from session database settings if applicable static void insert_settings_values(struct session_database *sdb, xmlDoc *doc, - struct conf_service *service) + xmlNode *root, + struct conf_service *service) { int i; @@ -326,8 +310,7 @@ static void insert_settings_values(struct session_database *sdb, xmlDoc *doc, const char *val = session_setting_oneval(sdb, offset); if (val) { - xmlNode *r = xmlDocGetRootElement(doc); - xmlNode *n = xmlNewTextChild(r, 0, (xmlChar *) "metadata", + xmlNode *n = xmlNewTextChild(root, 0, (xmlChar *) "metadata", (xmlChar *) val); xmlSetProp(n, (xmlChar *) "type", (xmlChar *) md->name); } @@ -345,24 +328,12 @@ static xmlDoc *normalize_record(struct session *se, if (rdoc) { char *parms[MAX_XSLT_ARGS*2+1]; - + insert_settings_parameters(sdb, service, parms, nmem); - + if (normalize_record_transform(sdb->map, &rdoc, (const char **)parms)) { - session_log(se, YLOG_WARN, "Normalize failed from %s", - sdb->database->id); - } - else - { - insert_settings_values(sdb, rdoc, service); - - if (global_parameters.dump_records) - { - session_log(se, YLOG_LOG, "Normalized record from %s", - sdb->database->id); - log_xml_doc(rdoc); - } + session_log(se, YLOG_WARN, "Normalize failed"); } } return rdoc; @@ -413,10 +384,10 @@ static int prepare_map(struct session *se, struct session_database *sdb) const char *s; if (sdb->settings[PZ_XSLT] && - (s = session_setting_oneval(sdb, PZ_XSLT))) + (s = session_setting_oneval(sdb, PZ_XSLT))) { char auto_stylesheet[256]; - + if (!strcmp(s, "auto")) { const char *request_syntax = session_setting_oneval( @@ -461,31 +432,31 @@ static void session_watch_cancel(void *data, struct http_channel *c, } // set watch. Returns 0=OK, -1 if watch is already set -int session_set_watch(struct session *s, int what, +int session_set_watch(struct session *s, int what, session_watchfun fun, void *data, struct http_channel *chan) { int ret; - session_enter(s); + session_enter(s, "session_set_watch"); if (s->watchlist[what].fun) ret = -1; else { - + s->watchlist[what].fun = fun; s->watchlist[what].data = data; s->watchlist[what].obs = http_add_observer(chan, &s->watchlist[what], session_watch_cancel); ret = 0; } - session_leave(s); + session_leave(s, "session_set_watch"); return ret; } void session_alert_watch(struct session *s, int what) { assert(s); - session_enter(s); + session_enter(s, "session_alert_watch"); if (s->watchlist[what].fun) { /* our watch is no longer associated with http_channel */ @@ -502,13 +473,13 @@ void session_alert_watch(struct session *s, int what) s->watchlist[what].data = 0; s->watchlist[what].obs = 0; - session_leave(s); + session_leave(s, "session_alert_watch"); session_log(s, YLOG_DEBUG, "Alert Watch: %d calling function: %p", what, fun); fun(data); } else - session_leave(s); + session_leave(s,"session_alert_watch"); } //callback for grep_databases @@ -548,16 +519,16 @@ static void session_reset_active_clients(struct session *se, { struct client_list *l; - session_enter(se); + session_enter(se, "session_reset_active_clients"); l = se->clients_active; se->clients_active = new_list; - session_leave(se); + session_leave(se, "session_reset_active_clients"); while (l) { struct client_list *l_next = l->next; - client_lock(l->client); + client_lock(l->client); client_set_session(l->client, 0); /* mark client inactive */ client_unlock(l->client); @@ -572,10 +543,10 @@ static void session_remove_cached_clients(struct session *se) session_reset_active_clients(se, 0); - session_enter(se); + session_enter(se, "session_remove_cached_clients"); l = se->clients_cached; se->clients_cached = 0; - session_leave(se); + session_leave(se, "session_remove_cached_clients"); while (l) { @@ -591,7 +562,7 @@ static void session_remove_cached_clients(struct session *se) } // Associates a set of clients with a session; -// Note: Session-databases represent databases with per-session +// Note: Session-databases represent databases with per-session // setting overrides static int select_targets(struct session *se, const char *filter) { @@ -622,73 +593,160 @@ int session_is_preferred_clients_ready(struct session *s) return res == 0; } -static void session_clear_set(struct session *se, - const char *sort_field, int increasing) +static void session_clear_set(struct session *se, struct reclist_sortparms *sp) { reclist_destroy(se->reclist); - se->reclist = 0; if (nmem_total(se->nmem)) session_log(se, YLOG_DEBUG, "NMEN operation usage %zd", nmem_total(se->nmem)); nmem_reset(se->nmem); se->total_records = se->total_merged = 0; - se->num_termlists = 0; - + se->termlists = 0; + relevance_clear(se->relevance); + /* reset list of sorted results and clear to relevance search */ se->sorted_results = nmem_malloc(se->nmem, sizeof(*se->sorted_results)); - se->sorted_results->field = nmem_strdup(se->nmem, sort_field); - se->sorted_results->increasing = increasing; + se->sorted_results->name = nmem_strdup(se->nmem, sp->name); + se->sorted_results->increasing = sp->increasing; + se->sorted_results->type = sp->type; se->sorted_results->next = 0; - + + session_log(se, YLOG_DEBUG, "clear_set session_sort: field=%s increasing=%d type=%d configured", + sp->name, sp->increasing, sp->type); + se->reclist = reclist_create(se->nmem); } -void session_sort(struct session *se, const char *field, int increasing, - int clear_set) +void session_sort(struct session *se, struct reclist_sortparms *sp, + const char *mergekey, const char *rank) { - struct session_sorted_results *sr; struct client_list *l; + const char *field = sp->name; + int increasing = sp->increasing; + int type = sp->type; + int clients_research = 0; - session_enter(se); + session_enter(se, "session_sort"); + session_log(se, YLOG_DEBUG, "session_sort field=%s increasing=%d type=%d", + field, increasing, type); - yaz_log(YLOG_LOG, "session_sort field=%s", field); - if (clear_set) + if (rank && (!se->rank || strcmp(se->rank, rank))) { - session_clear_set(se, field, increasing); + /* new rank must research/reingest anyway */ + assert(rank); + xfree(se->rank); + se->rank = *rank ? xstrdup(rank) : 0; + clients_research = 1; + session_log(se, YLOG_DEBUG, "session_sort: new rank = %s", + rank); } - else + if (mergekey && (!se->mergekey || strcmp(se->mergekey, mergekey))) + { + /* new mergekey must research/reingest anyway */ + assert(mergekey); + xfree(se->mergekey); + se->mergekey = *mergekey ? xstrdup(mergekey) : 0; + clients_research = 1; + session_log(se, YLOG_DEBUG, "session_sort: new mergekey = %s", + mergekey); + } + if (clients_research == 0) { - /* see if we already have sorted for this critieria */ + struct reclist_sortparms *sr; for (sr = se->sorted_results; sr; sr = sr->next) - { - if (!strcmp(field, sr->field) && increasing == sr->increasing) + if (!reclist_sortparms_cmp(sr, sp)) break; - } if (sr) { - session_log(se, YLOG_DEBUG, "search_sort: field=%s increasing=%d already fetched", - field, increasing); - session_leave(se); + session_log(se, YLOG_DEBUG, "session_sort: field=%s increasing=%d type=%d already fetched", + field, increasing, type); + session_leave(se, "session_sort"); return; } - session_log(se, YLOG_DEBUG, "search_sort: field=%s increasing=%d must fetch", - field, increasing); - sr = nmem_malloc(se->nmem, sizeof(*sr)); - sr->field = nmem_strdup(se->nmem, field); + } + session_log(se, YLOG_DEBUG, "session_sort: field=%s increasing=%d type=%d must fetch", + field, increasing, type); + + // We need to reset reclist on every sort that changes the records, not just for position + // So if just one client requires new searching, we need to clear set. + // Ask each of the client if sorting requires re-search due to native sort + // If it does it will require us to + for (l = se->clients_active; l; l = l->next) + { + struct client *cl = l->client; + // Assume no re-search is required. + client_parse_init(cl, 1); + clients_research += client_parse_sort(cl, sp); + } + if (!clients_research || se->clients_starting) + { + // A new sorting based on same record set + struct reclist_sortparms *sr = nmem_malloc(se->nmem, sizeof(*sr)); + sr->name = nmem_strdup(se->nmem, field); sr->increasing = increasing; + sr->type = type; sr->next = se->sorted_results; se->sorted_results = sr; + session_log(se, YLOG_DEBUG, "session_sort: no research/ingesting done"); + session_leave(se, "session_sort"); + } + else + { + se->clients_starting = 1; + session_log(se, YLOG_DEBUG, + "session_sort: reset results due to %d clients researching", + clients_research); + session_clear_set(se, sp); + session_log(se, YLOG_DEBUG, "Re- search/ingesting for clients due to change in sort order"); + + session_leave(se, "session_sort"); + for (l = se->clients_active; l; l = l->next) + { + struct client *cl = l->client; + if (client_get_state(cl) == Client_Connecting || + client_get_state(cl) == Client_Idle || + client_get_state(cl) == Client_Working) { + client_start_search(cl); + } + else + { + session_log(se, YLOG_DEBUG, + "session_sort: %s: No re-start/ingest in show. " + "Wrong client state: %d", + client_get_id(cl), client_get_state(cl)); + } + } + session_enter(se, "session_sort"); + se->clients_starting = 0; + session_leave(se, "session_sort"); + } +} + +void session_stop(struct session *se) +{ + struct client_list *l; + session_enter(se, "session_stop1"); + if (se->clients_starting) + { + session_leave(se, "session_stop1"); + return; } - + se->clients_starting = 1; + session_leave(se, "session_stop1"); + + session_alert_watch(se, SESSION_WATCH_SHOW); + session_alert_watch(se, SESSION_WATCH_BYTARGET); + session_alert_watch(se, SESSION_WATCH_TERMLIST); + session_alert_watch(se, SESSION_WATCH_SHOW_PREF); + for (l = se->clients_active; l; l = l->next) { struct client *cl = l->client; - if (client_get_state(cl) == Client_Connecting || - client_get_state(cl) == Client_Idle || - client_get_state(cl) == Client_Working) - client_start_search(cl); + client_stop(cl); } - session_leave(se); + session_enter(se, "session_stop2"); + se->clients_starting = 0; + session_leave(se, "session_stop2"); } enum pazpar2_error_code session_search(struct session *se, @@ -698,90 +756,104 @@ enum pazpar2_error_code session_search(struct session *se, const char *filter, const char *limit, const char **addinfo, - const char *sort_field, int increasing) + const char **addinfo2, + struct reclist_sortparms *sp, + const char *mergekey, + const char *rank) { int live_channels = 0; int no_working = 0; int no_failed_query = 0; int no_failed_limit = 0; - struct client_list *l, *l0; - struct timeval tval; - facet_limits_t facet_limits; + struct client_list *l; session_log(se, YLOG_DEBUG, "Search"); *addinfo = 0; - if (se->settings_modified) + session_enter(se, "session_search0"); + if (se->clients_starting) + { + session_leave(se, "session_search0"); + return PAZPAR2_NO_ERROR; + } + se->clients_starting = 1; + session_leave(se, "session_search0"); + + if (se->settings_modified) { session_remove_cached_clients(se); + } else session_reset_active_clients(se, 0); - - session_enter(se); + + session_enter(se, "session_search"); se->settings_modified = 0; - session_clear_set(se, sort_field, increasing); + + if (mergekey) + { + xfree(se->mergekey); + se->mergekey = *mergekey ? xstrdup(mergekey) : 0; + } + if (rank) + { + xfree(se->rank); + se->rank = *rank ? xstrdup(rank) : 0; + } + + session_clear_set(se, sp); relevance_destroy(&se->relevance); live_channels = select_targets(se, filter); if (!live_channels) { - session_leave(se); + session_leave(se, "session_search"); + se->clients_starting = 0; return PAZPAR2_NO_TARGETS; } - yaz_gettimeofday(&tval); - - tval.tv_sec += 5; - - facet_limits = facet_limits_create(limit); - if (!facet_limits) + facet_limits_destroy(se->facet_limits); + se->facet_limits = facet_limits_create(limit); + if (!se->facet_limits) { *addinfo = "limit"; - session_leave(se); + session_leave(se, "session_search"); + se->clients_starting = 0; return PAZPAR2_MALFORMED_PARAMETER_VALUE; } - l0 = se->clients_active; - se->clients_active = 0; - session_leave(se); + session_leave(se, "session_search"); + + session_alert_watch(se, SESSION_WATCH_SHOW); + session_alert_watch(se, SESSION_WATCH_BYTARGET); + session_alert_watch(se, SESSION_WATCH_TERMLIST); + session_alert_watch(se, SESSION_WATCH_SHOW_PREF); - for (l = l0; l; l = l->next) + for (l = se->clients_active; l; l = l->next) { int parse_ret; struct client *cl = l->client; - + client_parse_init(cl, 1); if (prepare_map(se, client_get_database(cl)) < 0) continue; - parse_ret = client_parse_query(cl, query, facet_limits, startrecs, - maxrecs); + parse_ret = client_parse_query(cl, query, se->facet_limits, addinfo2); if (parse_ret == -1) no_failed_query++; else if (parse_ret == -2) no_failed_limit++; + else if (parse_ret < 0) + no_working++; /* other error, such as bad CCL map */ else { - int r = - client_prep_connection(cl, se->service->z3950_operation_timeout, - se->service->z3950_session_timeout, - se->service->server->iochan_man, - &tval); - if (parse_ret == 1 && r == 2) - { - session_log(se, YLOG_LOG, "client %s REUSE result", client_get_id(cl)); - client_reingest(cl); - } - else if (r) - { - session_log(se, YLOG_LOG, "client %s NEW search", client_get_id(cl)); - client_start_search(cl); - } + client_parse_range(cl, startrecs, maxrecs); + client_parse_sort(cl, sp); + client_start_search(cl); no_working++; } } - facet_limits_destroy(facet_limits); - session_reset_active_clients(se, l0); - + session_enter(se, "session_search2"); + se->clients_starting = 0; + session_leave(se, "session_search2"); if (no_working == 0) { if (no_failed_query > 0) @@ -797,7 +869,6 @@ enum pazpar2_error_code session_search(struct session *se, else return PAZPAR2_NO_TARGETS; } - session_log(se, YLOG_LOG, "session_start_search done"); return PAZPAR2_NO_ERROR; } @@ -809,7 +880,7 @@ static void session_init_databases_fun(void *context, struct database *db) int i; new->database = db; - + new->map = 0; assert(db->settings); new->settings = nmem_malloc(se->session_nmem, @@ -840,8 +911,8 @@ void session_init_databases(struct session *se) // Probably session_init_databases_fun should be refactored instead of // called here. -static struct session_database *load_session_database(struct session *se, - char *id) +static struct session_database *load_session_database(struct session *se, + const char *id) { struct database *db = new_database_inherit_settings(id, se->session_nmem, se->service->settings); session_init_databases_fun((void*) se, db); @@ -851,8 +922,8 @@ static struct session_database *load_session_database(struct session *se, } // Find an existing session database. If not found, load it -static struct session_database *find_session_database(struct session *se, - char *id) +static struct session_database *find_session_database(struct session *se, + const char *id) { struct session_database *sdb; @@ -863,42 +934,45 @@ static struct session_database *find_session_database(struct session *se, } // Apply a session override to a database -void session_apply_setting(struct session *se, char *dbname, char *setting, - char *value) +void session_apply_setting(struct session *se, const char *dbname, + const char *name, const char *value) { - struct session_database *sdb = find_session_database(se, dbname); - struct conf_service *service = se->service; - struct setting *new = nmem_malloc(se->session_nmem, sizeof(*new)); - int offset = settings_create_offset(service, setting); - - expand_settings_array(&sdb->settings, &sdb->num_settings, offset, - se->session_nmem); - new->precedence = 0; - new->target = dbname; - new->name = setting; - new->value = value; - new->next = sdb->settings[offset]; - sdb->settings[offset] = new; - - se->settings_modified = 1; - - // Force later recompute of settings-driven data structures - // (happens when a search starts and client connections are prepared) - switch (offset) - { - case PZ_XSLT: - if (sdb->map) - { + session_enter(se, "session_apply_setting"); + { + struct session_database *sdb = find_session_database(se, dbname); + struct conf_service *service = se->service; + struct setting *s; + int offset = settings_create_offset(service, name); + + expand_settings_array(&sdb->settings, &sdb->num_settings, offset, + se->session_nmem); + // Force later recompute of settings-driven data structures + // (happens when a search starts and client connections are prepared) + if (offset == PZ_XSLT) sdb->map = 0; + se->settings_modified = 1; + for (s = sdb->settings[offset]; s; s = s->next) + if (!strcmp(s->name, name) && + dbname && s->target && !strcmp(dbname, s->target)) + break; + if (!s) + { + s = nmem_malloc(se->session_nmem, sizeof(*s)); + s->precedence = 0; + s->target = nmem_strdup(se->session_nmem, dbname); + s->name = nmem_strdup(se->session_nmem, name); + s->next = sdb->settings[offset]; + sdb->settings[offset] = s; } - break; + s->value = nmem_strdup(se->session_nmem, value); } + session_leave(se, "session_apply_setting"); } void session_destroy(struct session *se) { struct session_database *sdb; - session_log(se, YLOG_DEBUG, "Destroying"); + session_log(se, YLOG_LOG, "destroy"); session_use(-1); session_remove_cached_clients(se); @@ -907,10 +981,13 @@ void session_destroy(struct session *se) normalize_cache_destroy(se->normalize_cache); relevance_destroy(&se->relevance); reclist_destroy(se->reclist); + xfree(se->mergekey); + xfree(se->rank); if (nmem_total(se->nmem)) session_log(se, YLOG_DEBUG, "NMEN operation usage %zd", nmem_total(se->nmem)); if (nmem_total(se->session_nmem)) session_log(se, YLOG_DEBUG, "NMEN session usage %zd", nmem_total(se->session_nmem)); + facet_limits_destroy(se->facet_limits); nmem_destroy(se->nmem); service_destroy(se->service); yaz_mutex_destroy(&se->session_mutex); @@ -920,9 +997,9 @@ size_t session_get_memory_status(struct session *session) { size_t session_nmem; if (session == 0) return 0; - session_enter(session); + session_enter(session, "session_get_memory_status"); session_nmem = nmem_total(session->nmem); - session_leave(session); + session_leave(session, "session_get_memory_status"); return session_nmem; } @@ -944,14 +1021,20 @@ struct session *new_session(NMEM nmem, struct conf_service *service, session->total_records = 0; session->number_of_warnings_unknown_elements = 0; session->number_of_warnings_unknown_metadata = 0; - session->num_termlists = 0; - session->reclist = 0; + session->termlists = 0; + session->reclist = reclist_create(nmem); session->clients_active = 0; session->clients_cached = 0; session->settings_modified = 0; session->session_nmem = nmem; session->nmem = nmem_create(); session->databases = 0; + session->sorted_results = 0; + session->facet_limits = 0; + session->mergekey = 0; + session->rank = 0; + session->clients_starting = 0; + for (i = 0; i <= SESSION_WATCH_MAX; i++) { session->watchlist[i].data = 0; @@ -960,6 +1043,8 @@ struct session *new_session(NMEM nmem, struct conf_service *service, session->normalize_cache = normalize_cache_create(); session->session_mutex = 0; pazpar2_mutex_create(&session->session_mutex, tmp_str); + session_log(session, YLOG_LOG, "create"); + session_use(1); return session; } @@ -984,15 +1069,16 @@ static struct hitsbytarget *hitsbytarget_nb(struct session *se, WRBUF w = wrbuf_alloc(); const char *name = session_setting_oneval(client_get_database(cl), PZ_NAME); - res[*count].id = client_get_id(cl); res[*count].name = *name ? name : "Unknown"; res[*count].hits = client_get_hits(cl); res[*count].approximation = client_get_approximation(cl); - res[*count].records = client_get_num_records(cl); - res[*count].filtered = client_get_num_records_filtered(cl); + res[*count].records = client_get_num_records(cl, + &res[*count].filtered, + 0, 0); res[*count].diagnostic = - client_get_diagnostic(cl, &res[*count].addinfo); + client_get_diagnostic(cl, &res[*count].message, + &res[*count].addinfo); res[*count].state = client_get_state_str(cl); res[*count].connected = client_get_connection(cl) ? 1 : 0; session_settings_dump(se, client_get_database(cl), w); @@ -1009,28 +1095,11 @@ static struct hitsbytarget *hitsbytarget_nb(struct session *se, struct hitsbytarget *get_hitsbytarget(struct session *se, int *count, NMEM nmem) { struct hitsbytarget *p; - session_enter(se); + session_enter(se, "get_hitsbytarget"); p = hitsbytarget_nb(se, count, nmem); - session_leave(se); + session_leave(se, "get_hitsbytarget"); return p; } - -struct termlist_score **get_termlist_score(struct session *se, - const char *name, int *num) -{ - int i; - struct termlist_score **tl = 0; - - session_enter(se); - for (i = 0; i < se->num_termlists; i++) - if (!strcmp((const char *) se->termlists[i].name, name)) - { - tl = termlist_highscore(se->termlists[i].termlist, num); - break; - } - session_leave(se); - return tl; -} // Compares two hitsbytarget nodes by hitcount static int cmp_ht(const void *p1, const void *p2) @@ -1063,20 +1132,20 @@ static int targets_termlist_nb(WRBUF wrbuf, struct session *se, int num, { // do only print terms which have display names - + wrbuf_puts(wrbuf, "\n"); wrbuf_puts(wrbuf, ""); wrbuf_xmlputs(wrbuf, ht[i].id); wrbuf_puts(wrbuf, "\n"); - + wrbuf_puts(wrbuf, ""); if (!ht[i].name || !ht[i].name[0]) wrbuf_xmlputs(wrbuf, "NO TARGET NAME"); else wrbuf_xmlputs(wrbuf, ht[i].name); wrbuf_puts(wrbuf, "\n"); - + wrbuf_printf(wrbuf, "" ODR_INT_PRINTF "\n", ht[i].hits); @@ -1090,8 +1159,8 @@ static int targets_termlist_nb(WRBUF wrbuf, struct session *se, int num, wrbuf_puts(wrbuf, ""); wrbuf_xmlputs(wrbuf, ht[i].state); wrbuf_puts(wrbuf, "\n"); - - wrbuf_printf(wrbuf, "%d\n", + + wrbuf_printf(wrbuf, "%d\n", ht[i].diagnostic); wrbuf_puts(wrbuf, "\n"); } @@ -1101,7 +1170,7 @@ static int targets_termlist_nb(WRBUF wrbuf, struct session *se, int num, void perform_termlist(struct http_channel *c, struct session *se, const char *name, int num, int version) { - int i, j; + int j; NMEM nmem_tmp = nmem_create(); char **names; int num_names = 0; @@ -1111,16 +1180,17 @@ void perform_termlist(struct http_channel *c, struct session *se, nmem_strsplit(nmem_tmp, ",", name, &names, &num_names); - session_enter(se); + session_enter(se, "perform_termlist"); for (j = 0; j < num_names; j++) { const char *tname; int must_generate_empty = 1; /* bug 5350 */ - for (i = 0; i < se->num_termlists; i++) + struct named_termlist *t = se->termlists; + for (; t; t = t->next) { - tname = se->termlists[i].name; + tname = t->name; if (!strcmp(names[j], tname) || !strcmp(names[j], "*")) { struct termlist_score **p = 0; @@ -1131,7 +1201,7 @@ void perform_termlist(struct http_channel *c, struct session *se, wrbuf_puts(c->wrbuf, "\">\n"); must_generate_empty = 0; - p = termlist_highscore(se->termlists[i].termlist, &len); + p = termlist_highscore(t->termlist, &len, nmem_tmp); if (p) { int i; @@ -1140,14 +1210,14 @@ void perform_termlist(struct http_channel *c, struct session *se, // prevent sending empty term elements if (!p[i]->display_term || !p[i]->display_term[0]) continue; - + wrbuf_puts(c->wrbuf, ""); wrbuf_puts(c->wrbuf, ""); wrbuf_xmlputs(c->wrbuf, p[i]->display_term); wrbuf_puts(c->wrbuf, ""); - - wrbuf_printf(c->wrbuf, - "%d", + + wrbuf_printf(c->wrbuf, + "%d", p[i]->frequency); wrbuf_puts(c->wrbuf, "\n"); } @@ -1173,7 +1243,7 @@ void perform_termlist(struct http_channel *c, struct session *se, wrbuf_puts(c->wrbuf, "\"/>\n"); } } - session_leave(se); + session_leave(se, "perform_termlist"); nmem_destroy(nmem_tmp); } @@ -1185,7 +1255,7 @@ void report_nmem_stats(void) nmem_get_memory_in_use(&in_use); nmem_get_memory_free(&is_free); - yaz_log(YLOG_LOG, "nmem stat: use=%ld free=%ld", + yaz_log(YLOG_LOG, "nmem stat: use=%ld free=%ld", (long) in_use, (long) is_free); } #endif @@ -1196,107 +1266,186 @@ struct record_cluster *show_single_start(struct session *se, const char *id, { struct record_cluster *r = 0; - session_enter(se); + session_enter(se, "show_single_start"); *prev_r = 0; *next_r = 0; - if (se->reclist) + reclist_limit(se->reclist, se, 1); + + reclist_enter(se->reclist); + while ((r = reclist_read_record(se->reclist))) { - reclist_enter(se->reclist); - while ((r = reclist_read_record(se->reclist))) + if (!strcmp(r->recid, id)) { - if (!strcmp(r->recid, id)) - { - *next_r = reclist_read_record(se->reclist); - break; - } - *prev_r = r; + *next_r = reclist_read_record(se->reclist); + break; } - reclist_leave(se->reclist); + *prev_r = r; } + reclist_leave(se->reclist); if (!r) - session_leave(se); + session_leave(se, "show_single_start"); return r; } void show_single_stop(struct session *se, struct record_cluster *rec) { - session_leave(se); + session_leave(se, "show_single_stop"); +} + + +int session_fetch_more(struct session *se) +{ + struct client_list *l; + int ret = 0; + + for (l = se->clients_active; l; l = l->next) + { + struct client *cl = l->client; + if (client_get_state(cl) == Client_Idle) + { + if (client_fetch_more(cl)) + { + session_log(se, YLOG_LOG, "%s: more to fetch", + client_get_id(cl)); + ret = 1; + } + else + { + int filtered; + int ingest_failures; + int record_failures; + int num = client_get_num_records( + cl, &filtered, &ingest_failures, &record_failures); + + session_log(se, YLOG_LOG, "%s: hits=" ODR_INT_PRINTF + " fetched=%d filtered=%d", + client_get_id(cl), + client_get_hits(cl), + num, filtered); + if (ingest_failures || record_failures) + { + session_log(se, YLOG_WARN, "%s:" + " ingest failures=%d record failures=%d", + client_get_id(cl), + ingest_failures, record_failures); + } + } + } + else + { + session_log(se, YLOG_LOG, "%s: no fetch due to state=%s", + client_get_id(cl), client_get_state_str(cl)); + } + + } + return ret; } struct record_cluster **show_range_start(struct session *se, - struct reclist_sortparms *sp, - int start, int *num, int *total, Odr_int *sumhits, Odr_int *approx_hits) + struct reclist_sortparms *sp, + int start, int *num, int *total, + Odr_int *sumhits, Odr_int *approx_hits, + void (*show_records_ready)(void *data), + struct http_channel *chan) { - struct record_cluster **recs; + struct record_cluster **recs = 0; struct reclist_sortparms *spp; + struct client_list *l; int i; -#if USE_TIMING +#if USE_TIMING yaz_timing_t t = yaz_timing_create(); #endif - session_enter(se); - recs = nmem_malloc(se->nmem, *num * sizeof(struct record_cluster *)); - if (!se->relevance) + session_enter(se, "show_range_start"); + *sumhits = 0; + *approx_hits = 0; + *total = 0; + reclist_limit(se->reclist, se, 0); + if (se->relevance) { - *num = 0; - *total = 0; - *sumhits = 0; - *approx_hits = 0; - recs = 0; - } - else - { - struct client_list *l; - for (spp = sp; spp; spp = spp->next) - if (spp->type == Metadata_sortkey_relevance) + if (spp->type == Metadata_type_relevance) { relevance_prepare_read(se->relevance, se->reclist); break; } - reclist_sort(se->reclist, sp); - - reclist_enter(se->reclist); - *total = reclist_get_num_records(se->reclist); - - *sumhits = 0; - *approx_hits = 0; for (l = se->clients_active; l; l = l->next) { *sumhits += client_get_hits(l->client); *approx_hits += client_get_approximation(l->client); } - for (i = 0; i < start; i++) - if (!reclist_read_record(se->reclist)) - { - *num = 0; - recs = 0; - break; - } - - for (i = 0; i < *num; i++) + } + reclist_sort(se->reclist, sp); + + reclist_enter(se->reclist); + *total = reclist_get_num_records(se->reclist); + + for (l = se->clients_active; l; l = l->next) + client_update_show_stat(l->client, 0); + + for (i = 0; i < start; i++) + { + struct record_cluster *r = reclist_read_record(se->reclist); + if (!r) { - struct record_cluster *r = reclist_read_record(se->reclist); - if (!r) - { - *num = i; - break; - } + *num = 0; + break; + } + else + { + struct record *rec = r->records; + for (;rec; rec = rec->next) + client_update_show_stat(rec->client, 1); + } + } + recs = nmem_malloc(se->nmem, (*num > 0 ? *num : 1) * sizeof(*recs)); + for (i = 0; i < *num; i++) + { + struct record_cluster *r = reclist_read_record(se->reclist); + if (!r) + { + *num = i; + break; + } + else + { + struct record *rec = r->records; + for (;rec; rec = rec->next) + client_update_show_stat(rec->client, 1); recs[i] = r; } - reclist_leave(se->reclist); } + reclist_leave(se->reclist); #if USE_TIMING yaz_timing_stop(t); - yaz_log(YLOG_LOG, "show %6.5f %3.2f %3.2f", + session_log(se, YLOG_LOG, "show %6.5f %3.2f %3.2f", yaz_timing_get_real(t), yaz_timing_get_user(t), yaz_timing_get_sys(t)); yaz_timing_destroy(&t); #endif + + if (!session_fetch_more(se)) + session_log(se, YLOG_LOG, "can not fetch more"); + else + { + show_range_stop(se, recs); + session_log(se, YLOG_LOG, "fetching more in progress"); + if (session_set_watch(se, SESSION_WATCH_SHOW, + show_records_ready, chan, chan)) + { + session_log(se, YLOG_WARN, "Ignoring show block"); + session_enter(se, "show_range_start"); + } + else + { + session_log(se, YLOG_LOG, "session watch OK"); + return 0; + } + } return recs; } void show_range_stop(struct session *se, struct record_cluster **recs) { - session_leave(se); + session_leave(se, "show_range_stop"); } void statistics(struct session *se, struct statistics *stat) @@ -1329,20 +1478,22 @@ void statistics(struct session *se, struct statistics *stat) } static struct record_metadata *record_metadata_init( - NMEM nmem, const char *value, enum conf_metadata_type type, + NMEM nmem, const char *value, const char *norm, + enum conf_metadata_type type, struct _xmlAttr *attr) { struct record_metadata *rec_md = record_metadata_create(nmem); struct record_metadata_attr **attrp = &rec_md->attributes; - + for (; attr; attr = attr->next) { if (attr->children && attr->children->content) { - if (strcmp((const char *) attr->name, "type")) - { /* skip the "type" attribute.. Its value is already part of - the element in output (md-%s) and so repeating it here - is redundant */ + if (strcmp((const char *) attr->name, "type") + && strcmp((const char *) attr->name, "empty")) + { /* skip the "type" + "empty" attribute.. + The "Type" is already part of the element in output + (md-%s) and so repeating it here is redundant */ *attrp = nmem_malloc(nmem, sizeof(**attrp)); (*attrp)->name = nmem_strdup(nmem, (const char *) attr->name); @@ -1354,16 +1505,29 @@ static struct record_metadata *record_metadata_init( } *attrp = 0; - if (type == Metadata_type_generic) + switch (type) { - char *p = nmem_strdup(nmem, value); - - p = normalize7bit_generic(p, " ,/.:(["); - - rec_md->data.text.disp = p; + case Metadata_type_generic: + case Metadata_type_skiparticle: + if (norm) + { + rec_md->data.text.disp = nmem_strdup(nmem, value); + rec_md->data.text.norm = nmem_strdup(nmem, norm); + } + else + { + if (strstr(value, "://")) /* looks like a URL */ + rec_md->data.text.disp = nmem_strdup(nmem, value); + else + rec_md->data.text.disp = + normalize7bit_generic(nmem_strdup(nmem, value), " ,/.:(["); + rec_md->data.text.norm = rec_md->data.text.disp; + } rec_md->data.text.sort = 0; - } - else if (type == Metadata_type_year || type == Metadata_type_date) + rec_md->data.text.snippet = 0; + break; + case Metadata_type_year: + case Metadata_type_date: { int first, last; int longdate = 0; @@ -1376,11 +1540,37 @@ static struct record_metadata *record_metadata_init( rec_md->data.number.min = first; rec_md->data.number.max = last; } - else + break; + case Metadata_type_float: + rec_md->data.fnumber = atof(value); + break; + case Metadata_type_relevance: + case Metadata_type_position: return 0; + } return rec_md; } +static void mergekey_norm_wr(pp2_charset_fact_t charsets, + WRBUF norm_wr, const char *value) +{ + const char *norm_str; + pp2_charset_token_t prt = + pp2_charset_token_create(charsets, "mergekey"); + + pp2_charset_token_first(prt, value, 0); + while ((norm_str = pp2_charset_token_next(prt))) + { + if (*norm_str) + { + if (wrbuf_len(norm_wr)) + wrbuf_puts(norm_wr, " "); + wrbuf_puts(norm_wr, norm_str); + } + } + pp2_charset_token_destroy(prt); +} + static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, struct conf_service *service, WRBUF norm_wr) { @@ -1392,70 +1582,52 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, continue; if (!strcmp((const char *) n->name, "metadata")) { - xmlChar *type = xmlGetProp(n, (xmlChar *) "type"); + const char *type = yaz_xml_get_prop(n, "type"); if (type == NULL) { yaz_log(YLOG_FATAL, "Missing type attribute on metadata element. Skipping!"); } else if (!strcmp(name, (const char *) type)) { xmlChar *value = xmlNodeListGetString(doc, n->children, 1); - if (value) + if (value && *value) { - const char *norm_str; - pp2_charset_token_t prt = - pp2_charset_token_create(service->charsets, "mergekey"); - - pp2_charset_token_first(prt, (const char *) value, 0); if (wrbuf_len(norm_wr) > 0) wrbuf_puts(norm_wr, " "); wrbuf_puts(norm_wr, name); - while ((norm_str = - pp2_charset_token_next(prt))) - { - if (*norm_str) - { - wrbuf_puts(norm_wr, " "); - wrbuf_puts(norm_wr, norm_str); - } - } - xmlFree(value); - pp2_charset_token_destroy(prt); + mergekey_norm_wr(service->charsets, norm_wr, + (const char *) value); no_found++; } + if (value) + xmlFree(value); } - xmlFree(type); } } return no_found; } -static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, - struct conf_service *service, NMEM nmem) +static const char *get_mergekey(xmlDoc *doc, xmlNode *root, + struct client *cl, int record_no, + struct conf_service *service, NMEM nmem, + const char *session_mergekey) { char *mergekey_norm = 0; - xmlNode *root = xmlDocGetRootElement(doc); WRBUF norm_wr = wrbuf_alloc(); + const char *mergekey; - /* consider mergekey from XSL first */ - xmlChar *mergekey = xmlGetProp(root, (xmlChar *) "mergekey"); - if (mergekey) + if (session_mergekey) { - const char *norm_str; - pp2_charset_token_t prt = - pp2_charset_token_create(service->charsets, "mergekey"); + int i, num = 0; + char **values = 0; + nmem_strsplit_escape2(nmem, ",", session_mergekey, &values, + &num, 1, '\\', 1); - pp2_charset_token_first(prt, (const char *) mergekey, 0); - while ((norm_str = pp2_charset_token_next(prt))) - { - if (*norm_str) - { - if (wrbuf_len(norm_wr)) - wrbuf_puts(norm_wr, " "); - wrbuf_puts(norm_wr, norm_str); - } - } - pp2_charset_token_destroy(prt); - xmlFree(mergekey); + for (i = 0; i < num; i++) + get_mergekey_from_doc(doc, root, values[i], service, norm_wr); + } + else if ((mergekey = yaz_xml_get_prop(root, "mergekey"))) + { + mergekey_norm_wr(service->charsets, norm_wr, mergekey); } else { @@ -1470,7 +1642,7 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, service, norm_wr); if (r == 0 && ser_md->mergekey == Metadata_mergekey_required) { - /* no mergekey on this one and it is required.. + /* no mergekey on this one and it is required.. Generate unique key instead */ wrbuf_rewind(norm_wr); break; @@ -1496,7 +1668,7 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, return mergekey_norm; } -/** \brief see if metadata for pz:recordfilter exists +/** \brief see if metadata for pz:recordfilter exists \param root xml root element of normalized record \param sdb session database for client \retval 0 if there is no metadata for pz:recordfilter @@ -1505,7 +1677,7 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, If there is no pz:recordfilter defined, this function returns 1 as well. */ - + static int check_record_filter(xmlNode *root, struct session_database *sdb) { int match = 0; @@ -1522,7 +1694,7 @@ static int check_record_filter(xmlNode *root, struct session_database *sdb) continue; if (!strcmp((const char *) n->name, "metadata")) { - xmlChar *type = xmlGetProp(n, (xmlChar *) "type"); + const char *type = yaz_xml_get_prop(n, "type"); if (type) { size_t len; @@ -1550,19 +1722,47 @@ static int check_record_filter(xmlNode *root, struct session_database *sdb) } xmlFree(value); } - xmlFree(type); } } } return match; } - static int ingest_to_cluster(struct client *cl, + WRBUF wrbuf_disp, + WRBUF wrbuf_norm, xmlDoc *xdoc, xmlNode *root, int record_no, - const char *mergekey_norm); + struct record_metadata_attr *mergekey); + +static int ingest_sub_record(struct client *cl, xmlDoc *xdoc, xmlNode *root, + int record_no, NMEM nmem, + struct session_database *sdb, + struct record_metadata_attr *mergekeys) +{ + int ret = 0; + struct session *se = client_get_session(cl); + WRBUF wrbuf_disp, wrbuf_norm; + + if (!check_record_filter(root, sdb)) + { + session_log(se, YLOG_LOG, + "Filtered out record no %d from %s", + record_no, sdb->database->id); + return 0; + } + wrbuf_disp = wrbuf_alloc(); + wrbuf_norm = wrbuf_alloc(); + session_enter(se, "ingest_sub_record"); + if (client_get_session(cl) == se && se->relevance) + ret = ingest_to_cluster(cl, wrbuf_disp, wrbuf_norm, + xdoc, root, record_no, mergekeys); + session_leave(se, "ingest_sub_record"); + wrbuf_destroy(wrbuf_norm); + wrbuf_destroy(wrbuf_disp); + return ret; +} /** \brief ingest XML record \param cl client holds the result set for record @@ -1577,38 +1777,207 @@ int ingest_record(struct client *cl, const char *rec, int record_no, NMEM nmem) { struct session *se = client_get_session(cl); - int ret = 0; struct session_database *sdb = client_get_database(cl); struct conf_service *service = se->service; xmlDoc *xdoc = normalize_record(se, sdb, service, rec, nmem); + int r = ingest_xml_record(cl, xdoc, record_no, nmem, 0); + client_store_xdoc(cl, record_no, xdoc); + return r; +} + +int ingest_xml_record(struct client *cl, xmlDoc *xdoc, + int record_no, NMEM nmem, int cached_copy) +{ + struct session *se = client_get_session(cl); + struct session_database *sdb = client_get_database(cl); + struct conf_service *service = se->service; xmlNode *root; - const char *mergekey_norm; - + int r = 0; if (!xdoc) return -1; - + + if (global_parameters.dump_records) + { + session_log(se, YLOG_LOG, "Normalized record from %s", + sdb->database->id); + log_xml_doc(xdoc); + } + root = xmlDocGetRootElement(xdoc); - - if (!check_record_filter(root, sdb)) + + if (!strcmp((const char *) root->name, "cluster")) { - session_log(se, YLOG_LOG, "Filtered out record no %d from %s", record_no, sdb->database->id); - xmlFreeDoc(xdoc); - return -2; + int no_merge_keys = 0; + int no_merge_dups = 0; + xmlNode *sroot; + struct record_metadata_attr *mk = 0; + + for (sroot = root->children; sroot; sroot = sroot->next) + if (sroot->type == XML_ELEMENT_NODE && + !strcmp((const char *) sroot->name, "record")) + { + struct record_metadata_attr **mkp; + const char *mergekey_norm = + get_mergekey(xdoc, sroot, cl, record_no, service, nmem, + se->mergekey); + if (!mergekey_norm) + { + r = -1; + break; + } + for (mkp = &mk; *mkp; mkp = &(*mkp)->next) + if (!strcmp((*mkp)->value, mergekey_norm)) + break; + if (!*mkp) + { + *mkp = (struct record_metadata_attr*) + nmem_malloc(nmem, sizeof(**mkp)); + (*mkp)->name = 0; + (*mkp)->value = nmem_strdup(nmem, mergekey_norm); + (*mkp)->next = 0; + no_merge_keys++; + } + else + no_merge_dups++; + } + if (no_merge_keys > 1 || no_merge_dups > 0) + { + yaz_log(YLOG_LOG, "Got %d mergekeys, %d dups for position %d", + no_merge_keys, no_merge_dups, record_no); + } + for (sroot = root->children; !r && sroot; sroot = sroot->next) + if (sroot->type == XML_ELEMENT_NODE && + !strcmp((const char *) sroot->name, "record")) + { + if (!cached_copy) + insert_settings_values(sdb, xdoc, root, service); + r = ingest_sub_record(cl, xdoc, sroot, record_no, nmem, sdb, + mk); + } } - - mergekey_norm = get_mergekey(xdoc, cl, record_no, service, nmem); - if (!mergekey_norm) + else if (!strcmp((const char *) root->name, "record")) { - session_log(se, YLOG_WARN, "Got no mergekey"); - xmlFreeDoc(xdoc); - return -1; + const char *mergekey_norm = + get_mergekey(xdoc, root, cl, record_no, service, nmem, + se->mergekey); + if (mergekey_norm) + { + struct record_metadata_attr *mk = (struct record_metadata_attr*) + nmem_malloc(nmem, sizeof(*mk)); + mk->name = 0; + mk->value = nmem_strdup(nmem, mergekey_norm); + mk->next = 0; + + if (!cached_copy) + insert_settings_values(sdb, xdoc, root, service); + r = ingest_sub_record(cl, xdoc, root, record_no, nmem, sdb, mk); + } + } + else + { + session_log(se, YLOG_WARN, "Bad pz root element: %s", + (const char *) root->name); + r = -1; + } + return r; +} + + +// struct conf_metadata *ser_md = &service->metadata[md_field_id]; +// struct record_metadata *rec_md = record->metadata[md_field_id]; +static int match_metadata_local(struct conf_service *service, + struct conf_metadata *ser_md, + struct record_metadata *rec_md0, + char **values, int num_v) +{ + int i; + struct record_metadata *rec_md = rec_md0; + WRBUF val_wr = 0; + WRBUF text_wr = wrbuf_alloc(); + for (i = 0; i < num_v; ) + { + if (rec_md) + { + if (ser_md->type == Metadata_type_year + || ser_md->type == Metadata_type_date) + { + int y = atoi(values[i]); + if (y >= rec_md->data.number.min + && y <= rec_md->data.number.max) + break; + } + else + { + if (!val_wr) + { + val_wr = wrbuf_alloc(); + mergekey_norm_wr(service->charsets, val_wr, values[i]); + } + wrbuf_rewind(text_wr); + mergekey_norm_wr(service->charsets, text_wr, + rec_md->data.text.disp); + if (!strcmp(wrbuf_cstr(val_wr), wrbuf_cstr(text_wr))) + break; + } + rec_md = rec_md->next; + } + else + { + rec_md = rec_md0; + wrbuf_destroy(val_wr); + val_wr = 0; + i++; + } + } + wrbuf_destroy(val_wr); + wrbuf_destroy(text_wr); + return i < num_v ? 1 : 0; +} + +int session_check_cluster_limit(struct session *se, struct record_cluster *rec) +{ + int i; + struct conf_service *service = se->service; + int ret = 1; + const char *name; + const char *value; + NMEM nmem_tmp = nmem_create(); + + for (i = 0; (name = facet_limits_get(se->facet_limits, i, &value)); i++) + { + int j; + for (j = 0; j < service->num_metadata; j++) + { + struct conf_metadata *md = service->metadata + j; + if (!strcmp(md->name, name) && md->limitcluster) + { + char **values = 0; + int num = 0; + int md_field_id = + conf_service_metadata_field_id(service, + md->limitcluster); + + if (md_field_id < 0) + { + ret = 0; + break; + } + + nmem_strsplit_escape2(nmem_tmp, "|", value, &values, + &num, 1, '\\', 1); + + if (!match_metadata_local(service, + &service->metadata[md_field_id], + rec->metadata[md_field_id], + values, num)) + { + ret = 0; + break; + } + } + } } - session_enter(se); - if (client_get_session(cl) == se) - ret = ingest_to_cluster(cl, xdoc, root, record_no, mergekey_norm); - session_leave(se); - - xmlFreeDoc(xdoc); + nmem_destroy(nmem_tmp); return ret; } @@ -1625,92 +1994,77 @@ static int check_limit_local(struct client *cl, int l = 0; while (!skip_record) { - struct conf_metadata *ser_md = 0; - struct record_metadata *rec_md = 0; int md_field_id; char **values = 0; - int i, num_v = 0; - - const char *name = client_get_facet_limit_local(cl, sdb, &l, nmem_tmp, &num_v, &values); + int num_v = 0; + const char *name = + client_get_facet_limit_local(cl, sdb, &l, nmem_tmp, + &num_v, &values); if (!name) break; - - md_field_id = conf_service_metadata_field_id(service, name); - if (md_field_id < 0) + + if (!strcmp(name, "*")) { - skip_record = 1; - break; + for (md_field_id = 0; md_field_id < service->num_metadata; + md_field_id++) + { + if (match_metadata_local( + service, + &service->metadata[md_field_id], + record->metadata[md_field_id], + values, num_v)) + break; + } + if (md_field_id == service->num_metadata) + skip_record = 1; } - ser_md = &service->metadata[md_field_id]; - rec_md = record->metadata[md_field_id]; - yaz_log(YLOG_DEBUG, "check limit local %s", name); - for (i = 0; i < num_v; ) + else { - if (rec_md) + md_field_id = conf_service_metadata_field_id(service, name); + if (md_field_id < 0) { - if (ser_md->type == Metadata_type_year - || ser_md->type == Metadata_type_date) - { - int y = atoi(values[i]); - if (y >= rec_md->data.number.min - && y <= rec_md->data.number.max) - break; - } - else - { - yaz_log(YLOG_DEBUG, "cmp: '%s' '%s'", rec_md->data.text.disp, values[i]); - if (!strcmp(rec_md->data.text.disp, values[i])) - { - // Value equals, should not be filtered. - break; - } - } - rec_md = rec_md->next; + skip_record = 1; + break; } - else + if (!match_metadata_local( + service, + &service->metadata[md_field_id], + record->metadata[md_field_id], + values, num_v)) { - rec_md = record->metadata[md_field_id]; - i++; + skip_record = 1; } } - // At end , not match - if (i == num_v) - { - skip_record = 1; - break; - } } nmem_destroy(nmem_tmp); return skip_record; } - + static int ingest_to_cluster(struct client *cl, + WRBUF wrbuf_disp, + WRBUF wrbuf_norm, xmlDoc *xdoc, xmlNode *root, int record_no, - const char *mergekey_norm) + struct record_metadata_attr *merge_keys) { xmlNode *n; - xmlChar *type = 0; - xmlChar *value = 0; struct session *se = client_get_session(cl); struct conf_service *service = se->service; int term_factor = 1; struct record_cluster *cluster; + struct record_metadata **metadata0; struct session_database *sdb = client_get_database(cl); - struct record *record = record_create(se->nmem, + NMEM ingest_nmem = 0; + char **rank_values = 0; + int rank_num = 0; + struct record *record = record_create(se->nmem, service->num_metadata, service->num_sortkeys, cl, record_no); for (n = root->children; n; n = n->next) { - if (type) - xmlFree(type); - if (value) - xmlFree(value); - type = value = 0; - if (n->type != XML_ELEMENT_NODE) continue; if (!strcmp((const char *) n->name, "metadata")) @@ -1719,37 +2073,63 @@ static int ingest_to_cluster(struct client *cl, struct record_metadata **wheretoput = 0; struct record_metadata *rec_md = 0; int md_field_id = -1; - - type = xmlGetProp(n, (xmlChar *) "type"); - value = xmlNodeListGetString(xdoc, n->children, 1); - - if (!type || !value || !*value) + xmlChar *value0; + const char *type = yaz_xml_get_prop(n, "type"); + + if (!type) continue; - - md_field_id + + md_field_id = conf_service_metadata_field_id(service, (const char *) type); if (md_field_id < 0) { if (se->number_of_warnings_unknown_metadata == 0) { - session_log(se, YLOG_WARN, + session_log(se, YLOG_WARN, "Ignoring unknown metadata element: %s", type); } se->number_of_warnings_unknown_metadata++; continue; } - + + wrbuf_rewind(wrbuf_disp); + value0 = xmlNodeListGetString(xdoc, n->children, 1); + if (!value0 || !*value0) + { + const char *empty = yaz_xml_get_prop(n, "empty"); + if (!empty) + continue; + wrbuf_puts(wrbuf_disp, (const char *) empty); + } + else + { + wrbuf_puts(wrbuf_disp, (const char *) value0); + } + if (value0) + xmlFree(value0); ser_md = &service->metadata[md_field_id]; // non-merged metadata - rec_md = record_metadata_init(se->nmem, (const char *) value, + rec_md = record_metadata_init(se->nmem, wrbuf_cstr(wrbuf_disp), 0, ser_md->type, n->properties); if (!rec_md) { session_log(se, YLOG_WARN, "bad metadata data '%s' " - "for element '%s'", value, type); + "for element '%s'", wrbuf_cstr(wrbuf_disp), type); continue; } + + if (ser_md->type == Metadata_type_generic) + { + WRBUF w = wrbuf_alloc(); + if (relevance_snippet(se->relevance, + wrbuf_cstr(wrbuf_disp), ser_md->name, w)) + rec_md->data.text.snippet = nmem_strdup(se->nmem, + wrbuf_cstr(w)); + wrbuf_destroy(w); + } + + wheretoput = &record->metadata[md_field_id]; while (*wheretoput) wheretoput = &(*wheretoput)->next; @@ -1759,18 +2139,14 @@ static int ingest_to_cluster(struct client *cl, if (check_limit_local(cl, record, record_no)) { - session_log(se, YLOG_LOG, "Facet filtered out record no %d from %s", - record_no, sdb->database->id); - if (type) - xmlFree(type); - if (value) - xmlFree(value); return -2; } - cluster = reclist_insert(se->reclist, service, record, - mergekey_norm, &se->total_merged); + cluster = reclist_insert(se->reclist, se->relevance, service, record, + merge_keys, &se->total_merged); if (!cluster) - return -1; + { + return 0; // complete match with existing record + } { const char *use_term_factor_str = @@ -1781,7 +2157,8 @@ static int ingest_to_cluster(struct client *cl, int hits = (int) client_get_hits(cl); term_factor = MAX(hits, maxrecs) / MAX(1, maxrecs); assert(term_factor >= 1); - yaz_log(YLOG_DEBUG, "Using term factor: %d (%d / %d)", term_factor, MAX(hits, maxrecs), MAX(1, maxrecs)); + session_log(se, YLOG_DEBUG, "Using term factor: %d (%d / %d)", + term_factor, MAX(hits, maxrecs), MAX(1, maxrecs)); } } @@ -1789,19 +2166,22 @@ static int ingest_to_cluster(struct client *cl, session_log(se, YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid, sdb->database->id, record_no); + // original metadata, to check if first existence of a field + metadata0 = xmalloc(sizeof(*metadata0) * service->num_metadata); + memcpy(metadata0, cluster->metadata, + sizeof(*metadata0) * service->num_metadata); + + ingest_nmem = nmem_create(); + if (se->rank) + { + yaz_log(YLOG_LOG, "local in sort : %s", se->rank); + nmem_strsplit_escape2(ingest_nmem, ",", se->rank, &rank_values, + &rank_num, 1, '\\', 1); + } - relevance_newrec(se->relevance, cluster); - // now parsing XML record and adding data to cluster or record metadata for (n = root->children; n; n = n->next) { - pp2_charset_token_t prt; - if (type) - xmlFree(type); - if (value) - xmlFree(value); - type = value = 0; - if (n->type != XML_ELEMENT_NODE) continue; if (!strcmp((const char *) n->name, "metadata")) @@ -1812,53 +2192,104 @@ static int ingest_to_cluster(struct client *cl, struct record_metadata *rec_md = 0; int md_field_id = -1; int sk_field_id = -1; - int rank = 0; - xmlChar *rank_str = 0; - - type = xmlGetProp(n, (xmlChar *) "type"); - value = xmlNodeListGetString(xdoc, n->children, 1); - - if (!type || !value || !*value) + const char *rank = 0; + const char *xml_rank = 0; + const char *type = 0; + xmlChar *value0; + + type = yaz_xml_get_prop(n, "type"); + if (!type) continue; - - md_field_id + + md_field_id = conf_service_metadata_field_id(service, (const char *) type); if (md_field_id < 0) continue; - + ser_md = &service->metadata[md_field_id]; - rank_str = xmlGetProp(n, (xmlChar *) "rank"); - if (rank_str) - { - rank = atoi((const char *) rank_str); - xmlFree(rank_str); - } - else - rank = ser_md->rank; - if (ser_md->sortkey_offset >= 0) { sk_field_id = ser_md->sortkey_offset; ser_sk = &service->sortkeys[sk_field_id]; } - // merged metadata - rec_md = record_metadata_init(se->nmem, (const char *) value, - ser_md->type, 0); + wrbuf_rewind(wrbuf_disp); + wrbuf_rewind(wrbuf_norm); + + value0 = xmlNodeListGetString(xdoc, n->children, 1); + if (!value0 || !*value0) + { + if (value0) + xmlFree(value0); + continue; + } + + if (ser_md->icurule) + { + run_icu(se, ser_md->icurule, (const char *) value0, + wrbuf_norm, wrbuf_disp); + yaz_log(YLOG_LOG, "run_icu input=%s norm=%s disp=%s", + (const char *) value0, + wrbuf_cstr(wrbuf_norm), wrbuf_cstr(wrbuf_disp)); + rec_md = record_metadata_init(se->nmem, wrbuf_cstr(wrbuf_disp), + wrbuf_cstr(wrbuf_norm), + ser_md->type, 0); + } + else + { + wrbuf_puts(wrbuf_disp, (const char *) value0); + rec_md = record_metadata_init(se->nmem, wrbuf_cstr(wrbuf_disp), + 0, + ser_md->type, 0); + } + + xmlFree(value0); + + // see if the field was not in cluster already (from beginning) if (!rec_md) continue; + if (rank_num) + { + int i; + for (i = 0; i < rank_num; i++) + { + const char *val = rank_values[i]; + const char *cp = strchr(val, '='); + if (!cp) + continue; + if ((cp - val) == strlen((const char *) type) + && !memcmp(val, type, cp - val)) + { + rank = cp + 1; + break; + } + } + } + else + { + xml_rank = yaz_xml_get_prop(n, "rank"); + rank = xml_rank ? (const char *) xml_rank : ser_md->rank; + } + wheretoput = &cluster->metadata[md_field_id]; - // and polulate with data: - // assign cluster or record based on merge action - if (ser_md->merge == Metadata_merge_unique) + if (ser_md->merge == Metadata_merge_first) + { + if (!metadata0[md_field_id]) + { + while (*wheretoput) + wheretoput = &(*wheretoput)->next; + *wheretoput = rec_md; + } + } + else if (ser_md->merge == Metadata_merge_unique) { while (*wheretoput) { - if (!strcmp((const char *) (*wheretoput)->data.text.disp, - rec_md->data.text.disp)) + if (!strcmp((const char *) (*wheretoput)->data.text.norm, + rec_md->data.text.norm)) break; wheretoput = &(*wheretoput)->next; } @@ -1867,22 +2298,23 @@ static int ingest_to_cluster(struct client *cl, } else if (ser_md->merge == Metadata_merge_longest) { - if (!*wheretoput - || strlen(rec_md->data.text.disp) - > strlen((*wheretoput)->data.text.disp)) + if (!*wheretoput + || strlen(rec_md->data.text.norm) + > strlen((*wheretoput)->data.text.norm)) { *wheretoput = rec_md; if (ser_sk) { + pp2_charset_token_t prt; const char *sort_str = 0; - int skip_article = - ser_sk->type == Metadata_sortkey_skiparticle; + int skip_article = + ser_sk->type == Metadata_type_skiparticle; if (!cluster->sortkeys[sk_field_id]) - cluster->sortkeys[sk_field_id] = - nmem_malloc(se->nmem, + cluster->sortkeys[sk_field_id] = + nmem_malloc(se->nmem, sizeof(union data_types)); - + prt = pp2_charset_token_create(service->charsets, "sort"); @@ -1890,18 +2322,18 @@ static int ingest_to_cluster(struct client *cl, skip_article); pp2_charset_token_next(prt); - + sort_str = pp2_get_sort(prt); - - cluster->sortkeys[sk_field_id]->text.disp = + + cluster->sortkeys[sk_field_id]->text.disp = rec_md->data.text.disp; if (!sort_str) { sort_str = rec_md->data.text.disp; - session_log(se, YLOG_WARN, + session_log(se, YLOG_WARN, "Could not make sortkey. Bug #1858"); } - cluster->sortkeys[sk_field_id]->text.sort = + cluster->sortkeys[sk_field_id]->text.sort = nmem_strdup(se->nmem, sort_str); pp2_charset_token_destroy(prt); } @@ -1919,7 +2351,7 @@ static int ingest_to_cluster(struct client *cl, { *wheretoput = rec_md; if (ser_sk) - cluster->sortkeys[sk_field_id] + cluster->sortkeys[sk_field_id] = &rec_md->data; } else @@ -1933,14 +2365,13 @@ static int ingest_to_cluster(struct client *cl, } } - // ranking of _all_ fields enabled ... + // ranking of _all_ fields enabled ... if (rank) { - relevance_countwords(se->relevance, cluster, - (char *) value, rank, ser_md->name, - ser_md->frank); + relevance_countwords(se->relevance, cluster, + wrbuf_cstr(wrbuf_disp), + rank, ser_md->name); } - // construct facets ... unless the client already has reported them if (ser_md->termlist && !client_has_facet(cl, (char *) type)) { @@ -1957,13 +2388,8 @@ static int ingest_to_cluster(struct client *cl, } } else - add_facet(se, (char *) type, (char *) value, term_factor); + add_facet(se, type, wrbuf_cstr(wrbuf_disp), term_factor); } - - // cleaning up - xmlFree(type); - xmlFree(value); - type = value = 0; } else { @@ -1973,11 +2399,8 @@ static int ingest_to_cluster(struct client *cl, se->number_of_warnings_unknown_elements++; } } - if (type) - xmlFree(type); - if (value) - xmlFree(value); - + nmem_destroy(ingest_nmem); + xfree(metadata0); relevance_donerecord(se->relevance, cluster); se->total_records++; @@ -1991,7 +2414,7 @@ void session_log(struct session *s, int level, const char *fmt, ...) va_start(ap, fmt); yaz_vsnprintf(buf, sizeof(buf)-30, fmt, ap); - yaz_log(level, "Session %u: %s", s->session_id, buf); + yaz_log(level, "Session %u: %s", s ? s->session_id : 0, buf); va_end(ap); }