X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Fsession.c;h=d43d4074e59db1d2d5a519f8c3f8b5ab62c69400;hb=ab0c408e57e6d12f3b9fbe71727616e45e80e8d9;hp=373dd49e590e5f26260bb9304b354f37a1adcf73;hpb=9ac9ee0172c5440d7dab879756893d943e313143;p=pazpar2-moved-to-github.git diff --git a/src/session.c b/src/session.c index 373dd49..d43d407 100644 --- a/src/session.c +++ b/src/session.c @@ -1,5 +1,5 @@ /* This file is part of Pazpar2. - Copyright (C) 2006-2011 Index Data + Copyright (C) 2006-2012 Index Data Pazpar2 is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -56,7 +56,6 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #include #include -#include #define USE_TIMING 0 #if USE_TIMING @@ -76,14 +75,14 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include "settings.h" #include "normalize7bit.h" -#define TERMLIST_HIGH_SCORE 25 +#include #define MAX_CHUNK 15 #define MAX(a,b) ((a)>(b)?(a):(b)) // Note: Some things in this structure will eventually move to configuration -struct parameters global_parameters = +struct parameters global_parameters = { 0, // dump_records 0, // debug_mode @@ -111,15 +110,17 @@ static int session_use(int delta) no_session_total += delta; sessions = no_sessions; yaz_mutex_leave(g_session_mutex); - yaz_log(YLOG_DEBUG, "%s sesions=%d", delta == 0 ? "" : (delta > 0 ? "INC" : "DEC"), no_sessions); + yaz_log(YLOG_DEBUG, "%s sessions=%d", delta == 0 ? "" : (delta > 0 ? "INC" : "DEC"), no_sessions); return sessions; } -int sessions_count(void) { +int sessions_count(void) +{ return session_use(0); } -int session_count_total(void) { +int session_count_total(void) +{ int total = 0; if (!g_session_mutex) return 0; @@ -129,7 +130,6 @@ int session_count_total(void) { return total; } - static void log_xml_doc(xmlDoc *doc) { FILE *lf = yaz_log_file(); @@ -148,63 +148,73 @@ static void log_xml_doc(xmlDoc *doc) xmlFree(result); } -static void session_enter(struct session *s) +static void session_enter(struct session *s, const char *caller) { + if (caller) + session_log(s, YLOG_DEBUG, "Session lock by %s", caller); yaz_mutex_enter(s->session_mutex); } -static void session_leave(struct session *s) +static void session_leave(struct session *s, const char *caller) { yaz_mutex_leave(s->session_mutex); + if (caller) + session_log(s, YLOG_DEBUG, "Session unlock by %s", caller); } -// Recursively traverse query structure to extract terms. -void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) +static void session_normalize_facet(struct session *s, const char *type, + const char *value, + WRBUF display_wrbuf, + WRBUF facet_wrbuf) { - char **words; - int numwords; + struct conf_service *service = s->service; + pp2_charset_token_t prt; + const char *facet_component; int i; + const char *icu_chain_id = 0; - switch (n->kind) + for (i = 0; i < service->num_metadata; i++) + if (!strcmp((service->metadata + i)->name, type)) + icu_chain_id = (service->metadata + i)->facetrule; + if (!icu_chain_id) + icu_chain_id = "facet"; + prt = pp2_charset_token_create(service->charsets, icu_chain_id); + if (!prt) { - case CCL_RPN_AND: - case CCL_RPN_OR: - case CCL_RPN_NOT: - case CCL_RPN_PROX: - pull_terms(nmem, n->u.p[0], termlist, num); - pull_terms(nmem, n->u.p[1], termlist, num); - break; - case CCL_RPN_TERM: - nmem_strsplit(nmem, " ", n->u.t.term, &words, &numwords); - for (i = 0; i < numwords; i++) - termlist[(*num)++] = words[i]; - break; - default: // NOOP - break; + yaz_log(YLOG_FATAL, "Unknown ICU chain '%s' for facet of type '%s'", + icu_chain_id, type); + wrbuf_destroy(facet_wrbuf); + wrbuf_destroy(display_wrbuf); + return; } -} - - -void add_facet(struct session *s, const char *type, const char *value, int count) -{ - struct conf_service *service = s->service; - pp2_relevance_token_t prt; - const char *facet_component; - WRBUF facet_wrbuf = wrbuf_alloc(); - prt = pp2_relevance_tokenize(service->facet_pct); - - pp2_relevance_first(prt, value, 0); - while ((facet_component = pp2_relevance_token_next(prt))) + pp2_charset_token_first(prt, value, 0); + while ((facet_component = pp2_charset_token_next(prt))) { + const char *display_component; if (*facet_component) { if (wrbuf_len(facet_wrbuf)) wrbuf_puts(facet_wrbuf, " "); wrbuf_puts(facet_wrbuf, facet_component); } + display_component = pp2_get_display(prt); + if (display_component) + { + if (wrbuf_len(display_wrbuf)) + wrbuf_puts(display_wrbuf, " "); + wrbuf_puts(display_wrbuf, display_component); + } } - pp2_relevance_token_destroy(prt); - + pp2_charset_token_destroy(prt); +} + +void add_facet(struct session *s, const char *type, const char *value, int count) +{ + WRBUF facet_wrbuf = wrbuf_alloc(); + WRBUF display_wrbuf = wrbuf_alloc(); + + session_normalize_facet(s, type, value, display_wrbuf, facet_wrbuf); + if (wrbuf_len(facet_wrbuf)) { int i; @@ -217,22 +227,23 @@ void add_facet(struct session *s, const char *type, const char *value, int count { session_log(s, YLOG_FATAL, "Too many termlists"); wrbuf_destroy(facet_wrbuf); + wrbuf_destroy(display_wrbuf); return; } - + s->termlists[i].name = nmem_strdup(s->nmem, type); - s->termlists[i].termlist - = termlist_create(s->nmem, TERMLIST_HIGH_SCORE); + s->termlists[i].termlist = termlist_create(s->nmem); s->num_termlists = i + 1; } - + #if 0 - session_log(s, YLOG_DEBUG, "Facets for %s: %s norm:%s (%d)", type, value, wrbuf_cstr(facet_wrbuf), count); + session_log(s, YLOG_LOG, "Facets for %s: %s norm:%s (%d)", type, value, wrbuf_cstr(facet_wrbuf), count); #endif - termlist_insert(s->termlists[i].termlist, wrbuf_cstr(facet_wrbuf), - count); + termlist_insert(s->termlists[i].termlist, wrbuf_cstr(display_wrbuf), + wrbuf_cstr(facet_wrbuf), count); } wrbuf_destroy(facet_wrbuf); + wrbuf_destroy(display_wrbuf); } static xmlDoc *record_to_xml(struct session *se, @@ -245,14 +256,13 @@ static xmlDoc *record_to_xml(struct session *se, if (!rdoc) { - session_log(se, YLOG_FATAL, "Non-wellformed XML received from %s", - db->url); + session_log(se, YLOG_WARN, "Non-wellformed XML"); return 0; } if (global_parameters.dump_records) { - session_log(se, YLOG_LOG, "Un-normalized record from %s", db->url); + session_log(se, YLOG_LOG, "Un-normalized record from %s", db->id); log_xml_doc(rdoc); } @@ -334,22 +344,21 @@ static xmlDoc *normalize_record(struct session *se, if (rdoc) { char *parms[MAX_XSLT_ARGS*2+1]; - + insert_settings_parameters(sdb, service, parms, nmem); - + if (normalize_record_transform(sdb->map, &rdoc, (const char **)parms)) { - session_log(se, YLOG_WARN, "Normalize failed from %s", - sdb->database->url); + session_log(se, YLOG_WARN, "Normalize failed"); } else { insert_settings_values(sdb, rdoc, service); - + if (global_parameters.dump_records) { - session_log(se, YLOG_LOG, "Normalized record from %s", - sdb->database->url); + session_log(se, YLOG_LOG, "Normalized record from %s", + sdb->database->id); log_xml_doc(rdoc); } } @@ -397,63 +406,43 @@ const char *session_setting_oneval(struct session_database *db, int offset) // setting. However, this is not a realistic use scenario. static int prepare_map(struct session *se, struct session_database *sdb) { - const char *s; - - if (!sdb->settings) + if (sdb->settings && !sdb->map) { - session_log(se, YLOG_WARN, "No settings on %s", sdb->database->url); - return -1; - } - if ((s = session_setting_oneval(sdb, PZ_XSLT))) - { - char auto_stylesheet[256]; + const char *s; - if (!strcmp(s, "auto")) + if (sdb->settings[PZ_XSLT] && + (s = session_setting_oneval(sdb, PZ_XSLT))) { - const char *request_syntax = session_setting_oneval( - sdb, PZ_REQUESTSYNTAX); - if (request_syntax) + char auto_stylesheet[256]; + + if (!strcmp(s, "auto")) { - char *cp; - yaz_snprintf(auto_stylesheet, sizeof(auto_stylesheet), - "%s.xsl", request_syntax); - for (cp = auto_stylesheet; *cp; cp++) + const char *request_syntax = session_setting_oneval( + sdb, PZ_REQUESTSYNTAX); + if (request_syntax) { - /* deliberately only consider ASCII */ - if (*cp > 32 && *cp < 127) - *cp = tolower(*cp); + char *cp; + yaz_snprintf(auto_stylesheet, sizeof(auto_stylesheet), + "%s.xsl", request_syntax); + for (cp = auto_stylesheet; *cp; cp++) + { + /* deliberately only consider ASCII */ + if (*cp > 32 && *cp < 127) + *cp = tolower(*cp); + } + s = auto_stylesheet; + } + else + { + session_log(se, YLOG_WARN, + "No pz:requestsyntax for auto stylesheet"); } - s = auto_stylesheet; - } - else - { - session_log(se, YLOG_WARN, - "No pz:requestsyntax for auto stylesheet"); } + sdb->map = normalize_cache_get(se->normalize_cache, + se->service, s); + if (!sdb->map) + return -1; } - sdb->map = normalize_cache_get(se->normalize_cache, - se->service->server->config, s); - if (!sdb->map) - return -1; - } - return 0; -} - -// This analyzes settings and recomputes any supporting data structures -// if necessary. -static int prepare_session_database(struct session *se, - struct session_database *sdb) -{ - if (!sdb->settings) - { - session_log(se, YLOG_WARN, - "No settings associated with %s", sdb->database->url); - return -1; - } - if (sdb->settings[PZ_XSLT] && !sdb->map) - { - if (prepare_map(se, sdb) < 0) - return -1; } return 0; } @@ -470,31 +459,31 @@ static void session_watch_cancel(void *data, struct http_channel *c, } // set watch. Returns 0=OK, -1 if watch is already set -int session_set_watch(struct session *s, int what, +int session_set_watch(struct session *s, int what, session_watchfun fun, void *data, struct http_channel *chan) { int ret; - session_enter(s); + session_enter(s, "session_set_watch"); if (s->watchlist[what].fun) ret = -1; else { - + s->watchlist[what].fun = fun; s->watchlist[what].data = data; s->watchlist[what].obs = http_add_observer(chan, &s->watchlist[what], session_watch_cancel); ret = 0; } - session_leave(s); + session_leave(s, "session_set_watch"); return ret; } void session_alert_watch(struct session *s, int what) { assert(s); - session_enter(s); + session_enter(s, "session_alert_watch"); if (s->watchlist[what].fun) { /* our watch is no longer associated with http_channel */ @@ -511,39 +500,80 @@ void session_alert_watch(struct session *s, int what) s->watchlist[what].data = 0; s->watchlist[what].obs = 0; - session_leave(s); + session_leave(s, "session_alert_watch"); session_log(s, YLOG_DEBUG, "Alert Watch: %d calling function: %p", what, fun); fun(data); } else - session_leave(s); + session_leave(s,"session_alert_watch"); } //callback for grep_databases -static void select_targets_callback(void *context, struct session_database *db) +static void select_targets_callback(struct session *se, + struct session_database *db) { - struct session *se = (struct session*) context; - struct client *cl = client_create(); + struct client *cl; struct client_list *l; - client_set_database(cl, db); + for (l = se->clients_cached; l; l = l->next) + if (client_get_database(l->client) == db) + break; + + if (l) + cl = l->client; + else + { + cl = client_create(db->database->id); + client_set_database(cl, db); + + l = xmalloc(sizeof(*l)); + l->client = cl; + l->next = se->clients_cached; + se->clients_cached = l; + } + /* set session always. If may be 0 if client is not active */ client_set_session(cl, se); l = xmalloc(sizeof(*l)); l->client = cl; - l->next = se->clients; - se->clients = l; + l->next = se->clients_active; + se->clients_active = l; +} + +static void session_reset_active_clients(struct session *se, + struct client_list *new_list) +{ + struct client_list *l; + + session_enter(se, "session_reset_active_clients"); + l = se->clients_active; + se->clients_active = new_list; + session_leave(se, "session_reset_active_clients"); + + while (l) + { + struct client_list *l_next = l->next; + + client_lock(l->client); + client_set_session(l->client, 0); /* mark client inactive */ + client_unlock(l->client); + + xfree(l); + l = l_next; + } } -static void session_remove_clients(struct session *se) +static void session_remove_cached_clients(struct session *se) { struct client_list *l; - session_enter(se); - l = se->clients; - se->clients = 0; - session_leave(se); + session_reset_active_clients(se, 0); + + session_enter(se, "session_remove_cached_clients"); + l = se->clients_cached; + se->clients_cached = 0; + session_leave(se, "session_remove_cached_clients"); while (l) { @@ -559,7 +589,7 @@ static void session_remove_clients(struct session *se) } // Associates a set of clients with a session; -// Note: Session-databases represent databases with per-session +// Note: Session-databases represent databases with per-session // setting overrides static int select_targets(struct session *se, const char *filter) { @@ -571,7 +601,7 @@ int session_active_clients(struct session *s) struct client_list *l; int res = 0; - for (l = s->clients; l; l = l->next) + for (l = s->clients_active; l; l = l->next) if (client_is_active(l->client)) res++; @@ -583,93 +613,211 @@ int session_is_preferred_clients_ready(struct session *s) struct client_list *l; int res = 0; - for (l = s->clients; l; l = l->next) + for (l = s->clients_active; l; l = l->next) if (client_is_active_preferred(l->client)) res++; session_log(s, YLOG_DEBUG, "Has %d active preferred clients.", res); return res == 0; } -enum pazpar2_error_code search(struct session *se, - const char *query, - const char *startrecs, const char *maxrecs, - const char *filter, - const char *limit, - const char **addinfo) +static void session_clear_set(struct session *se, struct reclist_sortparms *sp) +{ + reclist_destroy(se->reclist); + se->reclist = 0; + if (nmem_total(se->nmem)) + session_log(se, YLOG_DEBUG, "NMEN operation usage %zd", + nmem_total(se->nmem)); + nmem_reset(se->nmem); + se->total_records = se->total_merged = 0; + se->num_termlists = 0; + + /* reset list of sorted results and clear to relevance search */ + se->sorted_results = nmem_malloc(se->nmem, sizeof(*se->sorted_results)); + se->sorted_results->name = nmem_strdup(se->nmem, sp->name); + se->sorted_results->increasing = sp->increasing; + se->sorted_results->type = sp->type; + se->sorted_results->next = 0; + + session_log(se, YLOG_DEBUG, "clear_set session_sort: field=%s increasing=%d type=%d configured", + sp->name, sp->increasing, sp->type); + + se->reclist = reclist_create(se->nmem); +} + +static void session_sort_unlocked(struct session *se, struct reclist_sortparms *sp) +{ + struct reclist_sortparms *sr; + struct client_list *l; + const char *field = sp->name; + int increasing = sp->increasing; + int type = sp->type; + int clients_research = 0; + + yaz_log(YLOG_LOG, "session_sort field=%s increasing=%d type=%d", field, increasing, type); + /* see if we already have sorted for this criteria */ + for (sr = se->sorted_results; sr; sr = sr->next) + { + if (!reclist_sortparms_cmp(sr,sp)) + break; + } + if (sr) + { + session_log(se, YLOG_DEBUG, "search_sort: field=%s increasing=%d type=%d already fetched", + field, increasing, type); + return; + } + session_log(se, YLOG_DEBUG, "search_sort: field=%s increasing=%d type=%d must fetch", + field, increasing, type); + + // We need to reset reclist on every sort that changes the records, not just for position + // So if just one client requires new searching, we need to clear set. + // Ask each of the client if sorting requires re-search due to native sort + // If it does it will require us to + for (l = se->clients_active; l; l = l->next) + { + struct client *cl = l->client; + // Assume no re-search is required. + client_parse_init(cl, 1); + clients_research += client_parse_sort(cl, sp); + } + if (clients_research) { + yaz_log(YLOG_DEBUG, "Reset results due to %d clients researching", clients_research); + session_clear_set(se, sp); + } + else { + // A new sorting based on same record set + sr = nmem_malloc(se->nmem, sizeof(*sr)); + sr->name = nmem_strdup(se->nmem, field); + sr->increasing = increasing; + sr->type = type; + sr->next = se->sorted_results; + se->sorted_results = sr; + session_log(se, YLOG_DEBUG, "No research/ingesting done"); + return ; + } + session_log(se, YLOG_DEBUG, "Re- search/ingesting for clients due to change in sort order"); + + for (l = se->clients_active; l; l = l->next) + { + struct client *cl = l->client; + if (client_get_state(cl) == Client_Connecting || + client_get_state(cl) == Client_Idle || + client_get_state(cl) == Client_Working) { + client_start_search(cl); + } + else { + yaz_log(YLOG_DEBUG, "Client %s: No re-start/ingest in show. Wrong client state: %d", + client_get_id(cl), client_get_state(cl)); + } + + } +} + +void session_sort(struct session *se, struct reclist_sortparms *sp) { + //session_enter(se, "session_sort"); + session_sort_unlocked(se, sp); + //session_leave(se, "session_sort"); +} + + +enum pazpar2_error_code session_search(struct session *se, + const char *query, + const char *startrecs, + const char *maxrecs, + const char *filter, + const char *limit, + const char **addinfo, + struct reclist_sortparms *sp) { int live_channels = 0; int no_working = 0; - int no_failed = 0; - struct client_list *l; - struct timeval tval; + int no_failed_query = 0; + int no_failed_limit = 0; + struct client_list *l, *l0; facet_limits_t facet_limits; + int same_sort_order = 0; session_log(se, YLOG_DEBUG, "Search"); *addinfo = 0; - session_remove_clients(se); - - session_enter(se); - reclist_destroy(se->reclist); - se->reclist = 0; + if (se->settings_modified) { + session_remove_cached_clients(se); + } + else + session_reset_active_clients(se, 0); + + session_enter(se, "session_search"); + se->settings_modified = 0; + + if (se->sorted_results) { + if (!reclist_sortparms_cmp(se->sorted_results, sp)) + same_sort_order = 1; + } + session_clear_set(se, sp); relevance_destroy(&se->relevance); - nmem_reset(se->nmem); - se->total_records = se->total_hits = se->total_merged = 0; - se->num_termlists = 0; + live_channels = select_targets(se, filter); if (!live_channels) { - session_leave(se); + session_leave(se, "session_search"); return PAZPAR2_NO_TARGETS; } - se->reclist = reclist_create(se->nmem); - - yaz_gettimeofday(&tval); - - tval.tv_sec += 5; facet_limits = facet_limits_create(limit); if (!facet_limits) { *addinfo = "limit"; - session_leave(se); + session_leave(se, "session_search"); return PAZPAR2_MALFORMED_PARAMETER_VALUE; } - for (l = se->clients; l; l = l->next) + + l0 = se->clients_active; + se->clients_active = 0; + session_leave(se, "session_search"); + + for (l = l0; l; l = l->next) { + int parse_ret; struct client *cl = l->client; + client_parse_init(cl, 1); + if (prepare_map(se, client_get_database(cl)) < 0) + continue; - if (maxrecs) - client_set_maxrecs(cl, atoi(maxrecs)); - if (startrecs) - client_set_startrecs(cl, atoi(startrecs)); - if (prepare_session_database(se, client_get_database(cl)) < 0) - ; - else if (client_parse_query(cl, query, facet_limits) < 0) - no_failed++; + parse_ret = client_parse_query(cl, query, facet_limits, se->service->ccl_bibset); + if (parse_ret == -1) + no_failed_query++; + else if (parse_ret == -2) + no_failed_limit++; + else if (parse_ret < 0) + no_working++; /* other error, such as bad CCL map */ else { + client_parse_range(cl, startrecs, maxrecs); + client_parse_sort(cl, sp); + client_start_search(cl); no_working++; - if (client_prep_connection(cl, se->service->z3950_operation_timeout, - se->service->z3950_session_timeout, - se->service->server->iochan_man, - &tval)) - client_start_search(cl); } } facet_limits_destroy(facet_limits); - session_leave(se); + session_reset_active_clients(se, l0); + if (no_working == 0) { - if (no_failed > 0) + if (no_failed_query > 0) { *addinfo = "query"; return PAZPAR2_MALFORMED_PARAMETER_VALUE; } + else if (no_failed_limit > 0) + { + *addinfo = "limit"; + return PAZPAR2_MALFORMED_PARAMETER_VALUE; + } else return PAZPAR2_NO_TARGETS; } + session_log(se, YLOG_LOG, "session_start_search done"); return PAZPAR2_NO_ERROR; } @@ -681,7 +829,7 @@ static void session_init_databases_fun(void *context, struct database *db) int i; new->database = db; - + new->map = 0; assert(db->settings); new->settings = nmem_malloc(se->session_nmem, @@ -712,13 +860,10 @@ void session_init_databases(struct session *se) // Probably session_init_databases_fun should be refactored instead of // called here. -static struct session_database *load_session_database(struct session *se, +static struct session_database *load_session_database(struct session *se, char *id) { - struct database *db = new_database(id, se->session_nmem); - - resolve_database(se->service, db); - + struct database *db = new_database_inherit_settings(id, se->session_nmem, se->service->settings); session_init_databases_fun((void*) se, db); // New sdb is head of se->databases list @@ -726,13 +871,13 @@ static struct session_database *load_session_database(struct session *se, } // Find an existing session database. If not found, load it -static struct session_database *find_session_database(struct session *se, +static struct session_database *find_session_database(struct session *se, char *id) { struct session_database *sdb; for (sdb = se->databases; sdb; sdb = sdb->next) - if (!strcmp(sdb->database->url, id)) + if (!strcmp(sdb->database->id, id)) return sdb; return load_session_database(se, id); } @@ -755,6 +900,8 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, new->next = sdb->settings[offset]; sdb->settings[offset] = new; + se->settings_modified = 1; + // Force later recompute of settings-driven data structures // (happens when a search starts and client connections are prepared) switch (offset) @@ -768,36 +915,34 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, } } -void session_destroy(struct session *se) { +void session_destroy(struct session *se) +{ struct session_database *sdb; session_log(se, YLOG_DEBUG, "Destroying"); session_use(-1); - session_remove_clients(se); + session_remove_cached_clients(se); for (sdb = se->databases; sdb; sdb = sdb->next) session_database_destroy(sdb); normalize_cache_destroy(se->normalize_cache); relevance_destroy(&se->relevance); reclist_destroy(se->reclist); + if (nmem_total(se->nmem)) + session_log(se, YLOG_DEBUG, "NMEN operation usage %zd", nmem_total(se->nmem)); + if (nmem_total(se->session_nmem)) + session_log(se, YLOG_DEBUG, "NMEN session usage %zd", nmem_total(se->session_nmem)); nmem_destroy(se->nmem); service_destroy(se->service); yaz_mutex_destroy(&se->session_mutex); - wrbuf_destroy(se->wrbuf); -} - -/* Depreciated: use session_destroy */ -void destroy_session(struct session *se) -{ - session_destroy(se); } size_t session_get_memory_status(struct session *session) { size_t session_nmem; if (session == 0) return 0; - session_enter(session); + session_enter(session, "session_get_memory_status"); session_nmem = nmem_total(session->nmem); - session_leave(session); + session_leave(session, "session_get_memory_status"); return session_nmem; } @@ -816,17 +961,19 @@ struct session *new_session(NMEM nmem, struct conf_service *service, session_log(session, YLOG_DEBUG, "New"); session->service = service; session->relevance = 0; - session->total_hits = 0; session->total_records = 0; session->number_of_warnings_unknown_elements = 0; session->number_of_warnings_unknown_metadata = 0; session->num_termlists = 0; session->reclist = 0; - session->clients = 0; + session->clients_active = 0; + session->clients_cached = 0; + session->settings_modified = 0; session->session_nmem = nmem; session->nmem = nmem_create(); - session->wrbuf = wrbuf_alloc(); session->databases = 0; + session->sorted_results = 0; + for (i = 0; i <= SESSION_WATCH_MAX; i++) { session->watchlist[i].data = 0; @@ -839,56 +986,202 @@ struct session *new_session(NMEM nmem, struct conf_service *service, return session; } -struct hitsbytarget *hitsbytarget(struct session *se, int *count, NMEM nmem) +const char * client_get_suggestions_xml(struct client *cl, WRBUF wrbuf); + +static struct hitsbytarget *hitsbytarget_nb(struct session *se, + int *count, NMEM nmem) { struct hitsbytarget *res = 0; struct client_list *l; size_t sz = 0; - session_enter(se); - for (l = se->clients; l; l = l->next) + for (l = se->clients_active; l; l = l->next) sz++; res = nmem_malloc(nmem, sizeof(*res) * sz); *count = 0; - for (l = se->clients; l; l = l->next) + for (l = se->clients_active; l; l = l->next) { struct client *cl = l->client; WRBUF w = wrbuf_alloc(); const char *name = session_setting_oneval(client_get_database(cl), PZ_NAME); - res[*count].id = client_get_database(cl)->database->url; + res[*count].id = client_get_id(cl); res[*count].name = *name ? name : "Unknown"; res[*count].hits = client_get_hits(cl); + res[*count].approximation = client_get_approximation(cl); res[*count].records = client_get_num_records(cl); - res[*count].diagnostic = client_get_diagnostic(cl); + res[*count].filtered = client_get_num_records_filtered(cl); + res[*count].diagnostic = + client_get_diagnostic(cl, &res[*count].message, + &res[*count].addinfo); res[*count].state = client_get_state_str(cl); res[*count].connected = client_get_connection(cl) ? 1 : 0; session_settings_dump(se, client_get_database(cl), w); res[*count].settings_xml = nmem_strdup(nmem, wrbuf_cstr(w)); + wrbuf_rewind(w); + wrbuf_puts(w, ""); + res[*count].suggestions_xml = nmem_strdup(nmem, client_get_suggestions_xml(cl, w)); wrbuf_destroy(w); (*count)++; } - session_leave(se); return res; } -struct termlist_score **get_termlist_score(struct session *se, - const char *name, int *num) +struct hitsbytarget *get_hitsbytarget(struct session *se, int *count, NMEM nmem) { - int i; - struct termlist_score **tl = 0; + struct hitsbytarget *p; + session_enter(se, "get_hitsbytarget"); + p = hitsbytarget_nb(se, count, nmem); + session_leave(se, "get_hitsbytarget"); + return p; +} + +// Compares two hitsbytarget nodes by hitcount +static int cmp_ht(const void *p1, const void *p2) +{ + const struct hitsbytarget *h1 = p1; + const struct hitsbytarget *h2 = p2; + return h2->hits - h1->hits; +} + +// Compares two hitsbytarget nodes by hitcount +static int cmp_ht_approx(const void *p1, const void *p2) +{ + const struct hitsbytarget *h1 = p1; + const struct hitsbytarget *h2 = p2; + return h2->approximation - h1->approximation; +} + +static int targets_termlist_nb(WRBUF wrbuf, struct session *se, int num, + NMEM nmem, int version) +{ + struct hitsbytarget *ht; + int count, i; + + ht = hitsbytarget_nb(se, &count, nmem); + if (version >= 2) + qsort(ht, count, sizeof(struct hitsbytarget), cmp_ht_approx); + else + qsort(ht, count, sizeof(struct hitsbytarget), cmp_ht); + for (i = 0; i < count && i < num && ht[i].hits > 0; i++) + { + + // do only print terms which have display names + + wrbuf_puts(wrbuf, "\n"); + + wrbuf_puts(wrbuf, ""); + wrbuf_xmlputs(wrbuf, ht[i].id); + wrbuf_puts(wrbuf, "\n"); + + wrbuf_puts(wrbuf, ""); + if (!ht[i].name || !ht[i].name[0]) + wrbuf_xmlputs(wrbuf, "NO TARGET NAME"); + else + wrbuf_xmlputs(wrbuf, ht[i].name); + wrbuf_puts(wrbuf, "\n"); + + wrbuf_printf(wrbuf, "" ODR_INT_PRINTF "\n", + ht[i].hits); + + if (version >= 2) { + // Should not print if we know it isn't a approximation. + wrbuf_printf(wrbuf, "" ODR_INT_PRINTF "\n", ht[i].approximation); + wrbuf_printf(wrbuf, "%d\n", ht[i].records - ht[i].filtered); + wrbuf_printf(wrbuf, "%d\n", ht[i].filtered); + } + + wrbuf_puts(wrbuf, ""); + wrbuf_xmlputs(wrbuf, ht[i].state); + wrbuf_puts(wrbuf, "\n"); + + wrbuf_printf(wrbuf, "%d\n", + ht[i].diagnostic); + wrbuf_puts(wrbuf, "\n"); + } + return count; +} + +void perform_termlist(struct http_channel *c, struct session *se, + const char *name, int num, int version) +{ + int i, j; + NMEM nmem_tmp = nmem_create(); + char **names; + int num_names = 0; + + if (!name) + name = "*"; + + nmem_strsplit(nmem_tmp, ",", name, &names, &num_names); + + session_enter(se, "perform_termlist"); - session_enter(se); - for (i = 0; i < se->num_termlists; i++) - if (!strcmp((const char *) se->termlists[i].name, name)) + for (j = 0; j < num_names; j++) + { + const char *tname; + int must_generate_empty = 1; /* bug 5350 */ + + for (i = 0; i < se->num_termlists; i++) { - tl = termlist_highscore(se->termlists[i].termlist, num); - break; + tname = se->termlists[i].name; + if (!strcmp(names[j], tname) || !strcmp(names[j], "*")) + { + struct termlist_score **p = 0; + int len; + + wrbuf_puts(c->wrbuf, "wrbuf, tname); + wrbuf_puts(c->wrbuf, "\">\n"); + must_generate_empty = 0; + + p = termlist_highscore(se->termlists[i].termlist, &len, + nmem_tmp); + if (p) + { + int i; + for (i = 0; i < len && i < num; i++) + { + // prevent sending empty term elements + if (!p[i]->display_term || !p[i]->display_term[0]) + continue; + + wrbuf_puts(c->wrbuf, ""); + wrbuf_puts(c->wrbuf, ""); + wrbuf_xmlputs(c->wrbuf, p[i]->display_term); + wrbuf_puts(c->wrbuf, ""); + + wrbuf_printf(c->wrbuf, + "%d", + p[i]->frequency); + wrbuf_puts(c->wrbuf, "\n"); + } + } + wrbuf_puts(c->wrbuf, "\n"); + } + } + tname = "xtargets"; + if (!strcmp(names[j], tname) || !strcmp(names[j], "*")) + { + wrbuf_puts(c->wrbuf, "wrbuf, tname); + wrbuf_puts(c->wrbuf, "\">\n"); + + targets_termlist_nb(c->wrbuf, se, num, c->nmem, version); + wrbuf_puts(c->wrbuf, "\n"); + must_generate_empty = 0; + } + if (must_generate_empty) + { + wrbuf_puts(c->wrbuf, "wrbuf, names[j]); + wrbuf_puts(c->wrbuf, "\"/>\n"); } - session_leave(se); - return tl; + } + session_leave(se, "perform_termlist"); + nmem_destroy(nmem_tmp); } #ifdef MISSING_HEADERS @@ -899,7 +1192,7 @@ void report_nmem_stats(void) nmem_get_memory_in_use(&in_use); nmem_get_memory_free(&is_free); - yaz_log(YLOG_LOG, "nmem stat: use=%ld free=%ld", + yaz_log(YLOG_LOG, "nmem stat: use=%ld free=%ld", (long) in_use, (long) is_free); } #endif @@ -910,7 +1203,7 @@ struct record_cluster *show_single_start(struct session *se, const char *id, { struct record_cluster *r = 0; - session_enter(se); + session_enter(se, "show_single_start"); *prev_r = 0; *next_r = 0; if (se->reclist) @@ -928,36 +1221,39 @@ struct record_cluster *show_single_start(struct session *se, const char *id, reclist_leave(se->reclist); } if (!r) - session_leave(se); + session_leave(se, "show_single_start"); return r; } void show_single_stop(struct session *se, struct record_cluster *rec) { - session_leave(se); + session_leave(se, "show_single_stop"); } struct record_cluster **show_range_start(struct session *se, - struct reclist_sortparms *sp, - int start, int *num, int *total, Odr_int *sumhits) + struct reclist_sortparms *sp, + int start, int *num, int *total, Odr_int *sumhits, Odr_int *approx_hits) { struct record_cluster **recs; struct reclist_sortparms *spp; int i; -#if USE_TIMING +#if USE_TIMING yaz_timing_t t = yaz_timing_create(); #endif - session_enter(se); + session_enter(se, "show_range_start"); recs = nmem_malloc(se->nmem, *num * sizeof(struct record_cluster *)); if (!se->relevance) { *num = 0; *total = 0; *sumhits = 0; + *approx_hits = 0; recs = 0; } else { + struct client_list *l; + for (spp = sp; spp; spp = spp->next) if (spp->type == Metadata_sortkey_relevance) { @@ -965,11 +1261,16 @@ struct record_cluster **show_range_start(struct session *se, break; } reclist_sort(se->reclist, sp); - + reclist_enter(se->reclist); *total = reclist_get_num_records(se->reclist); - *sumhits = se->total_hits; - + + *sumhits = 0; + *approx_hits = 0; + for (l = se->clients_active; l; l = l->next) { + *sumhits += client_get_hits(l->client); + *approx_hits += client_get_approximation(l->client); + } for (i = 0; i < start; i++) if (!reclist_read_record(se->reclist)) { @@ -977,7 +1278,7 @@ struct record_cluster **show_range_start(struct session *se, recs = 0; break; } - + for (i = 0; i < *num; i++) { struct record_cluster *r = reclist_read_record(se->reclist); @@ -992,7 +1293,7 @@ struct record_cluster **show_range_start(struct session *se, } #if USE_TIMING yaz_timing_stop(t); - yaz_log(YLOG_LOG, "show %6.5f %3.2f %3.2f", + yaz_log(YLOG_LOG, "show %6.5f %3.2f %3.2f", yaz_timing_get_real(t), yaz_timing_get_user(t), yaz_timing_get_sys(t)); yaz_timing_destroy(&t); @@ -1002,7 +1303,7 @@ struct record_cluster **show_range_start(struct session *se, void show_range_stop(struct session *se, struct record_cluster **recs) { - session_leave(se); + session_leave(se, "show_range_stop"); } void statistics(struct session *se, struct statistics *stat) @@ -1011,11 +1312,13 @@ void statistics(struct session *se, struct statistics *stat) int count = 0; memset(stat, 0, sizeof(*stat)); - for (l = se->clients; l; l = l->next) + stat->num_hits = 0; + for (l = se->clients_active; l; l = l->next) { struct client *cl = l->client; if (!client_get_connection(cl)) stat->num_no_connection++; + stat->num_hits += client_get_hits(cl); switch (client_get_state(cl)) { case Client_Connecting: stat->num_connecting++; break; @@ -1027,7 +1330,6 @@ void statistics(struct session *se, struct statistics *stat) } count++; } - stat->num_hits = se->total_hits; stat->num_records = se->total_records; stat->num_clients = count; @@ -1039,7 +1341,7 @@ static struct record_metadata *record_metadata_init( { struct record_metadata *rec_md = record_metadata_create(nmem); struct record_metadata_attr **attrp = &rec_md->attributes; - + for (; attr; attr = attr->next) { if (attr->children && attr->children->content) @@ -1064,7 +1366,7 @@ static struct record_metadata *record_metadata_init( char *p = nmem_strdup(nmem, value); p = normalize7bit_generic(p, " ,/.:(["); - + rec_md->data.text.disp = p; rec_md->data.text.sort = 0; } @@ -1107,15 +1409,15 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, if (value) { const char *norm_str; - pp2_relevance_token_t prt = - pp2_relevance_tokenize(service->mergekey_pct); - - pp2_relevance_first(prt, (const char *) value, 0); + pp2_charset_token_t prt = + pp2_charset_token_create(service->charsets, "mergekey"); + + pp2_charset_token_first(prt, (const char *) value, 0); if (wrbuf_len(norm_wr) > 0) wrbuf_puts(norm_wr, " "); wrbuf_puts(norm_wr, name); while ((norm_str = - pp2_relevance_token_next(prt))) + pp2_charset_token_next(prt))) { if (*norm_str) { @@ -1124,7 +1426,7 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, } } xmlFree(value); - pp2_relevance_token_destroy(prt); + pp2_charset_token_destroy(prt); no_found++; } } @@ -1146,11 +1448,11 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, if (mergekey) { const char *norm_str; - pp2_relevance_token_t prt = - pp2_relevance_tokenize(service->mergekey_pct); + pp2_charset_token_t prt = + pp2_charset_token_create(service->charsets, "mergekey"); - pp2_relevance_first(prt, (const char *) mergekey, 0); - while ((norm_str = pp2_relevance_token_next(prt))) + pp2_charset_token_first(prt, (const char *) mergekey, 0); + while ((norm_str = pp2_charset_token_next(prt))) { if (*norm_str) { @@ -1159,7 +1461,7 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, wrbuf_puts(norm_wr, norm_str); } } - pp2_relevance_token_destroy(prt); + pp2_charset_token_destroy(prt); xmlFree(mergekey); } else @@ -1175,7 +1477,7 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, service, norm_wr); if (r == 0 && ser_md->mergekey == Metadata_mergekey_required) { - /* no mergekey on this one and it is required.. + /* no mergekey on this one and it is required.. Generate unique key instead */ wrbuf_rewind(norm_wr); break; @@ -1187,8 +1489,13 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, /* generate unique key if none is not generated already or is empty */ if (wrbuf_len(norm_wr) == 0) { - wrbuf_printf(norm_wr, "%s-%d", - client_get_database(cl)->database->url, record_no); + wrbuf_printf(norm_wr, "position: %s-%d", + client_get_id(cl), record_no); + } + else + { + const char *lead = "content: "; + wrbuf_insert(norm_wr, 0, lead, strlen(lead)); } if (wrbuf_len(norm_wr) > 0) mergekey_norm = nmem_strdup(nmem, wrbuf_cstr(norm_wr)); @@ -1196,7 +1503,7 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, return mergekey_norm; } -/** \brief see if metadata for pz:recordfilter exists +/** \brief see if metadata for pz:recordfilter exists \param root xml root element of normalized record \param sdb session database for client \retval 0 if there is no metadata for pz:recordfilter @@ -1205,7 +1512,7 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, If there is no pz:recordfilter defined, this function returns 1 as well. */ - + static int check_record_filter(xmlNode *root, struct session_database *sdb) { int match = 0; @@ -1283,20 +1590,19 @@ int ingest_record(struct client *cl, const char *rec, xmlDoc *xdoc = normalize_record(se, sdb, service, rec, nmem); xmlNode *root; const char *mergekey_norm; - + if (!xdoc) return -1; - + root = xmlDocGetRootElement(xdoc); - + if (!check_record_filter(root, sdb)) { - session_log(se, YLOG_LOG, "Filtered out record no %d from %s", - record_no, sdb->database->url); + session_log(se, YLOG_LOG, "Filtered out record no %d from %s", record_no, sdb->database->id); xmlFreeDoc(xdoc); return -2; } - + mergekey_norm = get_mergekey(xdoc, cl, record_no, service, nmem); if (!mergekey_norm) { @@ -1304,15 +1610,87 @@ int ingest_record(struct client *cl, const char *rec, xmlFreeDoc(xdoc); return -1; } - session_enter(se); + session_enter(se, "ingest_record"); if (client_get_session(cl) == se) ret = ingest_to_cluster(cl, xdoc, root, record_no, mergekey_norm); - session_leave(se); - + session_leave(se, "ingest_record"); + xmlFreeDoc(xdoc); return ret; } +// Skip record on non-zero +static int check_limit_local(struct client *cl, + struct record *record, + int record_no) +{ + int skip_record = 0; + struct session *se = client_get_session(cl); + struct conf_service *service = se->service; + NMEM nmem_tmp = nmem_create(); + struct session_database *sdb = client_get_database(cl); + int l = 0; + while (!skip_record) + { + struct conf_metadata *ser_md = 0; + struct record_metadata *rec_md = 0; + int md_field_id; + char **values = 0; + int i, num_v = 0; + + const char *name = client_get_facet_limit_local(cl, sdb, &l, nmem_tmp, &num_v, &values); + if (!name) + break; + + md_field_id = conf_service_metadata_field_id(service, name); + if (md_field_id < 0) + { + skip_record = 1; + break; + } + ser_md = &service->metadata[md_field_id]; + rec_md = record->metadata[md_field_id]; + yaz_log(YLOG_DEBUG, "check limit local %s", name); + for (i = 0; i < num_v; ) + { + if (rec_md) + { + if (ser_md->type == Metadata_type_year + || ser_md->type == Metadata_type_date) + { + int y = atoi(values[i]); + if (y >= rec_md->data.number.min + && y <= rec_md->data.number.max) + break; + } + else + { + yaz_log(YLOG_DEBUG, "cmp: '%s' '%s'", rec_md->data.text.disp, values[i]); + if (!strcmp(rec_md->data.text.disp, values[i])) + { + // Value equals, should not be filtered. + break; + } + } + rec_md = rec_md->next; + } + else + { + rec_md = record->metadata[md_field_id]; + i++; + } + } + // At end , not match + if (i == num_v) + { + skip_record = 1; + break; + } + } + nmem_destroy(nmem_tmp); + return skip_record; +} + static int ingest_to_cluster(struct client *cl, xmlDoc *xdoc, xmlNode *root, @@ -1322,85 +1700,53 @@ static int ingest_to_cluster(struct client *cl, xmlNode *n; xmlChar *type = 0; xmlChar *value = 0; - struct session_database *sdb = client_get_database(cl); struct session *se = client_get_session(cl); struct conf_service *service = se->service; - struct record *record = record_create(se->nmem, + int term_factor = 1; + struct record_cluster *cluster; + struct session_database *sdb = client_get_database(cl); + struct record *record = record_create(se->nmem, service->num_metadata, service->num_sortkeys, cl, record_no); - struct record_cluster *cluster = reclist_insert(se->reclist, - service, - record, - mergekey_norm, - &se->total_merged); - - const char *use_term_factor_str = session_setting_oneval(sdb, PZ_TERMLIST_TERM_FACTOR); - int use_term_factor = 0; - int term_factor = 1; - if (use_term_factor_str && use_term_factor_str[0] != 0) - use_term_factor = atoi(use_term_factor_str); - if (use_term_factor) { - int maxrecs = client_get_maxrecs(cl); - int hits = (int) client_get_hits(cl); - term_factor = MAX(hits, maxrecs) / MAX(1, maxrecs); - assert(term_factor >= 1); - yaz_log(YLOG_DEBUG, "Using term factor: %d (%d / %d)", term_factor, MAX(hits, maxrecs), MAX(1, maxrecs)); - } - if (!cluster) - return -1; - if (global_parameters.dump_records) - session_log(se, YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid, - sdb->database->url, record_no); - relevance_newrec(se->relevance, cluster); - - // now parsing XML record and adding data to cluster or record metadata for (n = root->children; n; n = n->next) { - pp2_relevance_token_t prt; if (type) xmlFree(type); if (value) xmlFree(value); type = value = 0; - + if (n->type != XML_ELEMENT_NODE) continue; if (!strcmp((const char *) n->name, "metadata")) { struct conf_metadata *ser_md = 0; - struct conf_sortkey *ser_sk = 0; struct record_metadata **wheretoput = 0; struct record_metadata *rec_md = 0; int md_field_id = -1; - int sk_field_id = -1; - + type = xmlGetProp(n, (xmlChar *) "type"); value = xmlNodeListGetString(xdoc, n->children, 1); - + if (!type || !value || !*value) continue; - - md_field_id + + md_field_id = conf_service_metadata_field_id(service, (const char *) type); if (md_field_id < 0) { if (se->number_of_warnings_unknown_metadata == 0) { - session_log(se, YLOG_WARN, + session_log(se, YLOG_WARN, "Ignoring unknown metadata element: %s", type); } se->number_of_warnings_unknown_metadata++; continue; } - + ser_md = &service->metadata[md_field_id]; - - if (ser_md->sortkey_offset >= 0){ - sk_field_id = ser_md->sortkey_offset; - ser_sk = &service->sortkeys[sk_field_id]; - } // non-merged metadata rec_md = record_metadata_init(se->nmem, (const char *) value, @@ -1415,71 +1761,157 @@ static int ingest_to_cluster(struct client *cl, while (*wheretoput) wheretoput = &(*wheretoput)->next; *wheretoput = rec_md; + } + } + + if (check_limit_local(cl, record, record_no)) + { + session_log(se, YLOG_LOG, "Facet filtered out record no %d from %s", + record_no, sdb->database->id); + if (type) + xmlFree(type); + if (value) + xmlFree(value); + return -2; + } + cluster = reclist_insert(se->reclist, service, record, + mergekey_norm, &se->total_merged); + if (!cluster) + return -1; + + { + const char *use_term_factor_str = + session_setting_oneval(sdb, PZ_TERMLIST_TERM_FACTOR); + if (use_term_factor_str && use_term_factor_str[0] == '1') + { + int maxrecs = client_get_maxrecs(cl); + int hits = (int) client_get_hits(cl); + term_factor = MAX(hits, maxrecs) / MAX(1, maxrecs); + assert(term_factor >= 1); + yaz_log(YLOG_DEBUG, "Using term factor: %d (%d / %d)", term_factor, MAX(hits, maxrecs), MAX(1, maxrecs)); + } + } + + if (global_parameters.dump_records) + session_log(se, YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid, + sdb->database->id, record_no); + + + relevance_newrec(se->relevance, cluster); + + // now parsing XML record and adding data to cluster or record metadata + for (n = root->children; n; n = n->next) + { + pp2_charset_token_t prt; + if (type) + xmlFree(type); + if (value) + xmlFree(value); + type = value = 0; + + if (n->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) n->name, "metadata")) + { + struct conf_metadata *ser_md = 0; + struct conf_sortkey *ser_sk = 0; + struct record_metadata **wheretoput = 0; + struct record_metadata *rec_md = 0; + int md_field_id = -1; + int sk_field_id = -1; + const char *rank; + xmlChar *xml_rank; + + type = xmlGetProp(n, (xmlChar *) "type"); + value = xmlNodeListGetString(xdoc, n->children, 1); + + if (!type || !value || !*value) + continue; + + md_field_id + = conf_service_metadata_field_id(service, (const char *) type); + if (md_field_id < 0) + continue; + + ser_md = &service->metadata[md_field_id]; + + if (ser_md->sortkey_offset >= 0) + { + sk_field_id = ser_md->sortkey_offset; + ser_sk = &service->sortkeys[sk_field_id]; + } // merged metadata rec_md = record_metadata_init(se->nmem, (const char *) value, ser_md->type, 0); + if (!rec_md) + continue; + + xml_rank = xmlGetProp(n, (xmlChar *) "rank"); + rank = xml_rank ? (const char *) xml_rank : ser_md->rank; + wheretoput = &cluster->metadata[md_field_id]; // and polulate with data: // assign cluster or record based on merge action if (ser_md->merge == Metadata_merge_unique) { - struct record_metadata *mnode; - for (mnode = *wheretoput; mnode; mnode = mnode->next) - if (!strcmp((const char *) mnode->data.text.disp, + while (*wheretoput) + { + if (!strcmp((const char *) (*wheretoput)->data.text.disp, rec_md->data.text.disp)) break; - if (!mnode) - { - rec_md->next = *wheretoput; - *wheretoput = rec_md; + wheretoput = &(*wheretoput)->next; } + if (!*wheretoput) + *wheretoput = rec_md; } else if (ser_md->merge == Metadata_merge_longest) { - if (!*wheretoput - || strlen(rec_md->data.text.disp) + if (!*wheretoput + || strlen(rec_md->data.text.disp) > strlen((*wheretoput)->data.text.disp)) { *wheretoput = rec_md; if (ser_sk) { const char *sort_str = 0; - int skip_article = + int skip_article = ser_sk->type == Metadata_sortkey_skiparticle; if (!cluster->sortkeys[sk_field_id]) - cluster->sortkeys[sk_field_id] = - nmem_malloc(se->nmem, + cluster->sortkeys[sk_field_id] = + nmem_malloc(se->nmem, sizeof(union data_types)); - - prt = pp2_relevance_tokenize(service->sort_pct); - pp2_relevance_first(prt, rec_md->data.text.disp, - skip_article); + prt = + pp2_charset_token_create(service->charsets, "sort"); + + pp2_charset_token_first(prt, rec_md->data.text.disp, + skip_article); + + pp2_charset_token_next(prt); - pp2_relevance_token_next(prt); - sort_str = pp2_get_sort(prt); - - cluster->sortkeys[sk_field_id]->text.disp = + + cluster->sortkeys[sk_field_id]->text.disp = rec_md->data.text.disp; if (!sort_str) { sort_str = rec_md->data.text.disp; - session_log(se, YLOG_WARN, + session_log(se, YLOG_WARN, "Could not make sortkey. Bug #1858"); } - cluster->sortkeys[sk_field_id]->text.sort = + cluster->sortkeys[sk_field_id]->text.sort = nmem_strdup(se->nmem, sort_str); - pp2_relevance_token_destroy(prt); + pp2_charset_token_destroy(prt); } } } else if (ser_md->merge == Metadata_merge_all) { - rec_md->next = *wheretoput; + while (*wheretoput) + wheretoput = &(*wheretoput)->next; *wheretoput = rec_md; } else if (ser_md->merge == Metadata_merge_range) @@ -1488,7 +1920,7 @@ static int ingest_to_cluster(struct client *cl, { *wheretoput = rec_md; if (ser_sk) - cluster->sortkeys[sk_field_id] + cluster->sortkeys[sk_field_id] = &rec_md->data; } else @@ -1502,12 +1934,12 @@ static int ingest_to_cluster(struct client *cl, } } - - // ranking of _all_ fields enabled ... - if (ser_md->rank) - relevance_countwords(se->relevance, cluster, - (char *) value, ser_md->rank, - ser_md->name); + // ranking of _all_ fields enabled ... + if (rank) + { + relevance_countwords(se->relevance, cluster, + (char *) value, rank, ser_md->name); + } // construct facets ... unless the client already has reported them if (ser_md->termlist && !client_has_facet(cl, (char *) type)) @@ -1529,6 +1961,8 @@ static int ingest_to_cluster(struct client *cl, } // cleaning up + if (xml_rank) + xmlFree(xml_rank); xmlFree(type); xmlFree(value); type = value = 0; @@ -1559,7 +1993,7 @@ void session_log(struct session *s, int level, const char *fmt, ...) va_start(ap, fmt); yaz_vsnprintf(buf, sizeof(buf)-30, fmt, ap); - yaz_log(level, "Session (%u): %s", s->session_id, buf); + yaz_log(level, "Session %u: %s", s->session_id, buf); va_end(ap); }