X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Fsession.c;h=0e5427ea9950094aee04019090247f933a46004f;hb=e07a5d5dfcf5be4cc9eb3ab91a60866ff0aee09c;hp=bb0e915f0ee2b6a2135564adcf96e6eacaab51a8;hpb=ca9817524ebe069581ad72f6bca0e9775d61e30c;p=pazpar2-moved-to-github.git diff --git a/src/session.c b/src/session.c index bb0e915..0e5427e 100644 --- a/src/session.c +++ b/src/session.c @@ -1,5 +1,5 @@ /* This file is part of Pazpar2. - Copyright (C) 2006-2010 Index Data + Copyright (C) 2006-2011 Index Data Pazpar2 is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #endif +#include #include #include #include @@ -34,9 +35,13 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #if HAVE_UNISTD_H #include #endif +#ifdef WIN32 +#include +#endif #include #include #include +#include #include #include @@ -51,12 +56,14 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #include #include +#include #define USE_TIMING 0 #if USE_TIMING #include #endif +#include "ppmutex.h" #include "parameters.h" #include "session.h" #include "eventl.h" @@ -73,13 +80,63 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #define MAX_CHUNK 15 +#define MAX(a,b) ((a)>(b)?(a):(b)) + // Note: Some things in this structure will eventually move to configuration struct parameters global_parameters = { 0, // dump_records - 0 // debug_mode + 0, // debug_mode + 0, // predictable sessions +}; + +struct client_list { + struct client *client; + struct client_list *next; +}; + +struct session_sorted_results { + const char *field; + int increasing; + struct session_sorted_results *next; }; +/* session counting (1) , disable client counting (0) */ +static YAZ_MUTEX g_session_mutex = 0; +static int no_sessions = 0; +static int no_session_total = 0; + +static int session_use(int delta) +{ + int sessions; + if (!g_session_mutex) + yaz_mutex_create(&g_session_mutex); + yaz_mutex_enter(g_session_mutex); + no_sessions += delta; + if (delta > 0) + no_session_total += delta; + sessions = no_sessions; + yaz_mutex_leave(g_session_mutex); + yaz_log(YLOG_DEBUG, "%s sessions=%d", delta == 0 ? "" : (delta > 0 ? "INC" : "DEC"), no_sessions); + return sessions; +} + +int sessions_count(void) +{ + return session_use(0); +} + +int session_count_total(void) +{ + int total = 0; + if (!g_session_mutex) + return 0; + yaz_mutex_enter(g_session_mutex); + total = no_session_total; + yaz_mutex_leave(g_session_mutex); + return total; +} + static void log_xml_doc(xmlDoc *doc) { FILE *lf = yaz_log_file(); @@ -100,67 +157,92 @@ static void log_xml_doc(xmlDoc *doc) static void session_enter(struct session *s) { - yaz_mutex_enter(s->mutex); + yaz_mutex_enter(s->session_mutex); } static void session_leave(struct session *s) { - yaz_mutex_leave(s->mutex); + yaz_mutex_leave(s->session_mutex); } -// Recursively traverse query structure to extract terms. -void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) +void add_facet(struct session *s, const char *type, const char *value, int count) { - char **words; - int numwords; + struct conf_service *service = s->service; + pp2_charset_token_t prt; + const char *facet_component; + WRBUF facet_wrbuf = wrbuf_alloc(); + WRBUF display_wrbuf = wrbuf_alloc(); int i; + const char *icu_chain_id = 0; - switch (n->kind) + for (i = 0; i < service->num_metadata; i++) + if (!strcmp((service->metadata + i)->name, type)) + icu_chain_id = (service->metadata + i)->facetrule; + if (!icu_chain_id) + icu_chain_id = "facet"; + prt = pp2_charset_token_create(service->charsets, icu_chain_id); + if (!prt) { - case CCL_RPN_AND: - case CCL_RPN_OR: - case CCL_RPN_NOT: - case CCL_RPN_PROX: - pull_terms(nmem, n->u.p[0], termlist, num); - pull_terms(nmem, n->u.p[1], termlist, num); - break; - case CCL_RPN_TERM: - nmem_strsplit(nmem, " ", n->u.t.term, &words, &numwords); - for (i = 0; i < numwords; i++) - termlist[(*num)++] = words[i]; - break; - default: // NOOP - break; - } -} - - -static void add_facet(struct session *s, const char *type, const char *value) -{ - int i; - - if (!*value) + yaz_log(YLOG_FATAL, "Unknown ICU chain '%s' for facet of type '%s'", + icu_chain_id, type); + wrbuf_destroy(facet_wrbuf); + wrbuf_destroy(display_wrbuf); return; - for (i = 0; i < s->num_termlists; i++) - if (!strcmp(s->termlists[i].name, type)) - break; - if (i == s->num_termlists) + } + pp2_charset_token_first(prt, value, 0); + while ((facet_component = pp2_charset_token_next(prt))) { - if (i == SESSION_MAX_TERMLISTS) + const char *display_component; + if (*facet_component) { - yaz_log(YLOG_FATAL, "Too many termlists"); - return; + if (wrbuf_len(facet_wrbuf)) + wrbuf_puts(facet_wrbuf, " "); + wrbuf_puts(facet_wrbuf, facet_component); } - - s->termlists[i].name = nmem_strdup(s->nmem, type); - s->termlists[i].termlist - = termlist_create(s->nmem, TERMLIST_HIGH_SCORE); - s->num_termlists = i + 1; + display_component = pp2_get_display(prt); + if (display_component) + { + if (wrbuf_len(display_wrbuf)) + wrbuf_puts(display_wrbuf, " "); + wrbuf_puts(display_wrbuf, display_component); + } + } + pp2_charset_token_destroy(prt); + + if (wrbuf_len(facet_wrbuf)) + { + int i; + for (i = 0; i < s->num_termlists; i++) + if (!strcmp(s->termlists[i].name, type)) + break; + if (i == s->num_termlists) + { + if (i == SESSION_MAX_TERMLISTS) + { + session_log(s, YLOG_FATAL, "Too many termlists"); + wrbuf_destroy(facet_wrbuf); + wrbuf_destroy(display_wrbuf); + return; + } + + s->termlists[i].name = nmem_strdup(s->nmem, type); + s->termlists[i].termlist + = termlist_create(s->nmem, TERMLIST_HIGH_SCORE); + s->num_termlists = i + 1; + } + +#if 0 + session_log(s, YLOG_DEBUG, "Facets for %s: %s norm:%s (%d)", type, value, wrbuf_cstr(facet_wrbuf), count); +#endif + termlist_insert(s->termlists[i].termlist, wrbuf_cstr(display_wrbuf), + wrbuf_cstr(facet_wrbuf), count); } - termlist_insert(s->termlists[i].termlist, value); + wrbuf_destroy(facet_wrbuf); + wrbuf_destroy(display_wrbuf); } -static xmlDoc *record_to_xml(struct session_database *sdb, const char *rec) +static xmlDoc *record_to_xml(struct session *se, + struct session_database *sdb, const char *rec) { struct database *db = sdb->database; xmlDoc *rdoc = 0; @@ -169,14 +251,14 @@ static xmlDoc *record_to_xml(struct session_database *sdb, const char *rec) if (!rdoc) { - yaz_log(YLOG_FATAL, "Non-wellformed XML received from %s", - db->url); + session_log(se, YLOG_FATAL, "Non-wellformed XML received from %s", + db->id); return 0; } if (global_parameters.dump_records) { - yaz_log(YLOG_LOG, "Un-normalized record from %s", db->url); + session_log(se, YLOG_LOG, "Un-normalized record from %s", db->id); log_xml_doc(rdoc); } @@ -248,11 +330,12 @@ static void insert_settings_values(struct session_database *sdb, xmlDoc *doc, } } -static xmlDoc *normalize_record(struct session_database *sdb, +static xmlDoc *normalize_record(struct session *se, + struct session_database *sdb, struct conf_service *service, const char *rec, NMEM nmem) { - xmlDoc *rdoc = record_to_xml(sdb, rec); + xmlDoc *rdoc = record_to_xml(se, sdb, rec); if (rdoc) { @@ -262,7 +345,8 @@ static xmlDoc *normalize_record(struct session_database *sdb, if (normalize_record_transform(sdb->map, &rdoc, (const char **)parms)) { - yaz_log(YLOG_WARN, "Normalize failed from %s", sdb->database->url); + session_log(se, YLOG_WARN, "Normalize failed from %s", + sdb->database->id); } else { @@ -270,8 +354,8 @@ static xmlDoc *normalize_record(struct session_database *sdb, if (global_parameters.dump_records) { - yaz_log(YLOG_LOG, "Normalized record from %s", - sdb->database->url); + session_log(se, YLOG_LOG, "Normalized record from %s", + sdb->database->id); log_xml_doc(rdoc); } } @@ -323,7 +407,7 @@ static int prepare_map(struct session *se, struct session_database *sdb) if (!sdb->settings) { - yaz_log(YLOG_WARN, "No settings on %s", sdb->database->url); + session_log(se, YLOG_WARN, "No settings on %s", sdb->database->id); return -1; } if ((s = session_setting_oneval(sdb, PZ_XSLT))) @@ -349,11 +433,12 @@ static int prepare_map(struct session *se, struct session_database *sdb) } else { - yaz_log(YLOG_WARN, "No pz:requestsyntax for auto stylesheet"); + session_log(se, YLOG_WARN, + "No pz:requestsyntax for auto stylesheet"); } } sdb->map = normalize_cache_get(se->normalize_cache, - se->service, s); + se->service->server->config, s); if (!sdb->map) return -1; } @@ -367,8 +452,8 @@ static int prepare_session_database(struct session *se, { if (!sdb->settings) { - yaz_log(YLOG_WARN, - "No settings associated with %s", sdb->database->url); + session_log(se, YLOG_WARN, + "No settings associated with %s", sdb->database->id); return -1; } if (sdb->settings[PZ_XSLT] && !sdb->map) @@ -395,17 +480,27 @@ int session_set_watch(struct session *s, int what, session_watchfun fun, void *data, struct http_channel *chan) { + int ret; + session_enter(s); if (s->watchlist[what].fun) - return -1; - s->watchlist[what].fun = fun; - s->watchlist[what].data = data; - s->watchlist[what].obs = http_add_observer(chan, &s->watchlist[what], - session_watch_cancel); - return 0; + ret = -1; + else + { + + s->watchlist[what].fun = fun; + s->watchlist[what].data = data; + s->watchlist[what].obs = http_add_observer(chan, &s->watchlist[what], + session_watch_cancel); + ret = 0; + } + session_leave(s); + return ret; } void session_alert_watch(struct session *s, int what) { + assert(s); + session_enter(s); if (s->watchlist[what].fun) { /* our watch is no longer associated with http_channel */ @@ -413,7 +508,7 @@ void session_alert_watch(struct session *s, int what) session_watchfun fun; http_remove_observer(s->watchlist[what].obs); - fun = s->watchlist[what].fun; + fun = s->watchlist[what].fun; data = s->watchlist[what].data; /* reset watch before fun is invoked - in case fun wants to set @@ -422,17 +517,52 @@ void session_alert_watch(struct session *s, int what) s->watchlist[what].data = 0; s->watchlist[what].obs = 0; + session_leave(s); + session_log(s, YLOG_DEBUG, + "Alert Watch: %d calling function: %p", what, fun); fun(data); } + else + session_leave(s); } //callback for grep_databases -static void select_targets_callback(void *context, struct session_database *db) +static void select_targets_callback(struct session *se, + struct session_database *db) { - struct session *se = (struct session*) context; - struct client *cl = client_create(); + struct client *cl = client_create(db->database->id); + struct client_list *l; + client_set_database(cl, db); + client_set_session(cl, se); + + l = xmalloc(sizeof(*l)); + l->client = cl; + l->next = se->clients; + se->clients = l; +} + +static void session_remove_clients(struct session *se) +{ + struct client_list *l; + + session_enter(se); + l = se->clients; + se->clients = 0; + session_leave(se); + + while (l) + { + struct client_list *l_next = l->next; + client_lock(l->client); + client_set_session(l->client, 0); + client_set_database(l->client, 0); + client_unlock(l->client); + client_destroy(l->client); + xfree(l); + l = l_next; + } } // Associates a set of clients with a session; @@ -440,47 +570,139 @@ static void select_targets_callback(void *context, struct session_database *db) // setting overrides static int select_targets(struct session *se, const char *filter) { - while (se->clients) - client_destroy(se->clients); - return session_grep_databases(se, filter, select_targets_callback); } int session_active_clients(struct session *s) { - struct client *c; + struct client_list *l; int res = 0; - for (c = s->clients; c; c = client_next_in_session(c)) - if (client_is_active(c)) + for (l = s->clients; l; l = l->next) + if (client_is_active(l->client)) res++; return res; } +int session_is_preferred_clients_ready(struct session *s) +{ + struct client_list *l; + int res = 0; + + for (l = s->clients; l; l = l->next) + if (client_is_active_preferred(l->client)) + res++; + session_log(s, YLOG_DEBUG, "Has %d active preferred clients.", res); + return res == 0; +} -enum pazpar2_error_code search(struct session *se, - const char *query, - const char *startrecs, const char *maxrecs, - const char *filter, - const char **addinfo) +static const char *get_strategy_plus_sort(struct client *l, const char *field) +{ + struct session_database *sdb = client_get_database(l); + struct setting *s; + + const char *strategy_plus_sort = 0; + + for (s = sdb->settings[PZ_SORTMAP]; s; s = s->next) + { + char *p = strchr(s->name + 3, ':'); + if (!p) + { + yaz_log(YLOG_WARN, "Malformed sortmap name: %s", s->name); + continue; + } + p++; + if (!strcmp(p, field)) + { + strategy_plus_sort = s->value; + break; + } + } + return strategy_plus_sort; +} + +void session_sort(struct session *se, const char *field, int increasing) +{ + struct session_sorted_results *sr; + struct client_list *l; + + session_enter(se); + + /* see if we already have sorted for this critieria */ + for (sr = se->sorted_results; sr; sr = sr->next) + { + if (!strcmp(field, sr->field) && increasing == sr->increasing) + break; + } + if (sr) + { + yaz_log(YLOG_LOG, "search_sort: field=%s increasing=%d already fetched", + field, increasing); + session_leave(se); + return; + } + yaz_log(YLOG_LOG, "search_sort: field=%s increasing=%d must fetch", + field, increasing); + sr = nmem_malloc(se->nmem, sizeof(*sr)); + sr->field = nmem_strdup(se->nmem, field); + sr->increasing = increasing; + sr->next = se->sorted_results; + se->sorted_results = sr; + + for (l = se->clients; l; l = l->next) + { + struct client *cl = l->client; + const char *strategy_plus_sort = get_strategy_plus_sort(cl, field); + if (strategy_plus_sort) + { + struct timeval tval; + if (client_prep_connection(cl, se->service->z3950_operation_timeout, + se->service->z3950_session_timeout, + se->service->server->iochan_man, + &tval)) + client_start_search(cl, strategy_plus_sort, increasing); + } + } + session_leave(se); +} + +enum pazpar2_error_code session_search(struct session *se, + const char *query, + const char *startrecs, + const char *maxrecs, + const char *filter, + const char *limit, + const char **addinfo, + const char *sort_field, int increasing) { int live_channels = 0; int no_working = 0; int no_failed = 0; - struct client *cl; + struct client_list *l; + struct timeval tval; + facet_limits_t facet_limits; - yaz_log(YLOG_DEBUG, "Search"); + session_log(se, YLOG_DEBUG, "Search"); *addinfo = 0; + + session_remove_clients(se); session_enter(se); - nmem_reset(se->nmem); - se->relevance = 0; - se->total_records = se->total_hits = se->total_merged = 0; reclist_destroy(se->reclist); se->reclist = 0; + relevance_destroy(&se->relevance); + nmem_reset(se->nmem); + se->total_records = se->total_merged = 0; se->num_termlists = 0; + + /* reset list of sorted results and clear to relevance search */ + se->sorted_results = nmem_malloc(se->nmem, sizeof(*se->sorted_results)); + se->sorted_results->field = nmem_strdup(se->nmem, sort_field); + se->sorted_results->increasing = increasing; + se->sorted_results->next = 0; + live_channels = select_targets(se, filter); if (!live_channels) { @@ -489,26 +711,41 @@ enum pazpar2_error_code search(struct session *se, } se->reclist = reclist_create(se->nmem); - for (cl = se->clients; cl; cl = client_next_in_session(cl)) + yaz_gettimeofday(&tval); + + tval.tv_sec += 5; + + facet_limits = facet_limits_create(limit); + if (!facet_limits) + { + *addinfo = "limit"; + session_leave(se); + return PAZPAR2_MALFORMED_PARAMETER_VALUE; + } + for (l = se->clients; l; l = l->next) { + struct client *cl = l->client; + const char *strategy_plus_sort = get_strategy_plus_sort(cl, sort_field); + if (maxrecs) client_set_maxrecs(cl, atoi(maxrecs)); if (startrecs) client_set_startrecs(cl, atoi(startrecs)); if (prepare_session_database(se, client_get_database(cl)) < 0) - continue; - // Parse query for target - if (client_parse_query(cl, query) < 0) + ; + else if (client_parse_query(cl, query, facet_limits) < 0) no_failed++; else { no_working++; if (client_prep_connection(cl, se->service->z3950_operation_timeout, se->service->z3950_session_timeout, - se->service->server->iochan_man)) - client_start_search(cl); + se->service->server->iochan_man, + &tval)) + client_start_search(cl, strategy_plus_sort, increasing); } } + facet_limits_destroy(facet_limits); session_leave(se); if (no_working == 0) { @@ -567,8 +804,6 @@ static struct session_database *load_session_database(struct session *se, { struct database *db = new_database(id, se->session_nmem); - resolve_database(se->service, db); - session_init_databases_fun((void*) se, db); // New sdb is head of se->databases list @@ -582,7 +817,7 @@ static struct session_database *find_session_database(struct session *se, struct session_database *sdb; for (sdb = se->databases; sdb; sdb = sdb->next) - if (!strcmp(sdb->database->url, id)) + if (!strcmp(sdb->database->id, id)) return sdb; return load_session_database(se, id); } @@ -618,32 +853,53 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, } } -void destroy_session(struct session *s) -{ +void session_destroy(struct session *se) { struct session_database *sdb; + session_log(se, YLOG_DEBUG, "Destroying"); + session_use(-1); + session_remove_clients(se); - while (s->clients) - client_destroy(s->clients); - for (sdb = s->databases; sdb; sdb = sdb->next) + for (sdb = se->databases; sdb; sdb = sdb->next) session_database_destroy(sdb); - normalize_cache_destroy(s->normalize_cache); - reclist_destroy(s->reclist); - nmem_destroy(s->nmem); - service_destroy(s->service); - yaz_mutex_destroy(&s->mutex); - wrbuf_destroy(s->wrbuf); + normalize_cache_destroy(se->normalize_cache); + relevance_destroy(&se->relevance); + reclist_destroy(se->reclist); + nmem_destroy(se->nmem); + service_destroy(se->service); + yaz_mutex_destroy(&se->session_mutex); +} + +/* Depreciated: use session_destroy */ +void destroy_session(struct session *se) +{ + session_destroy(se); } -struct session *new_session(NMEM nmem, struct conf_service *service) +size_t session_get_memory_status(struct session *session) { + size_t session_nmem; + if (session == 0) + return 0; + session_enter(session); + session_nmem = nmem_total(session->nmem); + session_leave(session); + return session_nmem; +} + + +struct session *new_session(NMEM nmem, struct conf_service *service, + unsigned session_id) { int i; struct session *session = nmem_malloc(nmem, sizeof(*session)); - yaz_log(YLOG_DEBUG, "New Pazpar2 session"); + char tmp_str[50]; + sprintf(tmp_str, "session#%u", session_id); + + session->session_id = session_id; + session_log(session, YLOG_DEBUG, "New"); session->service = service; session->relevance = 0; - session->total_hits = 0; session->total_records = 0; session->number_of_warnings_unknown_elements = 0; session->number_of_warnings_unknown_metadata = 0; @@ -652,7 +908,6 @@ struct session *new_session(NMEM nmem, struct conf_service *service) session->clients = 0; session->session_nmem = nmem; session->nmem = nmem_create(); - session->wrbuf = wrbuf_alloc(); session->databases = 0; for (i = 0; i <= SESSION_WATCH_MAX; i++) { @@ -660,31 +915,32 @@ struct session *new_session(NMEM nmem, struct conf_service *service) session->watchlist[i].fun = 0; } session->normalize_cache = normalize_cache_create(); - session->mutex = 0; - yaz_mutex_create(&session->mutex); - + session->session_mutex = 0; + pazpar2_mutex_create(&session->session_mutex, tmp_str); + session_use(1); return session; } -struct hitsbytarget *hitsbytarget(struct session *se, int *count, NMEM nmem) +static struct hitsbytarget *hitsbytarget_nb(struct session *se, + int *count, NMEM nmem) { struct hitsbytarget *res = 0; - struct client *cl; + struct client_list *l; size_t sz = 0; - session_enter(se); - for (cl = se->clients; cl; cl = client_next_in_session(cl)) + for (l = se->clients; l; l = l->next) sz++; res = nmem_malloc(nmem, sizeof(*res) * sz); *count = 0; - for (cl = se->clients; cl; cl = client_next_in_session(cl)) + for (l = se->clients; l; l = l->next) { + struct client *cl = l->client; WRBUF w = wrbuf_alloc(); const char *name = session_setting_oneval(client_get_database(cl), PZ_NAME); - res[*count].id = client_get_database(cl)->database->url; + res[*count].id = client_get_id(cl); res[*count].name = *name ? name : "Unknown"; res[*count].hits = client_get_hits(cl); res[*count].records = client_get_num_records(cl); @@ -692,29 +948,151 @@ struct hitsbytarget *hitsbytarget(struct session *se, int *count, NMEM nmem) res[*count].state = client_get_state_str(cl); res[*count].connected = client_get_connection(cl) ? 1 : 0; session_settings_dump(se, client_get_database(cl), w); - res[*count].settings_xml = w; + res[*count].settings_xml = nmem_strdup(nmem, wrbuf_cstr(w)); + wrbuf_destroy(w); (*count)++; } - session_leave(se); return res; } -struct termlist_score **termlist(struct session *s, const char *name, int *num) +struct hitsbytarget *get_hitsbytarget(struct session *se, int *count, NMEM nmem) +{ + struct hitsbytarget *p; + session_enter(se); + p = hitsbytarget_nb(se, count, nmem); + session_leave(se); + return p; +} + +struct termlist_score **get_termlist_score(struct session *se, + const char *name, int *num) { int i; struct termlist_score **tl = 0; - session_enter(s); - for (i = 0; i < s->num_termlists; i++) - if (!strcmp((const char *) s->termlists[i].name, name)) + session_enter(se); + for (i = 0; i < se->num_termlists; i++) + if (!strcmp((const char *) se->termlists[i].name, name)) { - tl = termlist_highscore(s->termlists[i].termlist, num); + tl = termlist_highscore(se->termlists[i].termlist, num); break; } - session_leave(s); + session_leave(se); return tl; } +// Compares two hitsbytarget nodes by hitcount +static int cmp_ht(const void *p1, const void *p2) +{ + const struct hitsbytarget *h1 = p1; + const struct hitsbytarget *h2 = p2; + return h2->hits - h1->hits; +} + +static int targets_termlist_nb(WRBUF wrbuf, struct session *se, int num, + NMEM nmem) +{ + struct hitsbytarget *ht; + int count, i; + + ht = hitsbytarget_nb(se, &count, nmem); + qsort(ht, count, sizeof(struct hitsbytarget), cmp_ht); + for (i = 0; i < count && i < num && ht[i].hits > 0; i++) + { + + // do only print terms which have display names + + wrbuf_puts(wrbuf, "\n"); + + wrbuf_puts(wrbuf, ""); + wrbuf_xmlputs(wrbuf, ht[i].id); + wrbuf_puts(wrbuf, "\n"); + + wrbuf_puts(wrbuf, ""); + if (!ht[i].name || !ht[i].name[0]) + wrbuf_xmlputs(wrbuf, "NO TARGET NAME"); + else + wrbuf_xmlputs(wrbuf, ht[i].name); + wrbuf_puts(wrbuf, "\n"); + + wrbuf_printf(wrbuf, "" ODR_INT_PRINTF "\n", + ht[i].hits); + + wrbuf_puts(wrbuf, ""); + wrbuf_xmlputs(wrbuf, ht[i].state); + wrbuf_puts(wrbuf, "\n"); + + wrbuf_printf(wrbuf, "%d\n", + ht[i].diagnostic); + wrbuf_puts(wrbuf, "\n"); + } + return count; +} + +void perform_termlist(struct http_channel *c, struct session *se, + const char *name, int num) +{ + int i, j; + NMEM nmem_tmp = nmem_create(); + char **names; + int num_names = 0; + + if (name) + nmem_strsplit(nmem_tmp, ",", name, &names, &num_names); + + session_enter(se); + + for (j = 0; j < num_names; j++) + { + const char *tname; + for (i = 0; i < se->num_termlists; i++) + { + tname = se->termlists[i].name; + if (num_names > 0 && !strcmp(names[j], tname)) + { + struct termlist_score **p = 0; + int len; + p = termlist_highscore(se->termlists[i].termlist, &len); + if (p) + { + int i; + wrbuf_puts(c->wrbuf, "wrbuf, tname); + wrbuf_puts(c->wrbuf, "\">\n"); + for (i = 0; i < len && i < num; i++) + { + // prevent sending empty term elements + if (!p[i]->display_term || !p[i]->display_term[0]) + continue; + + wrbuf_puts(c->wrbuf, ""); + wrbuf_puts(c->wrbuf, ""); + wrbuf_xmlputs(c->wrbuf, p[i]->display_term); + wrbuf_puts(c->wrbuf, ""); + + wrbuf_printf(c->wrbuf, + "%d", + p[i]->frequency); + wrbuf_puts(c->wrbuf, "\n"); + } + wrbuf_puts(c->wrbuf, "\n"); + } + } + } + tname = "xtargets"; + if (num_names > 0 && !strcmp(names[j], tname)) + { + wrbuf_puts(c->wrbuf, "wrbuf, tname); + wrbuf_puts(c->wrbuf, "\">\n"); + targets_termlist_nb(c->wrbuf, se, num, c->nmem); + wrbuf_puts(c->wrbuf, "\n"); + } + } + session_leave(se); + nmem_destroy(nmem_tmp); +} + #ifdef MISSING_HEADERS void report_nmem_stats(void) { @@ -728,50 +1106,52 @@ void report_nmem_stats(void) } #endif -struct record_cluster *show_single_start(struct session *s, const char *id, +struct record_cluster *show_single_start(struct session *se, const char *id, struct record_cluster **prev_r, struct record_cluster **next_r) { - struct record_cluster *r; + struct record_cluster *r = 0; - session_enter(s); - reclist_enter(s->reclist); + session_enter(se); *prev_r = 0; *next_r = 0; - while ((r = reclist_read_record(s->reclist))) + if (se->reclist) { - if (!strcmp(r->recid, id)) + reclist_enter(se->reclist); + while ((r = reclist_read_record(se->reclist))) { - *next_r = reclist_read_record(s->reclist); - break; + if (!strcmp(r->recid, id)) + { + *next_r = reclist_read_record(se->reclist); + break; + } + *prev_r = r; } - *prev_r = r; + reclist_leave(se->reclist); } - reclist_leave(s->reclist); if (!r) - session_leave(s); + session_leave(se); return r; } -void show_single_stop(struct session *s, struct record_cluster *rec) +void show_single_stop(struct session *se, struct record_cluster *rec) { - session_leave(s); + session_leave(se); } -struct record_cluster **show_range_start(struct session *s, +struct record_cluster **show_range_start(struct session *se, struct reclist_sortparms *sp, int start, int *num, int *total, Odr_int *sumhits) { - struct record_cluster **recs = nmem_malloc(s->nmem, *num - * sizeof(struct record_cluster *)); + struct record_cluster **recs; struct reclist_sortparms *spp; int i; #if USE_TIMING yaz_timing_t t = yaz_timing_create(); #endif - - session_enter(s); - if (!s->relevance) + session_enter(se); + recs = nmem_malloc(se->nmem, *num * sizeof(struct record_cluster *)); + if (!se->relevance) { *num = 0; *total = 0; @@ -780,20 +1160,25 @@ struct record_cluster **show_range_start(struct session *s, } else { + struct client_list *l; + for (spp = sp; spp; spp = spp->next) if (spp->type == Metadata_sortkey_relevance) { - relevance_prepare_read(s->relevance, s->reclist); + relevance_prepare_read(se->relevance, se->reclist); break; } - reclist_sort(s->reclist, sp); + reclist_sort(se->reclist, sp); - reclist_enter(s->reclist); - *total = reclist_get_num_records(s->reclist); - *sumhits = s->total_hits; + reclist_enter(se->reclist); + *total = reclist_get_num_records(se->reclist); + + *sumhits = 0; + for (l = se->clients; l; l = l->next) + *sumhits += client_get_hits(l->client); for (i = 0; i < start; i++) - if (!reclist_read_record(s->reclist)) + if (!reclist_read_record(se->reclist)) { *num = 0; recs = 0; @@ -802,7 +1187,7 @@ struct record_cluster **show_range_start(struct session *s, for (i = 0; i < *num; i++) { - struct record_cluster *r = reclist_read_record(s->reclist); + struct record_cluster *r = reclist_read_record(se->reclist); if (!r) { *num = i; @@ -810,7 +1195,7 @@ struct record_cluster **show_range_start(struct session *s, } recs[i] = r; } - reclist_leave(s->reclist); + reclist_leave(se->reclist); } #if USE_TIMING yaz_timing_stop(t); @@ -822,21 +1207,24 @@ struct record_cluster **show_range_start(struct session *s, return recs; } -void show_range_stop(struct session *s, struct record_cluster **recs) +void show_range_stop(struct session *se, struct record_cluster **recs) { - session_leave(s); + session_leave(se); } void statistics(struct session *se, struct statistics *stat) { - struct client *cl; + struct client_list *l; int count = 0; memset(stat, 0, sizeof(*stat)); - for (cl = se->clients; cl; cl = client_next_in_session(cl)) + stat->num_hits = 0; + for (l = se->clients; l; l = l->next) { + struct client *cl = l->client; if (!client_get_connection(cl)) stat->num_no_connection++; + stat->num_hits += client_get_hits(cl); switch (client_get_state(cl)) { case Client_Connecting: stat->num_connecting++; break; @@ -848,7 +1236,6 @@ void statistics(struct session *se, struct statistics *stat) } count++; } - stat->num_hits = se->total_hits; stat->num_records = se->total_records; stat->num_clients = count; @@ -919,22 +1306,24 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, if (!strcmp((const char *) n->name, "metadata")) { xmlChar *type = xmlGetProp(n, (xmlChar *) "type"); - if (!strcmp(name, (const char *) type)) + if (type == NULL) { + yaz_log(YLOG_FATAL, "Missing type attribute on metadata element. Skipping!"); + } + else if (!strcmp(name, (const char *) type)) { xmlChar *value = xmlNodeListGetString(doc, n->children, 1); if (value) { const char *norm_str; - pp2_relevance_token_t prt = - pp2_relevance_tokenize( - service->mergekey_pct, - (const char *) value, 0); + pp2_charset_token_t prt = + pp2_charset_token_create(service->charsets, "mergekey"); + pp2_charset_token_first(prt, (const char *) value, 0); if (wrbuf_len(norm_wr) > 0) wrbuf_puts(norm_wr, " "); wrbuf_puts(norm_wr, name); while ((norm_str = - pp2_relevance_token_next(prt))) + pp2_charset_token_next(prt))) { if (*norm_str) { @@ -943,7 +1332,7 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, } } xmlFree(value); - pp2_relevance_token_destroy(prt); + pp2_charset_token_destroy(prt); no_found++; } } @@ -965,12 +1354,11 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, if (mergekey) { const char *norm_str; - pp2_relevance_token_t prt = - pp2_relevance_tokenize( - service->mergekey_pct, - (const char *) mergekey, 0); - - while ((norm_str = pp2_relevance_token_next(prt))) + pp2_charset_token_t prt = + pp2_charset_token_create(service->charsets, "mergekey"); + + pp2_charset_token_first(prt, (const char *) mergekey, 0); + while ((norm_str = pp2_charset_token_next(prt))) { if (*norm_str) { @@ -979,7 +1367,7 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, wrbuf_puts(norm_wr, norm_str); } } - pp2_relevance_token_destroy(prt); + pp2_charset_token_destroy(prt); xmlFree(mergekey); } else @@ -1008,7 +1396,7 @@ static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, if (wrbuf_len(norm_wr) == 0) { wrbuf_printf(norm_wr, "%s-%d", - client_get_database(cl)->database->url, record_no); + client_get_id(cl), record_no); } if (wrbuf_len(norm_wr) > 0) mergekey_norm = nmem_strdup(nmem, wrbuf_cstr(norm_wr)); @@ -1046,9 +1434,15 @@ static int check_record_filter(xmlNode *root, struct session_database *sdb) if (type) { size_t len; - const char *eq = strchr(s, '~'); - if (eq) - len = eq - s; + int substring; + const char *eq; + + if ((eq = strchr(s, '='))) + substring = 0; + else if ((eq = strchr(s, '~'))) + substring = 1; + if (eq) + len = eq - s; else len = strlen(s); if (len == strlen((const char *)type) && @@ -1057,7 +1451,9 @@ static int check_record_filter(xmlNode *root, struct session_database *sdb) xmlChar *value = xmlNodeGetContent(n); if (value && *value) { - if (!eq || strstr((const char *) value, eq+1)) + if (!eq || + (substring && strstr((const char *) value, eq+1)) || + (!substring && !strcmp((const char *) value, eq + 1))) match = 1; } xmlFree(value); @@ -1080,46 +1476,48 @@ static int ingest_to_cluster(struct client *cl, \param cl client holds the result set for record \param rec record buffer (0 terminated) \param record_no record position (1, 2, ..) + \param nmem working NMEM \retval 0 OK \retval -1 failure + \retval -2 Filtered */ int ingest_record(struct client *cl, const char *rec, int record_no, NMEM nmem) { - struct session_database *sdb = client_get_database(cl); struct session *se = client_get_session(cl); + int ret = 0; + struct session_database *sdb = client_get_database(cl); struct conf_service *service = se->service; - xmlDoc *xdoc = normalize_record(sdb, service, rec, nmem); + xmlDoc *xdoc = normalize_record(se, sdb, service, rec, nmem); xmlNode *root; const char *mergekey_norm; - int ret; - + if (!xdoc) return -1; - + root = xmlDocGetRootElement(xdoc); - + if (!check_record_filter(root, sdb)) { - yaz_log(YLOG_WARN, "Filtered out record no %d from %s", record_no, - sdb->database->url); + session_log(se, YLOG_LOG, "Filtered out record no %d from %s", + record_no, sdb->database->id); xmlFreeDoc(xdoc); - return -1; + return -2; } - + mergekey_norm = get_mergekey(xdoc, cl, record_no, service, nmem); if (!mergekey_norm) { - yaz_log(YLOG_WARN, "Got no mergekey"); + session_log(se, YLOG_WARN, "Got no mergekey"); xmlFreeDoc(xdoc); return -1; } session_enter(se); - ret = ingest_to_cluster(cl, xdoc, root, record_no, mergekey_norm); + if (client_get_session(cl) == se) + ingest_to_cluster(cl, xdoc, root, record_no, mergekey_norm); session_leave(se); - + xmlFreeDoc(xdoc); - return ret; } @@ -1132,29 +1530,18 @@ static int ingest_to_cluster(struct client *cl, xmlNode *n; xmlChar *type = 0; xmlChar *value = 0; - struct session_database *sdb = client_get_database(cl); struct session *se = client_get_session(cl); struct conf_service *service = se->service; + int term_factor = 1; + struct record_cluster *cluster; + struct session_database *sdb = client_get_database(cl); struct record *record = record_create(se->nmem, service->num_metadata, service->num_sortkeys, cl, record_no); - struct record_cluster *cluster = reclist_insert(se->reclist, - service, - record, - mergekey_norm, - &se->total_merged); - if (!cluster) - return -1; - if (global_parameters.dump_records) - yaz_log(YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid, - sdb->database->url, record_no); - relevance_newrec(se->relevance, cluster); - - // now parsing XML record and adding data to cluster or record metadata + for (n = root->children; n; n = n->next) { - pp2_relevance_token_t prt; if (type) xmlFree(type); if (value) @@ -1166,11 +1553,9 @@ static int ingest_to_cluster(struct client *cl, if (!strcmp((const char *) n->name, "metadata")) { struct conf_metadata *ser_md = 0; - struct conf_sortkey *ser_sk = 0; struct record_metadata **wheretoput = 0; struct record_metadata *rec_md = 0; int md_field_id = -1; - int sk_field_id = -1; type = xmlGetProp(n, (xmlChar *) "type"); value = xmlNodeListGetString(xdoc, n->children, 1); @@ -1184,33 +1569,95 @@ static int ingest_to_cluster(struct client *cl, { if (se->number_of_warnings_unknown_metadata == 0) { - yaz_log(YLOG_WARN, + session_log(se, YLOG_WARN, "Ignoring unknown metadata element: %s", type); } se->number_of_warnings_unknown_metadata++; continue; } - + ser_md = &service->metadata[md_field_id]; - - if (ser_md->sortkey_offset >= 0){ - sk_field_id = ser_md->sortkey_offset; - ser_sk = &service->sortkeys[sk_field_id]; - } // non-merged metadata rec_md = record_metadata_init(se->nmem, (const char *) value, ser_md->type, n->properties); if (!rec_md) { - yaz_log(YLOG_WARN, "bad metadata data '%s' for element '%s'", - value, type); + session_log(se, YLOG_WARN, "bad metadata data '%s' " + "for element '%s'", value, type); continue; } wheretoput = &record->metadata[md_field_id]; while (*wheretoput) wheretoput = &(*wheretoput)->next; *wheretoput = rec_md; + } + } + + cluster = reclist_insert(se->reclist, service, record, + mergekey_norm, &se->total_merged); + if (!cluster) + return -1; + + { + const char *use_term_factor_str = + session_setting_oneval(sdb, PZ_TERMLIST_TERM_FACTOR); + if (use_term_factor_str && use_term_factor_str[0] == '1') + { + int maxrecs = client_get_maxrecs(cl); + int hits = (int) client_get_hits(cl); + term_factor = MAX(hits, maxrecs) / MAX(1, maxrecs); + assert(term_factor >= 1); + yaz_log(YLOG_DEBUG, "Using term factor: %d (%d / %d)", term_factor, MAX(hits, maxrecs), MAX(1, maxrecs)); + } + } + + if (global_parameters.dump_records) + session_log(se, YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid, + sdb->database->id, record_no); + + + relevance_newrec(se->relevance, cluster); + + // now parsing XML record and adding data to cluster or record metadata + for (n = root->children; n; n = n->next) + { + pp2_charset_token_t prt; + if (type) + xmlFree(type); + if (value) + xmlFree(value); + type = value = 0; + + if (n->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) n->name, "metadata")) + { + struct conf_metadata *ser_md = 0; + struct conf_sortkey *ser_sk = 0; + struct record_metadata **wheretoput = 0; + struct record_metadata *rec_md = 0; + int md_field_id = -1; + int sk_field_id = -1; + + type = xmlGetProp(n, (xmlChar *) "type"); + value = xmlNodeListGetString(xdoc, n->children, 1); + + if (!type || !value || !*value) + continue; + + md_field_id + = conf_service_metadata_field_id(service, (const char *) type); + if (md_field_id < 0) + continue; + + ser_md = &service->metadata[md_field_id]; + + if (ser_md->sortkey_offset >= 0) + { + sk_field_id = ser_md->sortkey_offset; + ser_sk = &service->sortkeys[sk_field_id]; + } // merged metadata rec_md = record_metadata_init(se->nmem, (const char *) value, @@ -1221,16 +1668,15 @@ static int ingest_to_cluster(struct client *cl, // assign cluster or record based on merge action if (ser_md->merge == Metadata_merge_unique) { - struct record_metadata *mnode; - for (mnode = *wheretoput; mnode; mnode = mnode->next) - if (!strcmp((const char *) mnode->data.text.disp, + while (*wheretoput) + { + if (!strcmp((const char *) (*wheretoput)->data.text.disp, rec_md->data.text.disp)) break; - if (!mnode) - { - rec_md->next = *wheretoput; - *wheretoput = rec_md; + wheretoput = &(*wheretoput)->next; } + if (!*wheretoput) + *wheretoput = rec_md; } else if (ser_md->merge == Metadata_merge_longest) { @@ -1250,11 +1696,13 @@ static int ingest_to_cluster(struct client *cl, nmem_malloc(se->nmem, sizeof(union data_types)); - prt = pp2_relevance_tokenize( - service->sort_pct, - rec_md->data.text.disp, skip_article); + prt = + pp2_charset_token_create(service->charsets, "sort"); - pp2_relevance_token_next(prt); + pp2_charset_token_first(prt, rec_md->data.text.disp, + skip_article); + + pp2_charset_token_next(prt); sort_str = pp2_get_sort(prt); @@ -1263,24 +1711,19 @@ static int ingest_to_cluster(struct client *cl, if (!sort_str) { sort_str = rec_md->data.text.disp; - yaz_log(YLOG_WARN, + session_log(se, YLOG_WARN, "Could not make sortkey. Bug #1858"); } cluster->sortkeys[sk_field_id]->text.sort = nmem_strdup(se->nmem, sort_str); -#if 0 - yaz_log(YLOG_LOG, "text disp=%s", - cluster->sortkeys[sk_field_id]->text.disp); - yaz_log(YLOG_LOG, "text sort=%s", - cluster->sortkeys[sk_field_id]->text.sort); -#endif - pp2_relevance_token_destroy(prt); + pp2_charset_token_destroy(prt); } } } else if (ser_md->merge == Metadata_merge_all) { - rec_md->next = *wheretoput; + while (*wheretoput) + wheretoput = &(*wheretoput)->next; *wheretoput = rec_md; } else if (ser_md->merge == Metadata_merge_range) @@ -1310,22 +1753,23 @@ static int ingest_to_cluster(struct client *cl, (char *) value, ser_md->rank, ser_md->name); - // construct facets ... - if (ser_md->termlist) + // construct facets ... unless the client already has reported them + if (ser_md->termlist && !client_has_facet(cl, (char *) type)) { if (ser_md->type == Metadata_type_year) { char year[64]; sprintf(year, "%d", rec_md->data.number.max); - add_facet(se, (char *) type, year); + + add_facet(se, (char *) type, year, term_factor); if (rec_md->data.number.max != rec_md->data.number.min) { sprintf(year, "%d", rec_md->data.number.min); - add_facet(se, (char *) type, year); + add_facet(se, (char *) type, year, term_factor); } } else - add_facet(se, (char *) type, (char *) value); + add_facet(se, (char *) type, (char *) value, term_factor); } // cleaning up @@ -1336,7 +1780,7 @@ static int ingest_to_cluster(struct client *cl, else { if (se->number_of_warnings_unknown_elements == 0) - yaz_log(YLOG_WARN, + session_log(se, YLOG_WARN, "Unexpected element in internal record: %s", n->name); se->number_of_warnings_unknown_elements++; } @@ -1352,6 +1796,18 @@ static int ingest_to_cluster(struct client *cl, return 0; } +void session_log(struct session *s, int level, const char *fmt, ...) +{ + char buf[1024]; + va_list ap; + va_start(ap, fmt); + + yaz_vsnprintf(buf, sizeof(buf)-30, fmt, ap); + yaz_log(level, "Session (%u): %s", s->session_id, buf); + + va_end(ap); +} + /* * Local variables: * c-basic-offset: 4