X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Fsession.c;h=9e52d59f4492b128d9061973c747fbdf12e6fb6f;hb=7db45d2bc967511916800d163b2c3e290c62da40;hp=c5401cf24129431d152c3acc661a90df83b8d847;hpb=74c1fc846719bbfbfec81af3f967e56436f7d9c2;p=pazpar2-moved-to-github.git diff --git a/src/session.c b/src/session.c index c5401cf..9e52d59 100644 --- a/src/session.c +++ b/src/session.c @@ -1,5 +1,5 @@ /* This file is part of Pazpar2. - Copyright (C) 2006-2012 Index Data + Copyright (C) 2006-2013 Index Data Pazpar2 is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -56,9 +56,8 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #include #include -#include -#define USE_TIMING 0 +#define USE_TIMING 1 #if USE_TIMING #include #endif @@ -98,7 +97,6 @@ struct client_list { /* session counting (1) , disable client counting (0) */ static YAZ_MUTEX g_session_mutex = 0; static int no_sessions = 0; -static int no_session_total = 0; static int session_use(int delta) { @@ -107,11 +105,10 @@ static int session_use(int delta) yaz_mutex_create(&g_session_mutex); yaz_mutex_enter(g_session_mutex); no_sessions += delta; - if (delta > 0) - no_session_total += delta; sessions = no_sessions; yaz_mutex_leave(g_session_mutex); - yaz_log(YLOG_DEBUG, "%s sessions=%d", delta == 0 ? "" : (delta > 0 ? "INC" : "DEC"), no_sessions); + yaz_log(YLOG_DEBUG, "%s sessions=%d", delta == 0 ? "" : + (delta > 0 ? "INC" : "DEC"), no_sessions); return sessions; } @@ -120,17 +117,6 @@ int sessions_count(void) return session_use(0); } -int session_count_total(void) -{ - int total = 0; - if (!g_session_mutex) - return 0; - yaz_mutex_enter(g_session_mutex); - total = no_session_total; - yaz_mutex_leave(g_session_mutex); - return total; -} - static void log_xml_doc(xmlDoc *doc) { FILE *lf = yaz_log_file(); @@ -149,20 +135,23 @@ static void log_xml_doc(xmlDoc *doc) xmlFree(result); } -static void session_enter(struct session *s) +static void session_enter(struct session *s, const char *caller) { + if (caller) + session_log(s, YLOG_DEBUG, "Session lock by %s", caller); yaz_mutex_enter(s->session_mutex); } -static void session_leave(struct session *s) +static void session_leave(struct session *s, const char *caller) { yaz_mutex_leave(s->session_mutex); + if (caller) + session_log(s, YLOG_DEBUG, "Session unlock by %s", caller); } -static void session_normalize_facet(struct session *s, const char *type, - const char *value, - WRBUF display_wrbuf, - WRBUF facet_wrbuf) +static void session_normalize_facet(struct session *s, + const char *type, const char *value, + WRBUF display_wrbuf, WRBUF facet_wrbuf) { struct conf_service *service = s->service; pp2_charset_token_t prt; @@ -178,7 +167,8 @@ static void session_normalize_facet(struct session *s, const char *type, prt = pp2_charset_token_create(service->charsets, icu_chain_id); if (!prt) { - yaz_log(YLOG_FATAL, "Unknown ICU chain '%s' for facet of type '%s'", + session_log(s, YLOG_FATAL, + "Unknown ICU chain '%s' for facet of type '%s'", icu_chain_id, type); wrbuf_destroy(facet_wrbuf); wrbuf_destroy(display_wrbuf); @@ -253,8 +243,7 @@ static xmlDoc *record_to_xml(struct session *se, if (!rdoc) { - session_log(se, YLOG_FATAL, "Non-wellformed XML received from %s", - db->id); + session_log(se, YLOG_WARN, "Non-wellformed XML"); return 0; } @@ -308,7 +297,8 @@ static void insert_settings_parameters(struct session_database *sdb, // Add static values from session database settings if applicable static void insert_settings_values(struct session_database *sdb, xmlDoc *doc, - struct conf_service *service) + xmlNode *root, + struct conf_service *service) { int i; @@ -323,8 +313,7 @@ static void insert_settings_values(struct session_database *sdb, xmlDoc *doc, const char *val = session_setting_oneval(sdb, offset); if (val) { - xmlNode *r = xmlDocGetRootElement(doc); - xmlNode *n = xmlNewTextChild(r, 0, (xmlChar *) "metadata", + xmlNode *n = xmlNewTextChild(root, 0, (xmlChar *) "metadata", (xmlChar *) val); xmlSetProp(n, (xmlChar *) "type", (xmlChar *) md->name); } @@ -347,19 +336,7 @@ static xmlDoc *normalize_record(struct session *se, if (normalize_record_transform(sdb->map, &rdoc, (const char **)parms)) { - session_log(se, YLOG_WARN, "Normalize failed from %s", - sdb->database->id); - } - else - { - insert_settings_values(sdb, rdoc, service); - - if (global_parameters.dump_records) - { - session_log(se, YLOG_LOG, "Normalized record from %s", - sdb->database->id); - log_xml_doc(rdoc); - } + session_log(se, YLOG_WARN, "Normalize failed"); } } return rdoc; @@ -463,7 +440,7 @@ int session_set_watch(struct session *s, int what, struct http_channel *chan) { int ret; - session_enter(s); + session_enter(s, "session_set_watch"); if (s->watchlist[what].fun) ret = -1; else @@ -475,14 +452,14 @@ int session_set_watch(struct session *s, int what, session_watch_cancel); ret = 0; } - session_leave(s); + session_leave(s, "session_set_watch"); return ret; } void session_alert_watch(struct session *s, int what) { assert(s); - session_enter(s); + session_enter(s, "session_alert_watch"); if (s->watchlist[what].fun) { /* our watch is no longer associated with http_channel */ @@ -499,13 +476,13 @@ void session_alert_watch(struct session *s, int what) s->watchlist[what].data = 0; s->watchlist[what].obs = 0; - session_leave(s); + session_leave(s, "session_alert_watch"); session_log(s, YLOG_DEBUG, "Alert Watch: %d calling function: %p", what, fun); fun(data); } else - session_leave(s); + session_leave(s,"session_alert_watch"); } //callback for grep_databases @@ -545,10 +522,10 @@ static void session_reset_active_clients(struct session *se, { struct client_list *l; - session_enter(se); + session_enter(se, "session_reset_active_clients"); l = se->clients_active; se->clients_active = new_list; - session_leave(se); + session_leave(se, "session_reset_active_clients"); while (l) { @@ -569,10 +546,10 @@ static void session_remove_cached_clients(struct session *se) session_reset_active_clients(se, 0); - session_enter(se); + session_enter(se, "session_remove_cached_clients"); l = se->clients_cached; se->clients_cached = 0; - session_leave(se); + session_leave(se, "session_remove_cached_clients"); while (l) { @@ -619,11 +596,9 @@ int session_is_preferred_clients_ready(struct session *s) return res == 0; } -static void session_clear_set(struct session *se, - const char *sort_field, int increasing, int position) +static void session_clear_set(struct session *se, struct reclist_sortparms *sp) { reclist_destroy(se->reclist); - se->reclist = 0; if (nmem_total(se->nmem)) session_log(se, YLOG_DEBUG, "NMEN operation usage %zd", nmem_total(se->nmem)); @@ -631,57 +606,102 @@ static void session_clear_set(struct session *se, se->total_records = se->total_merged = 0; se->num_termlists = 0; + relevance_clear(se->relevance); + /* reset list of sorted results and clear to relevance search */ se->sorted_results = nmem_malloc(se->nmem, sizeof(*se->sorted_results)); - se->sorted_results->field = nmem_strdup(se->nmem, sort_field); - se->sorted_results->increasing = increasing; - se->sorted_results->position = position; + se->sorted_results->name = nmem_strdup(se->nmem, sp->name); + se->sorted_results->increasing = sp->increasing; + se->sorted_results->type = sp->type; se->sorted_results->next = 0; - session_log(se, YLOG_DEBUG, "clear_set session_sort: field=%s increasing=%d position=%d configured", - sort_field, increasing, position); + session_log(se, YLOG_DEBUG, "clear_set session_sort: field=%s increasing=%d type=%d configured", + sp->name, sp->increasing, sp->type); se->reclist = reclist_create(se->nmem); } -void session_sort(struct session *se, const char *field, int increasing, - int position) +static void session_sort_unlocked(struct session *se, + struct reclist_sortparms *sp, + const char *mergekey, + const char *rank) { - struct session_sorted_results *sr; struct client_list *l; + const char *field = sp->name; + int increasing = sp->increasing; + int type = sp->type; + int clients_research = 0; - session_enter(se); + session_log(se, YLOG_DEBUG, "session_sort field=%s increasing=%d type=%d", + field, increasing, type); - yaz_log(YLOG_LOG, "session_sort field=%s increasing=%d position=%d", field, increasing, position); - /* see if we already have sorted for this critieria */ - for (sr = se->sorted_results; sr; sr = sr->next) + if (rank && (!se->rank || strcmp(se->rank, rank))) { - if (!strcmp(field, sr->field) && increasing == sr->increasing && sr->position == position) - break; + /* new rank must research/reingest anyway */ + assert(rank); + xfree(se->rank); + se->rank = *rank ? xstrdup(rank) : 0; + clients_research = 1; + session_log(se, YLOG_DEBUG, "session_sort: new rank = %s", + rank); } - if (sr) + if (mergekey && (!se->mergekey || strcmp(se->mergekey, mergekey))) { - session_log(se, YLOG_DEBUG, "search_sort: field=%s increasing=%d position=%d already fetched", - field, increasing, position); - session_leave(se); - return; + /* new mergekey must research/reingest anyway */ + assert(mergekey); + xfree(se->mergekey); + se->mergekey = *mergekey ? xstrdup(mergekey) : 0; + clients_research = 1; + session_log(se, YLOG_DEBUG, "session_sort: new mergekey = %s", + mergekey); + } + if (clients_research == 0) + { + struct reclist_sortparms *sr; + for (sr = se->sorted_results; sr; sr = sr->next) + if (!reclist_sortparms_cmp(sr, sp)) + break; + if (sr) + { + session_log(se, YLOG_DEBUG, "session_sort: field=%s increasing=%d type=%d already fetched", + field, increasing, type); + return; + } } - session_log(se, YLOG_DEBUG, "search_sort: field=%s increasing=%d position=%d must fetch", - field, increasing, position); - if (position) + session_log(se, YLOG_DEBUG, "session_sort: field=%s increasing=%d type=%d must fetch", + field, increasing, type); + + // We need to reset reclist on every sort that changes the records, not just for position + // So if just one client requires new searching, we need to clear set. + // Ask each of the client if sorting requires re-search due to native sort + // If it does it will require us to + for (l = se->clients_active; l; l = l->next) { - yaz_log(YLOG_DEBUG, "Reset results due to position"); - session_clear_set(se, field, increasing, position); + struct client *cl = l->client; + // Assume no re-search is required. + client_parse_init(cl, 1); + clients_research += client_parse_sort(cl, sp); + } + if (clients_research) + { + session_log(se, YLOG_DEBUG, + "session_sort: reset results due to %d clients researching", + clients_research); + session_clear_set(se, sp); } - else { - sr = nmem_malloc(se->nmem, sizeof(*sr)); - sr->field = nmem_strdup(se->nmem, field); + else + { + // A new sorting based on same record set + struct reclist_sortparms *sr = nmem_malloc(se->nmem, sizeof(*sr)); + sr->name = nmem_strdup(se->nmem, field); sr->increasing = increasing; - sr->position = position; + sr->type = type; sr->next = se->sorted_results; se->sorted_results = sr; + session_log(se, YLOG_DEBUG, "session_sort: no research/ingesting done"); + return ; } - yaz_log(YLOG_DEBUG, "Restarting search for clients due to change in sort order"); + session_log(se, YLOG_DEBUG, "Re- search/ingesting for clients due to change in sort order"); for (l = se->clients_active; l; l = l->next) { @@ -689,13 +709,27 @@ void session_sort(struct session *se, const char *field, int increasing, if (client_get_state(cl) == Client_Connecting || client_get_state(cl) == Client_Idle || client_get_state(cl) == Client_Working) { - yaz_log(YLOG_DEBUG, "Client %s: Restarting search due to change in sort order", client_get_id(cl)); client_start_search(cl); } + else + { + session_log(se, YLOG_DEBUG, + "session_sort: %s: No re-start/ingest in show. " + "Wrong client state: %d", + client_get_id(cl), client_get_state(cl)); + } } - session_leave(se); } +void session_sort(struct session *se, struct reclist_sortparms *sp, + const char *mergekey, const char *rank) +{ + //session_enter(se, "session_sort"); + session_sort_unlocked(se, sp, mergekey, rank); + //session_leave(se, "session_sort"); +} + + enum pazpar2_error_code session_search(struct session *se, const char *query, const char *startrecs, @@ -703,88 +737,92 @@ enum pazpar2_error_code session_search(struct session *se, const char *filter, const char *limit, const char **addinfo, - struct reclist_sortparms *sp) + const char **addinfo2, + struct reclist_sortparms *sp, + const char *mergekey, + const char *rank) { int live_channels = 0; int no_working = 0; int no_failed_query = 0; int no_failed_limit = 0; struct client_list *l, *l0; - struct timeval tval; - facet_limits_t facet_limits; + + session_alert_watch(se, SESSION_WATCH_SHOW); + session_alert_watch(se, SESSION_WATCH_BYTARGET); + session_alert_watch(se, SESSION_WATCH_TERMLIST); + session_alert_watch(se, SESSION_WATCH_SHOW_PREF); session_log(se, YLOG_DEBUG, "Search"); *addinfo = 0; - if (se->settings_modified) + if (se->settings_modified) { session_remove_cached_clients(se); + } else session_reset_active_clients(se, 0); - session_enter(se); + session_enter(se, "session_search"); se->settings_modified = 0; - session_clear_set(se, sp->name, sp->increasing, sp->type == Metadata_sortkey_position); + + if (mergekey) + { + xfree(se->mergekey); + se->mergekey = *mergekey ? xstrdup(mergekey) : 0; + } + if (rank) + { + xfree(se->rank); + se->rank = *rank ? xstrdup(rank) : 0; + } + + session_clear_set(se, sp); relevance_destroy(&se->relevance); live_channels = select_targets(se, filter); if (!live_channels) { - session_leave(se); + session_leave(se, "session_search"); return PAZPAR2_NO_TARGETS; } - yaz_gettimeofday(&tval); - - tval.tv_sec += 5; - - facet_limits = facet_limits_create(limit); - if (!facet_limits) + facet_limits_destroy(se->facet_limits); + se->facet_limits = facet_limits_create(limit); + if (!se->facet_limits) { *addinfo = "limit"; - session_leave(se); + session_leave(se, "session_search"); return PAZPAR2_MALFORMED_PARAMETER_VALUE; } l0 = se->clients_active; se->clients_active = 0; - session_leave(se); + session_leave(se, "session_search"); for (l = l0; l; l = l->next) { int parse_ret; struct client *cl = l->client; - + client_parse_init(cl, 1); if (prepare_map(se, client_get_database(cl)) < 0) continue; - parse_ret = client_parse_query(cl, query, facet_limits, startrecs, - maxrecs, se->service->ccl_bibset); + parse_ret = client_parse_query(cl, query, se->facet_limits, addinfo2); if (parse_ret == -1) no_failed_query++; else if (parse_ret == -2) no_failed_limit++; + else if (parse_ret < 0) + no_working++; /* other error, such as bad CCL map */ else { - int r = - client_prep_connection(cl, se->service->z3950_operation_timeout, - se->service->z3950_session_timeout, - se->service->server->iochan_man, - &tval); - if (parse_ret == 1 && r == 2) - { - session_log(se, YLOG_LOG, "client %s REUSE result", client_get_id(cl)); - client_reingest(cl); - } - else if (r) - { - session_log(se, YLOG_LOG, "client %s NEW search", client_get_id(cl)); - client_start_search(cl); - } + client_parse_range(cl, startrecs, maxrecs); + client_parse_sort(cl, sp); + client_start_search(cl); no_working++; } } - facet_limits_destroy(facet_limits); session_reset_active_clients(se, l0); if (no_working == 0) @@ -802,7 +840,6 @@ enum pazpar2_error_code session_search(struct session *se, else return PAZPAR2_NO_TARGETS; } - session_log(se, YLOG_LOG, "session_start_search done"); return PAZPAR2_NO_ERROR; } @@ -903,7 +940,7 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, void session_destroy(struct session *se) { struct session_database *sdb; - session_log(se, YLOG_DEBUG, "Destroying"); + session_log(se, YLOG_LOG, "destroy"); session_use(-1); session_remove_cached_clients(se); @@ -912,10 +949,13 @@ void session_destroy(struct session *se) normalize_cache_destroy(se->normalize_cache); relevance_destroy(&se->relevance); reclist_destroy(se->reclist); + xfree(se->mergekey); + xfree(se->rank); if (nmem_total(se->nmem)) session_log(se, YLOG_DEBUG, "NMEN operation usage %zd", nmem_total(se->nmem)); if (nmem_total(se->session_nmem)) session_log(se, YLOG_DEBUG, "NMEN session usage %zd", nmem_total(se->session_nmem)); + facet_limits_destroy(se->facet_limits); nmem_destroy(se->nmem); service_destroy(se->service); yaz_mutex_destroy(&se->session_mutex); @@ -925,9 +965,9 @@ size_t session_get_memory_status(struct session *session) { size_t session_nmem; if (session == 0) return 0; - session_enter(session); + session_enter(session, "session_get_memory_status"); session_nmem = nmem_total(session->nmem); - session_leave(session); + session_leave(session, "session_get_memory_status"); return session_nmem; } @@ -950,13 +990,18 @@ struct session *new_session(NMEM nmem, struct conf_service *service, session->number_of_warnings_unknown_elements = 0; session->number_of_warnings_unknown_metadata = 0; session->num_termlists = 0; - session->reclist = 0; + session->reclist = reclist_create(nmem); session->clients_active = 0; session->clients_cached = 0; session->settings_modified = 0; session->session_nmem = nmem; session->nmem = nmem_create(); session->databases = 0; + session->sorted_results = 0; + session->facet_limits = 0; + session->mergekey = 0; + session->rank = 0; + for (i = 0; i <= SESSION_WATCH_MAX; i++) { session->watchlist[i].data = 0; @@ -965,6 +1010,8 @@ struct session *new_session(NMEM nmem, struct conf_service *service, session->normalize_cache = normalize_cache_create(); session->session_mutex = 0; pazpar2_mutex_create(&session->session_mutex, tmp_str); + session_log(session, YLOG_LOG, "create"); + session_use(1); return session; } @@ -997,7 +1044,8 @@ static struct hitsbytarget *hitsbytarget_nb(struct session *se, res[*count].records = client_get_num_records(cl); res[*count].filtered = client_get_num_records_filtered(cl); res[*count].diagnostic = - client_get_diagnostic(cl, &res[*count].addinfo); + client_get_diagnostic(cl, &res[*count].message, + &res[*count].addinfo); res[*count].state = client_get_state_str(cl); res[*count].connected = client_get_connection(cl) ? 1 : 0; session_settings_dump(se, client_get_database(cl), w); @@ -1014,9 +1062,9 @@ static struct hitsbytarget *hitsbytarget_nb(struct session *se, struct hitsbytarget *get_hitsbytarget(struct session *se, int *count, NMEM nmem) { struct hitsbytarget *p; - session_enter(se); + session_enter(se, "get_hitsbytarget"); p = hitsbytarget_nb(se, count, nmem); - session_leave(se); + session_leave(se, "get_hitsbytarget"); return p; } @@ -1099,7 +1147,7 @@ void perform_termlist(struct http_channel *c, struct session *se, nmem_strsplit(nmem_tmp, ",", name, &names, &num_names); - session_enter(se); + session_enter(se, "perform_termlist"); for (j = 0; j < num_names; j++) { @@ -1162,7 +1210,7 @@ void perform_termlist(struct http_channel *c, struct session *se, wrbuf_puts(c->wrbuf, "\"/>\n"); } } - session_leave(se); + session_leave(se, "perform_termlist"); nmem_destroy(nmem_tmp); } @@ -1185,107 +1233,170 @@ struct record_cluster *show_single_start(struct session *se, const char *id, { struct record_cluster *r = 0; - session_enter(se); + session_enter(se, "show_single_start"); *prev_r = 0; *next_r = 0; - if (se->reclist) + reclist_limit(se->reclist, se, 1); + + reclist_enter(se->reclist); + while ((r = reclist_read_record(se->reclist))) { - reclist_enter(se->reclist); - while ((r = reclist_read_record(se->reclist))) + if (!strcmp(r->recid, id)) { - if (!strcmp(r->recid, id)) - { - *next_r = reclist_read_record(se->reclist); - break; - } - *prev_r = r; + *next_r = reclist_read_record(se->reclist); + break; } - reclist_leave(se->reclist); + *prev_r = r; } + reclist_leave(se->reclist); if (!r) - session_leave(se); + session_leave(se, "show_single_start"); return r; } void show_single_stop(struct session *se, struct record_cluster *rec) { - session_leave(se); + session_leave(se, "show_single_stop"); +} + + +int session_fetch_more(struct session *se) +{ + struct client_list *l; + int ret = 0; + + for (l = se->clients_active; l; l = l->next) + { + struct client *cl = l->client; + if (client_get_state(cl) == Client_Idle) + { + if (client_fetch_more(cl)) + { + session_log(se, YLOG_LOG, "%s: more to fetch", + client_get_id(cl)); + ret = 1; + } + else + { + session_log(se, YLOG_LOG, "%s: no more to fetch", + client_get_id(cl)); + } + } + else + { + session_log(se, YLOG_LOG, "%s: no fetch due to state=%s", + client_get_id(cl), client_get_state_str(cl)); + } + + } + return ret; } struct record_cluster **show_range_start(struct session *se, struct reclist_sortparms *sp, - int start, int *num, int *total, Odr_int *sumhits, Odr_int *approx_hits) + int start, int *num, int *total, + Odr_int *sumhits, Odr_int *approx_hits, + void (*show_records_ready)(void *data), + struct http_channel *chan) { - struct record_cluster **recs; + struct record_cluster **recs = 0; struct reclist_sortparms *spp; + struct client_list *l; int i; #if USE_TIMING yaz_timing_t t = yaz_timing_create(); #endif - session_enter(se); - recs = nmem_malloc(se->nmem, *num * sizeof(struct record_cluster *)); - if (!se->relevance) + session_enter(se, "show_range_start"); + *sumhits = 0; + *approx_hits = 0; + *total = 0; + reclist_limit(se->reclist, se, 0); + if (se->relevance) { - *num = 0; - *total = 0; - *sumhits = 0; - *approx_hits = 0; - recs = 0; - } - else - { - struct client_list *l; - for (spp = sp; spp; spp = spp->next) if (spp->type == Metadata_sortkey_relevance) { relevance_prepare_read(se->relevance, se->reclist); break; } - reclist_sort(se->reclist, sp); - - reclist_enter(se->reclist); - *total = reclist_get_num_records(se->reclist); - - *sumhits = 0; - *approx_hits = 0; for (l = se->clients_active; l; l = l->next) { *sumhits += client_get_hits(l->client); *approx_hits += client_get_approximation(l->client); } - for (i = 0; i < start; i++) - if (!reclist_read_record(se->reclist)) - { - *num = 0; - recs = 0; - break; - } + } + reclist_sort(se->reclist, sp); + + reclist_enter(se->reclist); + *total = reclist_get_num_records(se->reclist); - for (i = 0; i < *num; i++) + for (l = se->clients_active; l; l = l->next) + client_update_show_stat(l->client, 0); + + for (i = 0; i < start; i++) + { + struct record_cluster *r = reclist_read_record(se->reclist); + if (!r) { - struct record_cluster *r = reclist_read_record(se->reclist); - if (!r) - { - *num = i; - break; - } + *num = 0; + break; + } + else + { + struct record *rec = r->records; + for (;rec; rec = rec->next) + client_update_show_stat(rec->client, 1); + } + } + recs = nmem_malloc(se->nmem, (*num > 0 ? *num : 1) * sizeof(*recs)); + for (i = 0; i < *num; i++) + { + struct record_cluster *r = reclist_read_record(se->reclist); + if (!r) + { + *num = i; + break; + } + else + { + struct record *rec = r->records; + for (;rec; rec = rec->next) + client_update_show_stat(rec->client, 1); recs[i] = r; } - reclist_leave(se->reclist); } + reclist_leave(se->reclist); #if USE_TIMING yaz_timing_stop(t); - yaz_log(YLOG_LOG, "show %6.5f %3.2f %3.2f", + session_log(se, YLOG_LOG, "show %6.5f %3.2f %3.2f", yaz_timing_get_real(t), yaz_timing_get_user(t), yaz_timing_get_sys(t)); yaz_timing_destroy(&t); #endif + + if (!session_fetch_more(se)) + session_log(se, YLOG_LOG, "can not fetch more"); + else + { + show_range_stop(se, recs); + session_log(se, YLOG_LOG, "fetching more in progress"); + if (session_set_watch(se, SESSION_WATCH_SHOW, + show_records_ready, chan, chan)) + { + session_log(se, YLOG_WARN, "Ignoring show block"); + session_enter(se, "show_range_start"); + } + else + { + session_log(se, YLOG_LOG, "session watch OK"); + return 0; + } + } return recs; } void show_range_stop(struct session *se, struct record_cluster **recs) { - session_leave(se); + session_leave(se, "show_range_stop"); } void statistics(struct session *se, struct statistics *stat) @@ -1328,10 +1439,11 @@ static struct record_metadata *record_metadata_init( { if (attr->children && attr->children->content) { - if (strcmp((const char *) attr->name, "type")) - { /* skip the "type" attribute.. Its value is already part of - the element in output (md-%s) and so repeating it here - is redundant */ + if (strcmp((const char *) attr->name, "type") + && strcmp((const char *) attr->name, "empty")) + { /* skip the "type" + "empty" attribute.. + The "Type" is already part of the element in output + (md-%s) and so repeating it here is redundant */ *attrp = nmem_malloc(nmem, sizeof(**attrp)); (*attrp)->name = nmem_strdup(nmem, (const char *) attr->name); @@ -1351,6 +1463,7 @@ static struct record_metadata *record_metadata_init( rec_md->data.text.disp = p; rec_md->data.text.sort = 0; + rec_md->data.text.snippet = 0; } else if (type == Metadata_type_year || type == Metadata_type_date) { @@ -1370,6 +1483,26 @@ static struct record_metadata *record_metadata_init( return rec_md; } +static void mergekey_norm_wr(pp2_charset_fact_t charsets, + WRBUF norm_wr, const char *value) +{ + const char *norm_str; + pp2_charset_token_t prt = + pp2_charset_token_create(charsets, "mergekey"); + + pp2_charset_token_first(prt, value, 0); + while ((norm_str = pp2_charset_token_next(prt))) + { + if (*norm_str) + { + if (wrbuf_len(norm_wr)) + wrbuf_puts(norm_wr, " "); + wrbuf_puts(norm_wr, norm_str); + } + } + pp2_charset_token_destroy(prt); +} + static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, struct conf_service *service, WRBUF norm_wr) { @@ -1388,29 +1521,17 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, else if (!strcmp(name, (const char *) type)) { xmlChar *value = xmlNodeListGetString(doc, n->children, 1); - if (value) + if (value && *value) { - const char *norm_str; - pp2_charset_token_t prt = - pp2_charset_token_create(service->charsets, "mergekey"); - - pp2_charset_token_first(prt, (const char *) value, 0); if (wrbuf_len(norm_wr) > 0) wrbuf_puts(norm_wr, " "); wrbuf_puts(norm_wr, name); - while ((norm_str = - pp2_charset_token_next(prt))) - { - if (*norm_str) - { - wrbuf_puts(norm_wr, " "); - wrbuf_puts(norm_wr, norm_str); - } - } - xmlFree(value); - pp2_charset_token_destroy(prt); + mergekey_norm_wr(service->charsets, norm_wr, + (const char *) value); no_found++; } + if (value) + xmlFree(value); } xmlFree(type); } @@ -1418,32 +1539,28 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, return no_found; } -static const char *get_mergekey(xmlDoc *doc, struct client *cl, int record_no, - struct conf_service *service, NMEM nmem) +static const char *get_mergekey(xmlDoc *doc, xmlNode *root, + struct client *cl, int record_no, + struct conf_service *service, NMEM nmem, + const char *session_mergekey) { char *mergekey_norm = 0; - xmlNode *root = xmlDocGetRootElement(doc); WRBUF norm_wr = wrbuf_alloc(); + xmlChar *mergekey; - /* consider mergekey from XSL first */ - xmlChar *mergekey = xmlGetProp(root, (xmlChar *) "mergekey"); - if (mergekey) + if (session_mergekey) { - const char *norm_str; - pp2_charset_token_t prt = - pp2_charset_token_create(service->charsets, "mergekey"); + int i, num = 0; + char **values = 0; + nmem_strsplit_escape2(nmem, ",", session_mergekey, &values, + &num, 1, '\\', 1); - pp2_charset_token_first(prt, (const char *) mergekey, 0); - while ((norm_str = pp2_charset_token_next(prt))) - { - if (*norm_str) - { - if (wrbuf_len(norm_wr)) - wrbuf_puts(norm_wr, " "); - wrbuf_puts(norm_wr, norm_str); - } - } - pp2_charset_token_destroy(prt); + for (i = 0; i < num; i++) + get_mergekey_from_doc(doc, root, values[i], service, norm_wr); + } + else if ((mergekey = xmlGetProp(root, (xmlChar *) "mergekey"))) + { + mergekey_norm_wr(service->charsets, norm_wr, (const char *) mergekey); xmlFree(mergekey); } else @@ -1546,12 +1663,37 @@ static int check_record_filter(xmlNode *root, struct session_database *sdb) return match; } - static int ingest_to_cluster(struct client *cl, xmlDoc *xdoc, xmlNode *root, int record_no, - const char *mergekey_norm); + struct record_metadata_attr *mergekey); + +static int ingest_sub_record(struct client *cl, xmlDoc *xdoc, xmlNode *root, + int record_no, NMEM nmem, + struct session_database *sdb, + struct record_metadata_attr *mergekeys) +{ + int ret = 0; + struct session *se = client_get_session(cl); + struct conf_service *service = se->service; + + insert_settings_values(sdb, xdoc, root, service); + + if (!check_record_filter(root, sdb)) + { + session_log(se, YLOG_LOG, + "Filtered out record no %d from %s", + record_no, sdb->database->id); + return 0; + } + session_enter(se, "ingest_sub_record"); + if (client_get_session(cl) == se && se->relevance) + ret = ingest_to_cluster(cl, xdoc, root, record_no, mergekeys); + session_leave(se, "ingest_sub_record"); + + return ret; +} /** \brief ingest XML record \param cl client holds the result set for record @@ -1566,38 +1708,168 @@ int ingest_record(struct client *cl, const char *rec, int record_no, NMEM nmem) { struct session *se = client_get_session(cl); - int ret = 0; struct session_database *sdb = client_get_database(cl); struct conf_service *service = se->service; xmlDoc *xdoc = normalize_record(se, sdb, service, rec, nmem); + int r = 0; xmlNode *root; - const char *mergekey_norm; if (!xdoc) return -1; + if (global_parameters.dump_records) + { + session_log(se, YLOG_LOG, "Normalized record from %s", + sdb->database->id); + log_xml_doc(xdoc); + } + root = xmlDocGetRootElement(xdoc); - if (!check_record_filter(root, sdb)) + if (!strcmp((const char *) root->name, "cluster")) { - session_log(se, YLOG_LOG, "Filtered out record no %d from %s", record_no, sdb->database->id); - xmlFreeDoc(xdoc); - return -2; + xmlNode *sroot; + for (sroot = root->children; sroot; sroot = sroot->next) + if (sroot->type == XML_ELEMENT_NODE) + { + const char *mergekey_norm = + get_mergekey(xdoc, sroot, cl, record_no, service, nmem, + se->mergekey); + + struct record_metadata_attr *mk = (struct record_metadata_attr*) + nmem_malloc(nmem, sizeof(*mk)); + mk->name = 0; + mk->value = nmem_strdup(nmem, mergekey_norm); + mk->next = 0; + + r = ingest_sub_record(cl, xdoc, sroot, record_no, nmem, sdb, + mk); + if (r) + break; + } } + else if (!strcmp((const char *) root->name, "record")) + { + const char *mergekey_norm = + get_mergekey(xdoc, root, cl, record_no, service, nmem, + se->mergekey); + if (mergekey_norm) + { + struct record_metadata_attr *mk = (struct record_metadata_attr*) + nmem_malloc(nmem, sizeof(*mk)); + mk->name = 0; + mk->value = nmem_strdup(nmem, mergekey_norm); + mk->next = 0; - mergekey_norm = get_mergekey(xdoc, cl, record_no, service, nmem); - if (!mergekey_norm) + r = ingest_sub_record(cl, xdoc, root, record_no, nmem, sdb, mk); + } + } + else { - session_log(se, YLOG_WARN, "Got no mergekey"); - xmlFreeDoc(xdoc); - return -1; + session_log(se, YLOG_WARN, "Bad pz root element: %s", + (const char *) root->name); + r = -1; } - session_enter(se); - if (client_get_session(cl) == se) - ret = ingest_to_cluster(cl, xdoc, root, record_no, mergekey_norm); - session_leave(se); - xmlFreeDoc(xdoc); + return r; +} + + +// struct conf_metadata *ser_md = &service->metadata[md_field_id]; +// struct record_metadata *rec_md = record->metadata[md_field_id]; +static int match_metadata_local(struct conf_service *service, + struct conf_metadata *ser_md, + struct record_metadata *rec_md0, + char **values, int num_v) +{ + int i; + struct record_metadata *rec_md = rec_md0; + WRBUF val_wr = 0; + WRBUF text_wr = wrbuf_alloc(); + for (i = 0; i < num_v; ) + { + if (rec_md) + { + if (ser_md->type == Metadata_type_year + || ser_md->type == Metadata_type_date) + { + int y = atoi(values[i]); + if (y >= rec_md->data.number.min + && y <= rec_md->data.number.max) + break; + } + else + { + if (!val_wr) + { + val_wr = wrbuf_alloc(); + mergekey_norm_wr(service->charsets, val_wr, values[i]); + } + wrbuf_rewind(text_wr); + mergekey_norm_wr(service->charsets, text_wr, + rec_md->data.text.disp); + if (!strcmp(wrbuf_cstr(val_wr), wrbuf_cstr(text_wr))) + break; + } + rec_md = rec_md->next; + } + else + { + rec_md = rec_md0; + wrbuf_destroy(val_wr); + val_wr = 0; + i++; + } + } + wrbuf_destroy(val_wr); + wrbuf_destroy(text_wr); + return i < num_v ? 1 : 0; +} + +int session_check_cluster_limit(struct session *se, struct record_cluster *rec) +{ + int i; + struct conf_service *service = se->service; + int ret = 1; + const char *name; + const char *value; + NMEM nmem_tmp = nmem_create(); + + for (i = 0; (name = facet_limits_get(se->facet_limits, i, &value)); i++) + { + int j; + for (j = 0; j < service->num_metadata; j++) + { + struct conf_metadata *md = service->metadata + j; + if (!strcmp(md->name, name) && md->limitcluster) + { + char **values = 0; + int num = 0; + int md_field_id = + conf_service_metadata_field_id(service, + md->limitcluster); + + if (md_field_id < 0) + { + ret = 0; + break; + } + + nmem_strsplit_escape2(nmem_tmp, "|", value, &values, + &num, 1, '\\', 1); + + if (!match_metadata_local(service, + &service->metadata[md_field_id], + rec->metadata[md_field_id], + values, num)) + { + ret = 0; + break; + } + } + } + } + nmem_destroy(nmem_tmp); return ret; } @@ -1614,60 +1886,47 @@ static int check_limit_local(struct client *cl, int l = 0; while (!skip_record) { - struct conf_metadata *ser_md = 0; - struct record_metadata *rec_md = 0; int md_field_id; char **values = 0; - int i, num_v = 0; - - const char *name = client_get_facet_limit_local(cl, sdb, &l, nmem_tmp, &num_v, &values); + int num_v = 0; + const char *name = + client_get_facet_limit_local(cl, sdb, &l, nmem_tmp, + &num_v, &values); if (!name) break; - md_field_id = conf_service_metadata_field_id(service, name); - if (md_field_id < 0) + if (!strcmp(name, "*")) { - skip_record = 1; - break; + for (md_field_id = 0; md_field_id < service->num_metadata; + md_field_id++) + { + if (match_metadata_local( + service, + &service->metadata[md_field_id], + record->metadata[md_field_id], + values, num_v)) + break; + } + if (md_field_id == service->num_metadata) + skip_record = 1; } - ser_md = &service->metadata[md_field_id]; - rec_md = record->metadata[md_field_id]; - yaz_log(YLOG_DEBUG, "check limit local %s", name); - for (i = 0; i < num_v; ) + else { - if (rec_md) + md_field_id = conf_service_metadata_field_id(service, name); + if (md_field_id < 0) { - if (ser_md->type == Metadata_type_year - || ser_md->type == Metadata_type_date) - { - int y = atoi(values[i]); - if (y >= rec_md->data.number.min - && y <= rec_md->data.number.max) - break; - } - else - { - yaz_log(YLOG_DEBUG, "cmp: '%s' '%s'", rec_md->data.text.disp, values[i]); - if (!strcmp(rec_md->data.text.disp, values[i])) - { - // Value equals, should not be filtered. - break; - } - } - rec_md = rec_md->next; + skip_record = 1; + break; } - else + if (!match_metadata_local( + service, + &service->metadata[md_field_id], + record->metadata[md_field_id], + values, num_v)) { - rec_md = record->metadata[md_field_id]; - i++; + skip_record = 1; } } - // At end , not match - if (i == num_v) - { - skip_record = 1; - break; - } } nmem_destroy(nmem_tmp); return skip_record; @@ -1677,7 +1936,7 @@ static int ingest_to_cluster(struct client *cl, xmlDoc *xdoc, xmlNode *root, int record_no, - const char *mergekey_norm) + struct record_metadata_attr *merge_keys) { xmlNode *n; xmlChar *type = 0; @@ -1686,7 +1945,11 @@ static int ingest_to_cluster(struct client *cl, struct conf_service *service = se->service; int term_factor = 1; struct record_cluster *cluster; + struct record_metadata **metadata0; struct session_database *sdb = client_get_database(cl); + NMEM ingest_nmem = 0; + char **rank_values = 0; + int rank_num = 0; struct record *record = record_create(se->nmem, service->num_metadata, service->num_sortkeys, cl, @@ -1711,10 +1974,17 @@ static int ingest_to_cluster(struct client *cl, type = xmlGetProp(n, (xmlChar *) "type"); value = xmlNodeListGetString(xdoc, n->children, 1); - - if (!type || !value || !*value) + if (!type) continue; - + if (!value || !*value) + { + xmlChar *empty = xmlGetProp(n, (xmlChar *) "empty"); + if (!empty) + continue; + if (value) + xmlFree(value); + value = empty; + } md_field_id = conf_service_metadata_field_id(service, (const char *) type); if (md_field_id < 0) @@ -1739,6 +2009,18 @@ static int ingest_to_cluster(struct client *cl, "for element '%s'", value, type); continue; } + + if (ser_md->type == Metadata_type_generic) + { + WRBUF w = wrbuf_alloc(); + if (relevance_snippet(se->relevance, + (char*) value, ser_md->name, w)) + rec_md->data.text.snippet = nmem_strdup(se->nmem, + wrbuf_cstr(w)); + wrbuf_destroy(w); + } + + wheretoput = &record->metadata[md_field_id]; while (*wheretoput) wheretoput = &(*wheretoput)->next; @@ -1757,9 +2039,9 @@ static int ingest_to_cluster(struct client *cl, return -2; } cluster = reclist_insert(se->reclist, service, record, - mergekey_norm, &se->total_merged); + merge_keys, &se->total_merged); if (!cluster) - return -1; + return 0; // complete match with existing record { const char *use_term_factor_str = @@ -1770,7 +2052,8 @@ static int ingest_to_cluster(struct client *cl, int hits = (int) client_get_hits(cl); term_factor = MAX(hits, maxrecs) / MAX(1, maxrecs); assert(term_factor >= 1); - yaz_log(YLOG_DEBUG, "Using term factor: %d (%d / %d)", term_factor, MAX(hits, maxrecs), MAX(1, maxrecs)); + session_log(se, YLOG_DEBUG, "Using term factor: %d (%d / %d)", + term_factor, MAX(hits, maxrecs), MAX(1, maxrecs)); } } @@ -1781,6 +2064,19 @@ static int ingest_to_cluster(struct client *cl, relevance_newrec(se->relevance, cluster); + // original metadata, to check if first existence of a field + metadata0 = xmalloc(sizeof(*metadata0) * service->num_metadata); + memcpy(metadata0, cluster->metadata, + sizeof(*metadata0) * service->num_metadata); + + ingest_nmem = nmem_create(); + if (se->rank) + { + yaz_log(YLOG_LOG, "local in sort : %s", se->rank); + nmem_strsplit_escape2(ingest_nmem, ",", se->rank, &rank_values, + &rank_num, 1, '\\', 1); + } + // now parsing XML record and adding data to cluster or record metadata for (n = root->children; n; n = n->next) { @@ -1801,8 +2097,8 @@ static int ingest_to_cluster(struct client *cl, struct record_metadata *rec_md = 0; int md_field_id = -1; int sk_field_id = -1; - const char *rank; - xmlChar *xml_rank; + const char *rank = 0; + xmlChar *xml_rank = 0; type = xmlGetProp(n, (xmlChar *) "type"); value = xmlNodeListGetString(xdoc, n->children, 1); @@ -1826,17 +2122,47 @@ static int ingest_to_cluster(struct client *cl, // merged metadata rec_md = record_metadata_init(se->nmem, (const char *) value, ser_md->type, 0); + + // see if the field was not in cluster already (from beginning) + if (!rec_md) continue; - xml_rank = xmlGetProp(n, (xmlChar *) "rank"); - rank = xml_rank ? (const char *) xml_rank : ser_md->rank; + if (rank_num) + { + int i; + for (i = 0; i < rank_num; i++) + { + const char *val = rank_values[i]; + const char *cp = strchr(val, '='); + if (!cp) + continue; + if ((cp - val) == strlen((const char *) type) + && !memcmp(val, type, cp - val)) + { + rank = cp + 1; + break; + } + } + } + else + { + xml_rank = xmlGetProp(n, (xmlChar *) "rank"); + rank = xml_rank ? (const char *) xml_rank : ser_md->rank; + } wheretoput = &cluster->metadata[md_field_id]; - // and polulate with data: - // assign cluster or record based on merge action - if (ser_md->merge == Metadata_merge_unique) + if (ser_md->merge == Metadata_merge_first) + { + if (!metadata0[md_field_id]) + { + while (*wheretoput) + wheretoput = &(*wheretoput)->next; + *wheretoput = rec_md; + } + } + else if (ser_md->merge == Metadata_merge_unique) { while (*wheretoput) { @@ -1922,7 +2248,6 @@ static int ingest_to_cluster(struct client *cl, relevance_countwords(se->relevance, cluster, (char *) value, rank, ser_md->name); } - // construct facets ... unless the client already has reported them if (ser_md->termlist && !client_has_facet(cl, (char *) type)) { @@ -1962,6 +2287,8 @@ static int ingest_to_cluster(struct client *cl, if (value) xmlFree(value); + nmem_destroy(ingest_nmem); + xfree(metadata0); relevance_donerecord(se->relevance, cluster); se->total_records++; @@ -1975,7 +2302,7 @@ void session_log(struct session *s, int level, const char *fmt, ...) va_start(ap, fmt); yaz_vsnprintf(buf, sizeof(buf)-30, fmt, ap); - yaz_log(level, "Session %u: %s", s->session_id, buf); + yaz_log(level, "Session %u: %s", s ? s->session_id : 0, buf); va_end(ap); }