X-Git-Url: http://git.indexdata.com/?p=pazpar2-moved-to-github.git;a=blobdiff_plain;f=src%2Fsession.c;h=f597d7c0ed7bdc169e272949f2ecc6a7cd25885a;hp=e9784256a1be46fa5243edda8de9228569ae755e;hb=HEAD;hpb=84aba98c6b20e5363e47249d40e928eec42ccd72 diff --git a/src/session.c b/src/session.c index e978425..f597d7c 100644 --- a/src/session.c +++ b/src/session.c @@ -56,6 +56,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #include #include +#include #define USE_TIMING 0 #if USE_TIMING @@ -112,7 +113,7 @@ static int session_use(int delta) return sessions; } -int sessions_count(void) +int sessions_get_count(void) { return session_use(0); } @@ -149,30 +150,19 @@ static void session_leave(struct session *s, const char *caller) session_log(s, YLOG_DEBUG, "Session unlock by %s", caller); } -static void session_normalize_facet(struct session *s, - const char *type, const char *value, - WRBUF display_wrbuf, WRBUF facet_wrbuf) +static int run_icu(struct session *s, const char *icu_chain_id, + const char *value, + WRBUF norm_wr, WRBUF disp_wr) { - struct conf_service *service = s->service; - pp2_charset_token_t prt; const char *facet_component; - int i; - const char *icu_chain_id = 0; - - for (i = 0; i < service->num_metadata; i++) - if (!strcmp((service->metadata + i)->name, type)) - icu_chain_id = (service->metadata + i)->facetrule; - if (!icu_chain_id) - icu_chain_id = "facet"; - prt = pp2_charset_token_create(service->charsets, icu_chain_id); + struct conf_service *service = s->service; + pp2_charset_token_t prt = + pp2_charset_token_create(service->charsets, icu_chain_id); if (!prt) { session_log(s, YLOG_FATAL, - "Unknown ICU chain '%s' for facet of type '%s'", - icu_chain_id, type); - wrbuf_destroy(facet_wrbuf); - wrbuf_destroy(display_wrbuf); - return; + "Unknown ICU chain '%s'", icu_chain_id); + return 0; } pp2_charset_token_first(prt, value, 0); while ((facet_component = pp2_charset_token_next(prt))) @@ -180,54 +170,126 @@ static void session_normalize_facet(struct session *s, const char *display_component; if (*facet_component) { - if (wrbuf_len(facet_wrbuf)) - wrbuf_puts(facet_wrbuf, " "); - wrbuf_puts(facet_wrbuf, facet_component); + if (wrbuf_len(norm_wr)) + wrbuf_puts(norm_wr, " "); + wrbuf_puts(norm_wr, facet_component); } display_component = pp2_get_display(prt); if (display_component) { - if (wrbuf_len(display_wrbuf)) - wrbuf_puts(display_wrbuf, " "); - wrbuf_puts(display_wrbuf, display_component); + if (wrbuf_len(disp_wr)) + wrbuf_puts(disp_wr, " "); + wrbuf_puts(disp_wr, display_component); } } pp2_charset_token_destroy(prt); + return 1; } -void add_facet(struct session *s, const char *type, const char *value, int count) +static void session_normalize_facet(struct session *s, + const char *type, const char *value, + WRBUF display_wrbuf, WRBUF facet_wrbuf) { - WRBUF facet_wrbuf = wrbuf_alloc(); - WRBUF display_wrbuf = wrbuf_alloc(); + struct conf_service *service = s->service; + int i; + const char *icu_chain_id = 0; - session_normalize_facet(s, type, value, display_wrbuf, facet_wrbuf); + for (i = 0; i < service->num_metadata; i++) + if (!strcmp((service->metadata + i)->name, type)) + icu_chain_id = (service->metadata + i)->facetrule; + if (!icu_chain_id) + icu_chain_id = "facet"; - if (wrbuf_len(facet_wrbuf)) + run_icu(s, icu_chain_id, value, facet_wrbuf, display_wrbuf); +} + +struct facet_id { + char *client_id; + char *type; + char *id; + char *term; + struct facet_id *next; +}; + +static void session_add_id_facet(struct session *s, struct client *cl, + const char *type, + const char *id, + size_t id_len, + const char *term) +{ + struct facet_id *t = nmem_malloc(s->session_nmem, sizeof(*t)); + + t->client_id = nmem_strdup(s->session_nmem, client_get_id(cl)); + t->type = nmem_strdup(s->session_nmem, type); + t->id = nmem_strdupn(s->session_nmem, id, id_len); + t->term = nmem_strdup(s->session_nmem, term); + t->next = s->facet_id_list; + s->facet_id_list = t; +} + + +// Look up a facet term, and return matching id +// If facet type not found, returns 0 +// If facet type found, but no matching term, returns "" +const char *session_lookup_id_facet(struct session *s, struct client *cl, + const char *type, + const char *term) +{ + char *retval = 0; + struct facet_id *t = s->facet_id_list; + for (; t; t = t->next) { - int i; - for (i = 0; i < s->num_termlists; i++) - if (!strcmp(s->termlists[i].name, type)) - break; - if (i == s->num_termlists) + if (!strcmp(client_get_id(cl), t->client_id) && !strcmp(t->type, type) ) { - if (i == SESSION_MAX_TERMLISTS) + retval = ""; + if ( !strcmp(t->term, term)) { - session_log(s, YLOG_FATAL, "Too many termlists"); - wrbuf_destroy(facet_wrbuf); - wrbuf_destroy(display_wrbuf); - return; + return t->id; } + } + } + return retval; +} + +void add_facet(struct session *s, const char *type, const char *value, int count, struct client *cl) +{ + WRBUF facet_wrbuf = wrbuf_alloc(); + WRBUF display_wrbuf = wrbuf_alloc(); + const char *id = 0; + size_t id_len = 0; - s->termlists[i].name = nmem_strdup(s->nmem, type); - s->termlists[i].termlist = termlist_create(s->nmem); - s->num_termlists = i + 1; + /* inspect pz:facetmap:split:name ?? */ + if (!strncmp(type, "split:", 6)) + { + const char *cp = strchr(value, ':'); + if (cp) + { + id = value; + id_len = cp - value; + value = cp + 1; } + type += 6; + } -#if 0 - session_log(s, YLOG_LOG, "Facets for %s: %s norm:%s (%d)", type, value, wrbuf_cstr(facet_wrbuf), count); -#endif - termlist_insert(s->termlists[i].termlist, wrbuf_cstr(display_wrbuf), - wrbuf_cstr(facet_wrbuf), count); + session_normalize_facet(s, type, value, display_wrbuf, facet_wrbuf); + if (wrbuf_len(facet_wrbuf)) + { + struct named_termlist **tp = &s->termlists; + for (; (*tp); tp = &(*tp)->next) + if (!strcmp((*tp)->name, type)) + break; + if (!*tp) + { + *tp = nmem_malloc(s->nmem, sizeof(**tp)); + (*tp)->name = nmem_strdup(s->nmem, type); + (*tp)->termlist = termlist_create(s->nmem); + (*tp)->next = 0; + } + termlist_insert((*tp)->termlist, wrbuf_cstr(display_wrbuf), + wrbuf_cstr(facet_wrbuf), id, id_len, count); + if (id) + session_add_id_facet(s, cl, type, id, id_len, + wrbuf_cstr(display_wrbuf)); } wrbuf_destroy(facet_wrbuf); wrbuf_destroy(display_wrbuf); @@ -508,7 +570,6 @@ static void select_targets_callback(struct session *se, l->next = se->clients_cached; se->clients_cached = l; } - /* set session always. If may be 0 if client is not active */ client_set_session(cl, se); l = xmalloc(sizeof(*l)); @@ -557,6 +618,7 @@ static void session_remove_cached_clients(struct session *se) client_lock(l->client); client_set_session(l->client, 0); client_set_database(l->client, 0); + client_mark_dead(l->client); client_unlock(l->client); client_destroy(l->client); xfree(l); @@ -604,8 +666,7 @@ static void session_clear_set(struct session *se, struct reclist_sortparms *sp) nmem_total(se->nmem)); nmem_reset(se->nmem); se->total_records = se->total_merged = 0; - se->num_termlists = 0; - + se->termlists = 0; relevance_clear(se->relevance); /* reset list of sorted results and clear to relevance search */ @@ -621,10 +682,8 @@ static void session_clear_set(struct session *se, struct reclist_sortparms *sp) se->reclist = reclist_create(se->nmem); } -static void session_sort_unlocked(struct session *se, - struct reclist_sortparms *sp, - const char *mergekey, - const char *rank) +void session_sort(struct session *se, struct reclist_sortparms *sp, + const char *mergekey, const char *rank) { struct client_list *l; const char *field = sp->name; @@ -632,6 +691,7 @@ static void session_sort_unlocked(struct session *se, int type = sp->type; int clients_research = 0; + session_enter(se, "session_sort"); session_log(se, YLOG_DEBUG, "session_sort field=%s increasing=%d type=%d", field, increasing, type); @@ -663,8 +723,9 @@ static void session_sort_unlocked(struct session *se, break; if (sr) { - session_log(se, YLOG_DEBUG, "session_sort: field=%s increasing=%d type=%d already fetched", + session_log(se, YLOG_LOG, "session_sort: field=%s increasing=%d type=%d already fetched", field, increasing, type); + session_leave(se, "session_sort"); return; } } @@ -680,16 +741,9 @@ static void session_sort_unlocked(struct session *se, struct client *cl = l->client; // Assume no re-search is required. client_parse_init(cl, 1); - clients_research += client_parse_sort(cl, sp); + clients_research += client_parse_sort(cl, sp, 0); } - if (clients_research) - { - session_log(se, YLOG_DEBUG, - "session_sort: reset results due to %d clients researching", - clients_research); - session_clear_set(se, sp); - } - else + if (!clients_research || se->clients_starting) { // A new sorting based on same record set struct reclist_sortparms *sr = nmem_malloc(se->nmem, sizeof(*sr)); @@ -699,36 +753,67 @@ static void session_sort_unlocked(struct session *se, sr->next = se->sorted_results; se->sorted_results = sr; session_log(se, YLOG_DEBUG, "session_sort: no research/ingesting done"); - return ; + session_leave(se, "session_sort"); } - session_log(se, YLOG_DEBUG, "Re- search/ingesting for clients due to change in sort order"); - - for (l = se->clients_active; l; l = l->next) + else { - struct client *cl = l->client; - if (client_get_state(cl) == Client_Connecting || - client_get_state(cl) == Client_Idle || - client_get_state(cl) == Client_Working) { - client_start_search(cl); - } - else + se->clients_starting = 1; + session_log(se, YLOG_DEBUG, + "session_sort: reset results due to %d clients researching", + clients_research); + session_clear_set(se, sp); + session_log(se, YLOG_DEBUG, "Re- search/ingesting for clients due to change in sort order"); + + session_leave(se, "session_sort"); + for (l = se->clients_active; l; l = l->next) { - session_log(se, YLOG_DEBUG, - "session_sort: %s: No re-start/ingest in show. " - "Wrong client state: %d", - client_get_id(cl), client_get_state(cl)); + struct client *cl = l->client; + if (client_get_state(cl) == Client_Connecting || + client_get_state(cl) == Client_Idle || + client_get_state(cl) == Client_Working) { + client_start_search(cl); + } + else + { + session_log(se, YLOG_DEBUG, + "session_sort: %s: No re-start/ingest in show. " + "Wrong client state: %d", + client_get_id(cl), client_get_state(cl)); + } } + session_enter(se, "session_sort"); + se->clients_starting = 0; + se->force_position = 0; + session_leave(se, "session_sort"); } } -void session_sort(struct session *se, struct reclist_sortparms *sp, - const char *mergekey, const char *rank) +void session_stop(struct session *se) { - //session_enter(se, "session_sort"); - session_sort_unlocked(se, sp, mergekey, rank); - //session_leave(se, "session_sort"); -} + struct client_list *l; + session_enter(se, "session_stop1"); + if (se->clients_starting) + { + session_leave(se, "session_stop1"); + return; + } + se->clients_starting = 1; + session_leave(se, "session_stop1"); + session_alert_watch(se, SESSION_WATCH_SHOW); + session_alert_watch(se, SESSION_WATCH_BYTARGET); + session_alert_watch(se, SESSION_WATCH_TERMLIST); + session_alert_watch(se, SESSION_WATCH_SHOW_PREF); + + for (l = se->clients_active; l; l = l->next) + { + struct client *cl = l->client; + client_stop(cl); + } + session_enter(se, "session_stop2"); + se->clients_starting = 0; + session_leave(se, "session_stop2"); +} enum pazpar2_error_code session_search(struct session *se, const char *query, @@ -746,17 +831,27 @@ enum pazpar2_error_code session_search(struct session *se, int no_working = 0; int no_failed_query = 0; int no_failed_limit = 0; - struct client_list *l, *l0; - - session_alert_watch(se, SESSION_WATCH_SHOW); - session_alert_watch(se, SESSION_WATCH_BYTARGET); - session_alert_watch(se, SESSION_WATCH_TERMLIST); - session_alert_watch(se, SESSION_WATCH_SHOW_PREF); + int no_sortmap = 0; + struct client_list *l; - session_log(se, YLOG_DEBUG, "Search"); + session_log(se, YLOG_LOG, "search query %s", query); + if (filter) + session_log(se, YLOG_LOG, "search filter %s", filter); + if (limit) + session_log(se, YLOG_LOG, "search limit %s", limit); *addinfo = 0; + session_enter(se, "session_search0"); + if (se->clients_starting) + { + session_leave(se, "session_search0"); + return PAZPAR2_NO_ERROR; + } + se->clients_starting = 1; + se->force_position = 0; + session_leave(se, "session_search0"); + if (se->settings_modified) { session_remove_cached_clients(se); } @@ -784,6 +879,7 @@ enum pazpar2_error_code session_search(struct session *se, if (!live_channels) { session_leave(se, "session_search"); + se->clients_starting = 0; return PAZPAR2_NO_TARGETS; } @@ -793,14 +889,19 @@ enum pazpar2_error_code session_search(struct session *se, { *addinfo = "limit"; session_leave(se, "session_search"); + se->clients_starting = 0; + session_reset_active_clients(se, 0); return PAZPAR2_MALFORMED_PARAMETER_VALUE; } - l0 = se->clients_active; - se->clients_active = 0; session_leave(se, "session_search"); - for (l = l0; l; l = l->next) + session_alert_watch(se, SESSION_WATCH_SHOW); + session_alert_watch(se, SESSION_WATCH_BYTARGET); + session_alert_watch(se, SESSION_WATCH_TERMLIST); + session_alert_watch(se, SESSION_WATCH_SHOW_PREF); + + for (l = se->clients_active; l; l = l->next) { int parse_ret; struct client *cl = l->client; @@ -818,13 +919,22 @@ enum pazpar2_error_code session_search(struct session *se, else { client_parse_range(cl, startrecs, maxrecs); - client_parse_sort(cl, sp); + client_parse_sort(cl, sp, &no_sortmap); client_start_search(cl); no_working++; } } - session_reset_active_clients(se, l0); - + session_log(se, YLOG_LOG, "search " + "working %d sortmap %d failed-query %d failed-limit %d", + no_working, no_sortmap, no_failed_query, no_failed_limit); + session_enter(se, "session_search2"); + if (no_working == 1 && no_sortmap == 1) + { + se->force_position = 1; + yaz_log(YLOG_LOG, "force_position=1"); + } + se->clients_starting = 0; + session_leave(se, "session_search2"); if (no_working == 0) { if (no_failed_query > 0) @@ -883,7 +993,7 @@ void session_init_databases(struct session *se) // Probably session_init_databases_fun should be refactored instead of // called here. static struct session_database *load_session_database(struct session *se, - char *id) + const char *id) { struct database *db = new_database_inherit_settings(id, se->session_nmem, se->service->settings); session_init_databases_fun((void*) se, db); @@ -894,7 +1004,7 @@ static struct session_database *load_session_database(struct session *se, // Find an existing session database. If not found, load it static struct session_database *find_session_database(struct session *se, - char *id) + const char *id) { struct session_database *sdb; @@ -905,45 +1015,55 @@ static struct session_database *find_session_database(struct session *se, } // Apply a session override to a database -void session_apply_setting(struct session *se, char *dbname, char *name, - char *value) +void session_apply_setting(struct session *se, const char *dbname, + const char *name, const char *value) { - struct session_database *sdb = find_session_database(se, dbname); - struct conf_service *service = se->service; - struct setting *s; - int offset = settings_create_offset(service, name); - - expand_settings_array(&sdb->settings, &sdb->num_settings, offset, - se->session_nmem); - - // Force later recompute of settings-driven data structures - // (happens when a search starts and client connections are prepared) - if (offset == PZ_XSLT) - sdb->map = 0; - - se->settings_modified = 1; - for (s = sdb->settings[offset]; s; s = s->next) - if (!strcmp(s->name, name) && - dbname && s->target && !strcmp(dbname, s->target)) + session_enter(se, "session_apply_setting"); + { + struct session_database *sdb = find_session_database(se, dbname); + struct conf_service *service = se->service; + struct setting *s; + int offset = settings_create_offset(service, name); + + expand_settings_array(&sdb->settings, &sdb->num_settings, offset, + se->session_nmem); + // Force later recompute of settings-driven data structures + // (happens when a search starts and client connections are prepared) + if (offset == PZ_XSLT) + sdb->map = 0; + se->settings_modified = 1; + for (s = sdb->settings[offset]; s; s = s->next) + if (!strcmp(s->name, name) && + dbname && s->target && !strcmp(dbname, s->target)) + break; + if (!s) { - s->value = value; - return; + s = nmem_malloc(se->session_nmem, sizeof(*s)); + s->precedence = 0; + s->target = nmem_strdup(se->session_nmem, dbname); + s->name = nmem_strdup(se->session_nmem, name); + s->next = sdb->settings[offset]; + sdb->settings[offset] = s; } - s = nmem_malloc(se->session_nmem, sizeof(*s)); - s->precedence = 0; - s->target = dbname; - s->name = name; - s->value = value; - s->next = sdb->settings[offset]; - sdb->settings[offset] = s; - + s->value = nmem_strdup(se->session_nmem, value); + } + session_leave(se, "session_apply_setting"); } void session_destroy(struct session *se) { struct session_database *sdb; - session_log(se, YLOG_LOG, "destroy"); - session_use(-1); + struct facet_id *t; + int sessions_total = session_use(-1); + int no_facet_ids = 0; + + for (t = se->facet_id_list; t; t = t->next) + no_facet_ids++; + session_log(se, YLOG_LOG, "destroy " + "session-total %d nmem-op %zd nmem-ses %zd facets-ids %d", + sessions_total, + nmem_total(se->nmem), nmem_total(se->session_nmem), + no_facet_ids); session_remove_cached_clients(se); for (sdb = se->databases; sdb; sdb = sdb->next) @@ -953,10 +1073,6 @@ void session_destroy(struct session *se) reclist_destroy(se->reclist); xfree(se->mergekey); xfree(se->rank); - if (nmem_total(se->nmem)) - session_log(se, YLOG_DEBUG, "NMEN operation usage %zd", nmem_total(se->nmem)); - if (nmem_total(se->session_nmem)) - session_log(se, YLOG_DEBUG, "NMEN session usage %zd", nmem_total(se->session_nmem)); facet_limits_destroy(se->facet_limits); nmem_destroy(se->nmem); service_destroy(se->service); @@ -974,8 +1090,8 @@ size_t session_get_memory_status(struct session *session) { } -struct session *new_session(NMEM nmem, struct conf_service *service, - unsigned session_id) +struct session *session_create(NMEM nmem, struct conf_service *service, + unsigned session_id) { int i; struct session *session = nmem_malloc(nmem, sizeof(*session)); @@ -991,18 +1107,21 @@ struct session *new_session(NMEM nmem, struct conf_service *service, session->total_records = 0; session->number_of_warnings_unknown_elements = 0; session->number_of_warnings_unknown_metadata = 0; - session->num_termlists = 0; + session->termlists = 0; session->reclist = reclist_create(nmem); session->clients_active = 0; session->clients_cached = 0; session->settings_modified = 0; session->session_nmem = nmem; + session->facet_id_list = 0; session->nmem = nmem_create(); session->databases = 0; session->sorted_results = 0; session->facet_limits = 0; session->mergekey = 0; session->rank = 0; + session->clients_starting = 0; + session->force_position = 0; for (i = 0; i <= SESSION_WATCH_MAX; i++) { @@ -1012,14 +1131,12 @@ struct session *new_session(NMEM nmem, struct conf_service *service, session->normalize_cache = normalize_cache_create(); session->session_mutex = 0; pazpar2_mutex_create(&session->session_mutex, tmp_str); - session_log(session, YLOG_LOG, "create"); - session_use(1); + i = session_use(1); + session_log(session, YLOG_LOG, "create session-total %d", i); return session; } -const char * client_get_suggestions_xml(struct client *cl, WRBUF wrbuf); - static struct hitsbytarget *hitsbytarget_nb(struct session *se, int *count, NMEM nmem) { @@ -1038,13 +1155,13 @@ static struct hitsbytarget *hitsbytarget_nb(struct session *se, WRBUF w = wrbuf_alloc(); const char *name = session_setting_oneval(client_get_database(cl), PZ_NAME); - res[*count].id = client_get_id(cl); res[*count].name = *name ? name : "Unknown"; res[*count].hits = client_get_hits(cl); res[*count].approximation = client_get_approximation(cl); - res[*count].records = client_get_num_records(cl); - res[*count].filtered = client_get_num_records_filtered(cl); + res[*count].records = client_get_num_records(cl, + &res[*count].filtered, + 0, 0); res[*count].diagnostic = client_get_diagnostic(cl, &res[*count].message, &res[*count].addinfo); @@ -1053,8 +1170,11 @@ static struct hitsbytarget *hitsbytarget_nb(struct session *se, session_settings_dump(se, client_get_database(cl), w); res[*count].settings_xml = nmem_strdup(nmem, wrbuf_cstr(w)); wrbuf_rewind(w); - wrbuf_puts(w, ""); - res[*count].suggestions_xml = nmem_strdup(nmem, client_get_suggestions_xml(cl, w)); + res[*count].suggestions_xml = + nmem_strdup(nmem, client_get_suggestions_xml(cl, w)); + + res[*count].query_data = + client_get_query(cl, &res[*count].query_type, nmem); wrbuf_destroy(w); (*count)++; } @@ -1139,7 +1259,7 @@ static int targets_termlist_nb(WRBUF wrbuf, struct session *se, int num, void perform_termlist(struct http_channel *c, struct session *se, const char *name, int num, int version) { - int i, j; + int j; NMEM nmem_tmp = nmem_create(); char **names; int num_names = 0; @@ -1156,9 +1276,10 @@ void perform_termlist(struct http_channel *c, struct session *se, const char *tname; int must_generate_empty = 1; /* bug 5350 */ - for (i = 0; i < se->num_termlists; i++) + struct named_termlist *t = se->termlists; + for (; t; t = t->next) { - tname = se->termlists[i].name; + tname = t->name; if (!strcmp(names[j], tname) || !strcmp(names[j], "*")) { struct termlist_score **p = 0; @@ -1169,8 +1290,7 @@ void perform_termlist(struct http_channel *c, struct session *se, wrbuf_puts(c->wrbuf, "\">\n"); must_generate_empty = 0; - p = termlist_highscore(se->termlists[i].termlist, &len, - nmem_tmp); + p = termlist_highscore(t->termlist, &len, nmem_tmp); if (p) { int i; @@ -1184,7 +1304,6 @@ void perform_termlist(struct http_channel *c, struct session *se, wrbuf_puts(c->wrbuf, ""); wrbuf_xmlputs(c->wrbuf, p[i]->display_term); wrbuf_puts(c->wrbuf, ""); - wrbuf_printf(c->wrbuf, "%d", p[i]->frequency); @@ -1280,14 +1399,24 @@ int session_fetch_more(struct session *se) } else { - session_log(se, YLOG_LOG, "%s: no more to fetch", - client_get_id(cl)); + int filtered; + int ingest_failures; + int record_failures; + int num = client_get_num_records( + cl, &filtered, &ingest_failures, &record_failures); + session_log(se, YLOG_LOG, "%s: hits=" ODR_INT_PRINTF - " records=%d filtered=%d", + " fetched=%d filtered=%d", client_get_id(cl), client_get_hits(cl), - client_get_num_records(cl), - client_get_num_records_filtered(cl)); + num, filtered); + if (ingest_failures || record_failures) + { + session_log(se, YLOG_WARN, "%s:" + " ingest failures=%d record failures=%d", + client_get_id(cl), + ingest_failures, record_failures); + } } } else @@ -1311,6 +1440,7 @@ struct record_cluster **show_range_start(struct session *se, struct reclist_sortparms *spp; struct client_list *l; int i; + NMEM nmem_tmp = 0; #if USE_TIMING yaz_timing_t t = yaz_timing_create(); #endif @@ -1332,7 +1462,15 @@ struct record_cluster **show_range_start(struct session *se, *approx_hits += client_get_approximation(l->client); } } + if (se->force_position) + { + nmem_tmp = nmem_create(); + sp = reclist_parse_sortparms(nmem_tmp, "position:1", 0); + assert(sp); + } reclist_sort(se->reclist, sp); + if (nmem_tmp) + nmem_destroy(nmem_tmp); reclist_enter(se->reclist); *total = reclist_get_num_records(se->reclist); @@ -1437,7 +1575,8 @@ void statistics(struct session *se, struct statistics *stat) } static struct record_metadata *record_metadata_init( - NMEM nmem, const char *value, enum conf_metadata_type type, + NMEM nmem, const char *value, const char *norm, + enum conf_metadata_type type, struct _xmlAttr *attr) { struct record_metadata *rec_md = record_metadata_create(nmem); @@ -1467,11 +1606,20 @@ static struct record_metadata *record_metadata_init( { case Metadata_type_generic: case Metadata_type_skiparticle: - if (strstr(value, "://")) /* looks like a URL */ + if (norm) + { rec_md->data.text.disp = nmem_strdup(nmem, value); + rec_md->data.text.norm = nmem_strdup(nmem, norm); + } else - rec_md->data.text.disp = - normalize7bit_generic(nmem_strdup(nmem, value), " ,/.:(["); + { + if (strstr(value, "://")) /* looks like a URL */ + rec_md->data.text.disp = nmem_strdup(nmem, value); + else + rec_md->data.text.disp = + normalize7bit_generic(nmem_strdup(nmem, value), " ,/.:(["); + rec_md->data.text.norm = rec_md->data.text.disp; + } rec_md->data.text.sort = 0; rec_md->data.text.snippet = 0; break; @@ -1495,6 +1643,7 @@ static struct record_metadata *record_metadata_init( break; case Metadata_type_relevance: case Metadata_type_position: + case Metadata_type_retrieval: return 0; } return rec_md; @@ -1531,7 +1680,7 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, continue; if (!strcmp((const char *) n->name, "metadata")) { - xmlChar *type = xmlGetProp(n, (xmlChar *) "type"); + const char *type = yaz_xml_get_prop(n, "type"); if (type == NULL) { yaz_log(YLOG_FATAL, "Missing type attribute on metadata element. Skipping!"); } @@ -1550,7 +1699,6 @@ static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, if (value) xmlFree(value); } - xmlFree(type); } } return no_found; @@ -1563,7 +1711,7 @@ static const char *get_mergekey(xmlDoc *doc, xmlNode *root, { char *mergekey_norm = 0; WRBUF norm_wr = wrbuf_alloc(); - xmlChar *mergekey; + const char *mergekey; if (session_mergekey) { @@ -1575,10 +1723,9 @@ static const char *get_mergekey(xmlDoc *doc, xmlNode *root, for (i = 0; i < num; i++) get_mergekey_from_doc(doc, root, values[i], service, norm_wr); } - else if ((mergekey = xmlGetProp(root, (xmlChar *) "mergekey"))) + else if ((mergekey = yaz_xml_get_prop(root, "mergekey"))) { - mergekey_norm_wr(service->charsets, norm_wr, (const char *) mergekey); - xmlFree(mergekey); + mergekey_norm_wr(service->charsets, norm_wr, mergekey); } else { @@ -1605,7 +1752,7 @@ static const char *get_mergekey(xmlDoc *doc, xmlNode *root, /* generate unique key if none is not generated already or is empty */ if (wrbuf_len(norm_wr) == 0) { - wrbuf_printf(norm_wr, "position: %s-%d", + wrbuf_printf(norm_wr, "position: %s-%06d", client_get_id(cl), record_no); } else @@ -1645,7 +1792,7 @@ static int check_record_filter(xmlNode *root, struct session_database *sdb) continue; if (!strcmp((const char *) n->name, "metadata")) { - xmlChar *type = xmlGetProp(n, (xmlChar *) "type"); + const char *type = yaz_xml_get_prop(n, "type"); if (type) { size_t len; @@ -1673,7 +1820,6 @@ static int check_record_filter(xmlNode *root, struct session_database *sdb) } xmlFree(value); } - xmlFree(type); } } } @@ -1681,6 +1827,8 @@ static int check_record_filter(xmlNode *root, struct session_database *sdb) } static int ingest_to_cluster(struct client *cl, + WRBUF wrbuf_disp, + WRBUF wrbuf_norm, xmlDoc *xdoc, xmlNode *root, int record_no, @@ -1693,6 +1841,7 @@ static int ingest_sub_record(struct client *cl, xmlDoc *xdoc, xmlNode *root, { int ret = 0; struct session *se = client_get_session(cl); + WRBUF wrbuf_disp, wrbuf_norm; if (!check_record_filter(root, sdb)) { @@ -1701,11 +1850,15 @@ static int ingest_sub_record(struct client *cl, xmlDoc *xdoc, xmlNode *root, record_no, sdb->database->id); return 0; } + wrbuf_disp = wrbuf_alloc(); + wrbuf_norm = wrbuf_alloc(); session_enter(se, "ingest_sub_record"); if (client_get_session(cl) == se && se->relevance) - ret = ingest_to_cluster(cl, xdoc, root, record_no, mergekeys); + ret = ingest_to_cluster(cl, wrbuf_disp, wrbuf_norm, + xdoc, root, record_no, mergekeys); session_leave(se, "ingest_sub_record"); - + wrbuf_destroy(wrbuf_norm); + wrbuf_destroy(wrbuf_disp); return ret; } @@ -1986,14 +2139,14 @@ static int check_limit_local(struct client *cl, } static int ingest_to_cluster(struct client *cl, + WRBUF wrbuf_disp, + WRBUF wrbuf_norm, xmlDoc *xdoc, xmlNode *root, int record_no, struct record_metadata_attr *merge_keys) { xmlNode *n; - xmlChar *type = 0; - xmlChar *value = 0; struct session *se = client_get_session(cl); struct conf_service *service = se->service; int term_factor = 1; @@ -2010,12 +2163,6 @@ static int ingest_to_cluster(struct client *cl, for (n = root->children; n; n = n->next) { - if (type) - xmlFree(type); - if (value) - xmlFree(value); - type = value = 0; - if (n->type != XML_ELEMENT_NODE) continue; if (!strcmp((const char *) n->name, "metadata")) @@ -2024,20 +2171,12 @@ static int ingest_to_cluster(struct client *cl, struct record_metadata **wheretoput = 0; struct record_metadata *rec_md = 0; int md_field_id = -1; + xmlChar *value0; + const char *type = yaz_xml_get_prop(n, "type"); - type = xmlGetProp(n, (xmlChar *) "type"); - value = xmlNodeListGetString(xdoc, n->children, 1); if (!type) continue; - if (!value || !*value) - { - xmlChar *empty = xmlGetProp(n, (xmlChar *) "empty"); - if (!empty) - continue; - if (value) - xmlFree(value); - value = empty; - } + md_field_id = conf_service_metadata_field_id(service, (const char *) type); if (md_field_id < 0) @@ -2051,15 +2190,31 @@ static int ingest_to_cluster(struct client *cl, continue; } + wrbuf_rewind(wrbuf_disp); + value0 = xmlNodeListGetString(xdoc, n->children, 1); + if (!value0 || !*value0) + { + const char *empty = yaz_xml_get_prop(n, "empty"); + if (value0) + xmlFree(value0); + if (!empty) + continue; + wrbuf_puts(wrbuf_disp, (const char *) empty); + } + else + { + wrbuf_puts(wrbuf_disp, (const char *) value0); + xmlFree(value0); + } ser_md = &service->metadata[md_field_id]; // non-merged metadata - rec_md = record_metadata_init(se->nmem, (const char *) value, + rec_md = record_metadata_init(se->nmem, wrbuf_cstr(wrbuf_disp), 0, ser_md->type, n->properties); if (!rec_md) { session_log(se, YLOG_WARN, "bad metadata data '%s' " - "for element '%s'", value, type); + "for element '%s'", wrbuf_cstr(wrbuf_disp), type); continue; } @@ -2067,7 +2222,7 @@ static int ingest_to_cluster(struct client *cl, { WRBUF w = wrbuf_alloc(); if (relevance_snippet(se->relevance, - (char*) value, ser_md->name, w)) + wrbuf_cstr(wrbuf_disp), ser_md->name, w)) rec_md->data.text.snippet = nmem_strdup(se->nmem, wrbuf_cstr(w)); wrbuf_destroy(w); @@ -2083,20 +2238,12 @@ static int ingest_to_cluster(struct client *cl, if (check_limit_local(cl, record, record_no)) { - if (type) - xmlFree(type); - if (value) - xmlFree(value); return -2; } cluster = reclist_insert(se->reclist, se->relevance, service, record, merge_keys, &se->total_merged); if (!cluster) { - if (type) - xmlFree(type); - if (value) - xmlFree(value); return 0; // complete match with existing record } @@ -2134,13 +2281,6 @@ static int ingest_to_cluster(struct client *cl, // now parsing XML record and adding data to cluster or record metadata for (n = root->children; n; n = n->next) { - pp2_charset_token_t prt; - if (type) - xmlFree(type); - if (value) - xmlFree(value); - type = value = 0; - if (n->type != XML_ELEMENT_NODE) continue; if (!strcmp((const char *) n->name, "metadata")) @@ -2152,12 +2292,12 @@ static int ingest_to_cluster(struct client *cl, int md_field_id = -1; int sk_field_id = -1; const char *rank = 0; - xmlChar *xml_rank = 0; + const char *xml_rank = 0; + const char *type = 0; + xmlChar *value0; - type = xmlGetProp(n, (xmlChar *) "type"); - value = xmlNodeListGetString(xdoc, n->children, 1); - - if (!type || !value || !*value) + type = yaz_xml_get_prop(n, "type"); + if (!type) continue; md_field_id @@ -2173,12 +2313,39 @@ static int ingest_to_cluster(struct client *cl, ser_sk = &service->sortkeys[sk_field_id]; } - // merged metadata - rec_md = record_metadata_init(se->nmem, (const char *) value, - ser_md->type, 0); + wrbuf_rewind(wrbuf_disp); + wrbuf_rewind(wrbuf_norm); - // see if the field was not in cluster already (from beginning) + value0 = xmlNodeListGetString(xdoc, n->children, 1); + if (!value0 || !*value0) + { + if (value0) + xmlFree(value0); + continue; + } + if (ser_md->icurule) + { + run_icu(se, ser_md->icurule, (const char *) value0, + wrbuf_norm, wrbuf_disp); + yaz_log(YLOG_LOG, "run_icu input=%s norm=%s disp=%s", + (const char *) value0, + wrbuf_cstr(wrbuf_norm), wrbuf_cstr(wrbuf_disp)); + rec_md = record_metadata_init(se->nmem, wrbuf_cstr(wrbuf_disp), + wrbuf_cstr(wrbuf_norm), + ser_md->type, 0); + } + else + { + wrbuf_puts(wrbuf_disp, (const char *) value0); + rec_md = record_metadata_init(se->nmem, wrbuf_cstr(wrbuf_disp), + 0, + ser_md->type, 0); + } + + xmlFree(value0); + + // see if the field was not in cluster already (from beginning) if (!rec_md) continue; @@ -2201,7 +2368,7 @@ static int ingest_to_cluster(struct client *cl, } else { - xml_rank = xmlGetProp(n, (xmlChar *) "rank"); + xml_rank = yaz_xml_get_prop(n, "rank"); rank = xml_rank ? (const char *) xml_rank : ser_md->rank; } @@ -2220,8 +2387,8 @@ static int ingest_to_cluster(struct client *cl, { while (*wheretoput) { - if (!strcmp((const char *) (*wheretoput)->data.text.disp, - rec_md->data.text.disp)) + if (!strcmp((const char *) (*wheretoput)->data.text.norm, + rec_md->data.text.norm)) break; wheretoput = &(*wheretoput)->next; } @@ -2231,12 +2398,13 @@ static int ingest_to_cluster(struct client *cl, else if (ser_md->merge == Metadata_merge_longest) { if (!*wheretoput - || strlen(rec_md->data.text.disp) - > strlen((*wheretoput)->data.text.disp)) + || strlen(rec_md->data.text.norm) + > strlen((*wheretoput)->data.text.norm)) { *wheretoput = rec_md; if (ser_sk) { + pp2_charset_token_t prt; const char *sort_str = 0; int skip_article = ser_sk->type == Metadata_type_skiparticle; @@ -2300,7 +2468,8 @@ static int ingest_to_cluster(struct client *cl, if (rank) { relevance_countwords(se->relevance, cluster, - (char *) value, rank, ser_md->name); + wrbuf_cstr(wrbuf_disp), + rank, ser_md->name); } // construct facets ... unless the client already has reported them if (ser_md->termlist && !client_has_facet(cl, (char *) type)) @@ -2310,23 +2479,16 @@ static int ingest_to_cluster(struct client *cl, char year[64]; sprintf(year, "%d", rec_md->data.number.max); - add_facet(se, (char *) type, year, term_factor); + add_facet(se, (char *) type, year, term_factor, cl); if (rec_md->data.number.max != rec_md->data.number.min) { sprintf(year, "%d", rec_md->data.number.min); - add_facet(se, (char *) type, year, term_factor); + add_facet(se, (char *) type, year, term_factor, cl); } } else - add_facet(se, (char *) type, (char *) value, term_factor); + add_facet(se, type, wrbuf_cstr(wrbuf_disp), term_factor, cl); } - - // cleaning up - if (xml_rank) - xmlFree(xml_rank); - xmlFree(type); - xmlFree(value); - type = value = 0; } else { @@ -2336,11 +2498,6 @@ static int ingest_to_cluster(struct client *cl, se->number_of_warnings_unknown_elements++; } } - if (type) - xmlFree(type); - if (value) - xmlFree(value); - nmem_destroy(ingest_nmem); xfree(metadata0); relevance_donerecord(se->relevance, cluster);