#include "relevance.h"
#include "incref.h"
+#define XDOC_CACHE_SIZE 100
+
static YAZ_MUTEX g_mutex = 0;
static int no_clients = 0;
return clients;
}
-int clients_count(void)
+int clients_get_count(void)
{
return client_use(0);
}
Odr_int hits;
int record_offset;
int show_stat_no;
- int filtered; // When using local:, this will count the number of filtered records.
+ int filtered; /* number of records ignored for local filtering */
+ int ingest_failures; /* number of records where XSLT/other failed */
+ int record_failures; /* number of records where ZOOM reported error */
int maxrecs;
int startrecs;
int diagnostic;
int same_search;
char *sort_strategy;
char *sort_criteria;
+ xmlDoc **xdoc;
};
struct suggestions {
}
}
+static void client_init_xdoc(struct client *cl)
+{
+ int i;
+
+ cl->xdoc = xmalloc(sizeof(*cl->xdoc) * XDOC_CACHE_SIZE);
+ for (i = 0; i < XDOC_CACHE_SIZE; i++)
+ cl->xdoc[i] = 0;
+}
+
+static void client_destroy_xdoc(struct client *cl)
+{
+ int i;
+
+ assert(cl->xdoc);
+ for (i = 0; i < XDOC_CACHE_SIZE; i++)
+ if (cl->xdoc[i])
+ xmlFreeDoc(cl->xdoc[i]);
+ xfree(cl->xdoc);
+}
+
+xmlDoc *client_get_xdoc(struct client *cl, int record_no)
+{
+ assert(cl->xdoc);
+ if (record_no >= 0 && record_no < XDOC_CACHE_SIZE)
+ return cl->xdoc[record_no];
+ return 0;
+}
+
+void client_store_xdoc(struct client *cl, int record_no, xmlDoc *xdoc)
+{
+ assert(cl->xdoc);
+ if (record_no >= 0 && record_no < XDOC_CACHE_SIZE)
+ {
+ if (cl->xdoc[record_no])
+ xmlFreeDoc(cl->xdoc[record_no]);
+ cl->xdoc[record_no] = xdoc;
+ }
+ else
+ {
+ xmlFreeDoc(xdoc);
+ }
+}
+
+
static void client_show_raw_error(struct client *cl, const char *addinfo);
struct connection *client_get_connection(struct client *cl)
ZOOM_facet_field_get_term(facets[facet_idx],
term_idx, &freq);
if (term)
- add_facet(se, p, term, freq);
+ add_facet(se, p, term, freq, cl);
}
break;
}
client_show_raw_dequeue(cl);
}
-void client_check_preferred_watch(struct client *cl)
-{
- struct session *se = cl->session;
- yaz_log(YLOG_DEBUG, "client_check_preferred_watch: %s ", client_get_id(cl));
- if (se)
- {
- client_unlock(cl);
- /* TODO possible threading issue. Session can have been destroyed */
- if (session_is_preferred_clients_ready(se)) {
- session_alert_watch(se, SESSION_WATCH_SHOW_PREF);
- }
- else
- yaz_log(YLOG_DEBUG, "client_check_preferred_watch: Still locked on preferred targets.");
-
- client_lock(cl);
- }
- else
- yaz_log(YLOG_WARN, "client_check_preferred_watch: %s. No session!", client_get_id(cl));
-
-}
-
struct suggestions* client_suggestions_create(const char* suggestions_string);
static void client_suggestions_destroy(struct client *cl);
struct connection *co = cl->connection;
ZOOM_connection link = connection_get_link(co);
ZOOM_resultset resultset = cl->resultset;
+ struct session *se = client_get_session(cl);
const char *error, *addinfo = 0;
if (ZOOM_connection_error(link, &error, &addinfo))
{
cl->hits = 0;
+ session_log(se, YLOG_WARN, "%s: Error %s (%s)",
+ client_get_id(cl), error, addinfo);
client_set_state(cl, Client_Error);
- yaz_log(YLOG_WARN, "Search error %s (%s): %s",
- error, addinfo, client_get_id(cl));
}
else
{
client_report_facets(cl, resultset);
cl->record_offset = cl->startrecs;
cl->hits = ZOOM_resultset_size(resultset);
- yaz_log(YLOG_DEBUG, "client_search_response: hits " ODR_INT_PRINTF,
- cl->hits);
+ session_log(se, YLOG_LOG, "%s: hits: " ODR_INT_PRINTF,
+ client_get_id(cl), cl->hits);
if (cl->suggestions)
client_suggestions_destroy(cl);
cl->suggestions =
struct session *se = cl->session;
if (se)
{
+ client_unlock(cl);
+ /* TODO possible threading issue. Session can have been destroyed */
+ if (session_is_preferred_clients_ready(se))
+ session_alert_watch(se, SESSION_WATCH_SHOW_PREF);
+ client_lock(cl);
if (reclist_get_num_records(se->reclist) > 0)
{
client_unlock(cl);
ZOOM_record rec = 0;
ZOOM_resultset resultset = cl->resultset;
struct session *se = client_get_session(cl);
+ xmlDoc *xdoc;
+ int offset = cl->record_offset + 1; /* 0 versus 1 numbered offsets */
- if ((rec = ZOOM_resultset_record_immediate(resultset, cl->record_offset)))
+ xdoc = client_get_xdoc(cl, offset);
+ if (xdoc)
+ {
+ if (cl->session)
+ {
+ NMEM nmem = nmem_create();
+ int rc = ingest_xml_record(cl, xdoc, offset, nmem, 1);
+ if (rc == -1)
+ {
+ session_log(se, YLOG_WARN,
+ "%s: #%d: failed to ingest xdoc",
+ client_get_id(cl), offset);
+ cl->ingest_failures++;
+ }
+ else if (rc == -2)
+ cl->filtered++;
+ nmem_destroy(nmem);
+ }
+ }
+ else if ((rec = ZOOM_resultset_record_immediate(resultset,
+ cl->record_offset)))
{
- int offset = ++cl->record_offset;
if (cl->session == 0)
; /* no operation */
else if (ZOOM_record_error(rec, &msg, &addinfo, 0))
{
session_log(se, YLOG_WARN, "Record error %s (%s): %s #%d",
msg, addinfo, client_get_id(cl), offset);
+ cl->record_failures++;
}
else
{
if (!xmlrec)
{
const char *rec_syn = ZOOM_record_get(rec, "syntax", NULL);
- session_log(se, YLOG_WARN, "ZOOM_record_get failed from %s #%d",
+ session_log(se, YLOG_WARN, "%s: #%d: ZOOM_record_get failed",
client_get_id(cl), offset);
session_log(se, YLOG_LOG, "pz:nativesyntax=%s . "
"ZOOM record type=%s . Actual record syntax=%s",
s ? s : "null", type,
rec_syn ? rec_syn : "null");
+ cl->ingest_failures++;
}
else
{
/* OK = 0, -1 = failure, -2 = Filtered */
- int rc = ingest_record(cl, xmlrec, cl->record_offset, nmem);
+ int rc = ingest_record(cl, xmlrec, offset, nmem);
if (rc == -1)
{
const char *rec_syn = ZOOM_record_get(rec, "syntax", NULL);
session_log(se, YLOG_WARN,
- "Failed to ingest record from %s #%d",
+ "%s: #%d: failed to ingest record",
client_get_id(cl), offset);
session_log(se, YLOG_LOG, "pz:nativesyntax=%s . "
"ZOOM record type=%s . Actual record syntax=%s",
s ? s : "null", type,
rec_syn ? rec_syn : "null");
+ cl->ingest_failures++;
}
- if (rc == -2)
- cl->filtered += 1;
+ else if (rc == -2)
+ cl->filtered++;
}
nmem_destroy(nmem);
}
else
{
session_log(se, YLOG_WARN, "Got NULL record from %s #%d",
- client_get_id(cl), cl->record_offset);
+ client_get_id(cl), offset);
}
+ cl->record_offset++;
}
void client_record_response(struct client *cl, int *got_records)
if (ZOOM_connection_error(link, &error, &addinfo))
{
+ struct session *se = client_get_session(cl);
+ session_log(se, YLOG_WARN, "%s: Error %s (%s)",
+ client_get_id(cl), error, addinfo);
client_set_state(cl, Client_Error);
- yaz_log(YLOG_WARN, "Search error %s (%s): %s",
- error, addinfo, client_get_id(cl));
}
else
{
{
int i = cl->startrecs;
int to = cl->record_offset;
- cl->filtered = 0;
+ cl->record_failures = cl->ingest_failures = cl->filtered = 0;
cl->record_offset = i;
for (; i < to; i++)
for (s = sdb->settings[PZ_FACETMAP]; s; s = s->next)
{
const char *p = strchr(s->name + 3, ':');
+ if ( !strncmp(p, ":split:", 7) )
+ p += 6; // PAZ-1009
if (p && !strcmp(name, p + 1))
return 1;
}
const char *str;
int extend_recs = 0;
int number = cl->hits - cl->record_offset;
+ struct connection *co = client_get_connection(cl);
str = session_setting_oneval(sdb, PZ_EXTENDRECS);
if (!str || !*str)
extend_recs = atoi(str);
- yaz_log(YLOG_LOG, "cl=%s show_stat_no=%d got=%d",
+ yaz_log(YLOG_DEBUG, "cl=%s show_stat_no=%d got=%d",
client_get_id(cl), cl->show_stat_no, cl->record_offset);
if (cl->show_stat_no < cl->record_offset)
return 0;
- yaz_log(YLOG_LOG, "cl=%s Trying to fetch more", client_get_id(cl));
+ yaz_log(YLOG_DEBUG, "cl=%s Trying to fetch more", client_get_id(cl));
if (number > extend_recs)
number = extend_recs;
- if (number > 0)
+ if (number <= 0)
+ yaz_log(YLOG_DEBUG, "cl=%s. OK no more in total set", client_get_id(cl));
+ else if (!co)
+ yaz_log(YLOG_DEBUG, "cl=%s. No connection", client_get_id(cl));
+ else
{
ZOOM_resultset set = cl->resultset;
- struct connection *co = client_get_connection(cl);
str = session_setting_oneval(sdb, PZ_REQUESTSYNTAX);
ZOOM_resultset_option_set(set, "preferredRecordSyntax", str);
connection_continue(co);
return 1;
}
- else
- {
- yaz_log(YLOG_LOG, "cl=%s. OK no more in total set", client_get_id(cl));
- }
return 0;
}
return 0;
}
+const char *client_get_query(struct client *cl, const char **type, NMEM nmem)
+{
+ if (cl->pquery)
+ {
+ *type = "pqf";
+ return nmem_strdup(nmem, cl->pquery);
+ }
+ if (cl->cqlquery)
+ {
+ *type = "cql";
+ return nmem_strdup(nmem, cl->cqlquery);
+ }
+ *type = 0;
+ return 0;
+}
+
int client_start_search(struct client *cl)
{
struct session_database *sdb = client_get_database(cl);
const char *opt_preferred = session_setting_oneval(sdb, PZ_PREFERRED);
const char *extra_args = session_setting_oneval(sdb, PZ_EXTRA_ARGS);
const char *opt_present_chunk = session_setting_oneval(sdb, PZ_PRESENT_CHUNK);
+ const char *opt_timeout = session_setting_oneval(sdb, PZ_TIMEOUT);
ZOOM_query query;
char maxrecs_str[24], startrecs_str[24], present_chunk_str[24];
struct timeval tval;
int present_chunk = 20; // Default chunk size
int rc_prep_connection;
+ int operation_timeout = se->service->z3950_operation_timeout;
+ cl->diagnostic = 0;
+ cl->record_failures = cl->ingest_failures = cl->filtered = 0;
yaz_gettimeofday(&tval);
tval.tv_sec += 5;
+ if (opt_timeout && *opt_timeout)
+ operation_timeout = atoi(opt_timeout);
+
if (opt_present_chunk && strcmp(opt_present_chunk,"")) {
present_chunk = atoi(opt_present_chunk);
yaz_log(YLOG_DEBUG, "Present chunk set to %d", present_chunk);
}
rc_prep_connection =
- client_prep_connection(cl, se->service->z3950_operation_timeout,
+ client_prep_connection(cl, operation_timeout,
se->service->z3950_session_timeout,
se->service->server->iochan_man,
&tval);
/* Nothing has changed and we already have a result */
if (cl->same_search == 1 && rc_prep_connection == 2)
{
- session_log(se, YLOG_LOG, "client %s resuse result", client_get_id(cl));
+ session_log(se, YLOG_LOG, "%s: reuse result", client_get_id(cl));
client_report_facets(cl, cl->resultset);
return client_reingest(cl);
}
else if (!rc_prep_connection)
{
- session_log(se, YLOG_LOG, "client %s postponing search: No connection",
- client_get_id(cl));
- client_set_state_nb(cl, Client_Working);
+ client_set_diagnostic(cl, 2,
+ ZOOM_diag_str(2),
+ "Cannot create connection");
+ client_set_state_nb(cl, Client_Error);
return -1;
}
co = client_get_connection(cl);
link = connection_get_link(co);
assert(link);
- session_log(se, YLOG_LOG, "client %s new search", client_get_id(cl));
-
- cl->diagnostic = 0;
- cl->filtered = 0;
+ client_destroy_xdoc(cl);
+ client_init_xdoc(cl);
if (extra_args && *extra_args)
ZOOM_connection_option_set(link, "extraArgs", extra_args);
query = ZOOM_query_create();
if (cl->cqlquery)
{
- yaz_log(YLOG_LOG, "Client %s: Search CQL: %s", client_get_id(cl),
- cl->cqlquery);
+ session_log(se, YLOG_LOG, "%s: Search CQL: %s", client_get_id(cl),
+ cl->cqlquery);
ZOOM_query_cql(query, cl->cqlquery);
if (*opt_sort)
ZOOM_query_sortby(query, opt_sort);
}
else
{
- yaz_log(YLOG_LOG, "Client %s: Search PQF: %s", client_get_id(cl),
- cl->pquery);
-
+ session_log(se, YLOG_LOG, "%s: Search PQF: %s", client_get_id(cl),
+ cl->pquery);
ZOOM_query_prefix(query, cl->pquery);
}
if (cl->sort_strategy && cl->sort_criteria) {
cl->sort_criteria = 0;
assert(id);
cl->id = xstrdup(id);
+ client_init_xdoc(cl);
client_use(1);
yaz_log(YLOG_DEBUG, "client_create c=%p %s", cl, id);
assert(!c->connection);
facet_limits_destroy(c->facet_limits);
+ client_destroy_xdoc(c);
if (c->resultset)
{
ZOOM_resultset_destroy(c->resultset);
}
else
{
+ client_lock(cl);
cl->connection = con;
+ client_unlock(cl);
client_destroy(cl);
}
}
client_set_connection(cl, 0);
}
+void client_mark_dead(struct client *cl)
+{
+ if (cl->connection)
+ connection_mark_dead(cl->connection);
+}
+
+void client_stop(struct client *cl)
+{
+ client_lock(cl);
+ if (cl->state == Client_Working || cl->state == Client_Connecting)
+ {
+ yaz_log(YLOG_LOG, "client_stop: %s release", client_get_id(cl));
+ if (cl->connection)
+ {
+ connection_release2(cl->connection);
+ assert(cl->ref_count > 1);
+ cl->ref_count--;
+ cl->connection = 0;
+ }
+ cl->state = Client_Disconnected;
+ }
+ else
+ yaz_log(YLOG_LOG, "client_stop: %s ignore", client_get_id(cl));
+ client_unlock(cl);
+}
+
// Initialize CCL map for a target
static CCL_bibset prepare_cclmap(struct client *cl, CCL_bibset base_bibset)
{
return 0;
}
-static int apply_limit(struct session_database *sdb,
+static void ccl_quote_map_term(CCL_bibset ccl_map, WRBUF w,
+ const char *term)
+{
+ int quote_it = 0;
+ const char *cp;
+ for (cp = term; *cp; cp++)
+ if ((*cp >= '0' && *cp <= '9') || strchr(" +-", *cp))
+ ;
+ else
+ quote_it = 1;
+ if (!quote_it)
+ wrbuf_puts(w, term);
+ else
+ {
+ wrbuf_putc(w, '\"');
+ for (cp = term; *cp; cp++)
+ {
+ if (strchr( "\\\"", *cp))
+ wrbuf_putc(w, '\\');
+ wrbuf_putc(w, *cp);
+ }
+ wrbuf_putc(w, '\"');
+ }
+}
+
+static int apply_limit(struct client *cl,
facet_limits_t facet_limits,
WRBUF w_pqf, CCL_bibset ccl_map,
struct conf_service *service)
int i = 0;
const char *name;
const char *value;
+ struct session_database *sdb = client_get_database(cl);
NMEM nmem_tmp = nmem_create();
for (i = 0; (name = facet_limits_get(facet_limits, i, &value)); i++)
nmem_strsplit_escape2(nmem_tmp, "|", value, &values,
&num, 1, '\\', 1);
+ for (i = 0; i < num; i++)
+ {
+ const char *id = session_lookup_id_facet(cl->session,
+ cl, name,
+ values[i]);
+ if (id)
+ {
+ if ( *id )
+ {
+ values[i] = nmem_strdup(nmem_tmp, id);
+ yaz_log(YLOG_DEBUG,
+ "apply_limit: s='%s' found id '%s'",s->name,id );
+ }
+ else
+ {
+ yaz_log(YLOG_DEBUG,
+ "apply_limit: %s: term '%s' not found, failing client",
+ s->name, values[i] );
+ ret = -1;
+ }
+ }
+ }
nmem_strsplit_escape2(nmem_tmp, ",", s->value, &cvalues,
&cnum, 1, '\\', 1);
struct ccl_rpn_node *cn;
wrbuf_rewind(ccl_w);
wrbuf_puts(ccl_w, ccl);
- wrbuf_puts(ccl_w, "=\"");
- wrbuf_puts(ccl_w, values[i]);
- wrbuf_puts(ccl_w, "\"");
-
+ wrbuf_putc(ccl_w, '=');
+ ccl_quote_map_term(ccl_map, ccl_w, values[i]);
cn = ccl_find_str(ccl_map, wrbuf_cstr(ccl_w),
&cerror, &cpos);
if (cn)
if (!ccl_map)
return -3;
+ xfree(cl->cqlquery);
+ cl->cqlquery = 0;
+
w_ccl = wrbuf_alloc();
wrbuf_puts(w_ccl, query);
wrbuf_puts(w_pqf, " ");
}
- if (apply_limit(sdb, facet_limits, w_pqf, ccl_map, service))
+ if (apply_limit(cl, facet_limits, w_pqf, ccl_map, service))
{
+ client_set_state(cl, Client_Error);
ccl_qual_rm(&ccl_map);
+
+ wrbuf_destroy(w_ccl);
+ wrbuf_destroy(w_pqf);
+
+ xfree(cl->pquery);
+ cl->pquery = 0;
+
return -2;
}
facet_limits_destroy(cl->facet_limits);
cl->facet_limits = facet_limits_dup(facet_limits);
- yaz_log(YLOG_LOG, "Client %s: CCL query: %s limit: %s",
- client_get_id(cl), wrbuf_cstr(w_ccl), wrbuf_cstr(w_pqf));
cn = ccl_find_str(ccl_map, wrbuf_cstr(w_ccl), &cerror, &cpos);
ccl_qual_rm(&ccl_map);
if (!cn)
wrbuf_cstr(w_ccl));
wrbuf_destroy(w_ccl);
wrbuf_destroy(w_pqf);
+
+ xfree(cl->pquery);
+ cl->pquery = 0;
+
return -1;
}
wrbuf_destroy(w_ccl);
{
if (cl->pquery)
session_log(se, YLOG_LOG, "Client %s: "
- "Re-search due query/limit change: %s to %s",
+ "Re-search due query/limit change: %s to %s",
client_get_id(cl), cl->pquery, wrbuf_cstr(w_pqf));
xfree(cl->pquery);
cl->pquery = xstrdup(wrbuf_cstr(w_pqf));
}
wrbuf_destroy(w_pqf);
- xfree(cl->cqlquery);
- cl->cqlquery = 0;
-
odr_out = odr_createmem(ODR_ENCODE);
zquery = p_query_rpn(odr_out, cl->pquery);
if (!zquery)
{
-
session_log(se, YLOG_WARN, "Invalid PQF query for Client %s: %s",
client_get_id(cl), cl->pquery);
ret_value = -1;
}
else
{
- session_log(se, YLOG_LOG, "PQF for Client %s: %s",
- client_get_id(cl), cl->pquery);
-
/* Support for PQF on SRU targets. */
if (strcmp(query_syntax, "pqf") != 0 && *sru)
{
return ret_value;
}
-int client_parse_sort(struct client *cl, struct reclist_sortparms *sp)
+int client_parse_sort(struct client *cl, struct reclist_sortparms *sp,
+ int *has_sortmap)
{
+ if (has_sortmap)
+ *has_sortmap = 0;
if (sp)
{
const char *sort_strategy_and_spec =
xfree(cl->sort_criteria);
cl->sort_criteria = xstrdup(p);
}
+ if (has_sortmap)
+ (*has_sortmap)++;
}
else {
yaz_log(YLOG_LOG, "Client %s: "
return cl->hits;
}
-int client_get_num_records(struct client *cl)
+int client_get_num_records(struct client *cl, int *filtered, int *ingest,
+ int *failed)
{
+ if (filtered)
+ *filtered = cl->filtered;
+ if (ingest)
+ *ingest = cl->ingest_failures;
+ if (failed)
+ *failed = cl->record_failures;
return cl->record_offset;
}
-int client_get_num_records_filtered(struct client *cl)
-{
- return cl->filtered;
-}
-
void client_set_diagnostic(struct client *cl, int diagnostic,
const char *message, const char *addinfo)
{
return cl->diagnostic;
}
-const char * client_get_suggestions_xml(struct client *cl, WRBUF wrbuf)
+const char *client_get_suggestions_xml(struct client *cl, WRBUF wrbuf)
{
/* int idx; */
struct suggestions *suggestions = cl->suggestions;
- if (!suggestions)
+ if (!suggestions)
return "";
if (suggestions->passthrough)
{
return wrbuf_cstr(wrbuf);
}
-
void client_set_database(struct client *cl, struct session_database *db)
{
cl->database = db;
projects / pazpar2-moved-to-github.git / blobdiff