From: Adam Dickmeiss Date: Mon, 27 Jan 2014 14:00:31 +0000 (+0100) Subject: Caches NamePlusRecord BER; works well X-Git-Tag: v5.0.13~37 X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=commitdiff_plain;h=9c32d63a1113f58ceb168e8321f1e360b2006e3b Caches NamePlusRecord BER; works well But we must be smarter WRT Z39.50 present/search cache lookup --- diff --git a/src/zoom-c.c b/src/zoom-c.c index 41d6e1c..0cc006b 100644 --- a/src/zoom-c.c +++ b/src/zoom-c.c @@ -700,7 +700,7 @@ ZOOM_resultset ZOOM_resultset_create(void) yaz_log(log_details0, "%p ZOOM_resultset_create", r); r->refcount = 1; r->size = 0; - r->odr = odr_createmem(ODR_ENCODE); + r->odr = odr_createmem(ODR_DECODE); r->piggyback = 1; r->setname = 0; r->step = 0; @@ -815,6 +815,7 @@ ZOOM_API(ZOOM_resultset) wrbuf_destroy(w); event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH); ZOOM_connection_put_event(c, event); + r->live_set = 1; return r; } else @@ -842,7 +843,6 @@ ZOOM_API(ZOOM_resultset) task->u.search.resultset = r; task->u.search.start = start; task->u.search.count = count; - task->u.search.recv_search_fired = 0; syntax = ZOOM_options_get(r->options, "preferredRecordSyntax"); task->u.search.syntax = syntax ? xstrdup(syntax) : 0; diff --git a/src/zoom-p.h b/src/zoom-p.h index 5d4ee61..37ef8a7 100644 --- a/src/zoom-p.h +++ b/src/zoom-p.h @@ -143,7 +143,7 @@ struct ZOOM_resultset_p { int num_facets; char **facets_names; WRBUF mc_key; - int live_set; + int live_set; /* 0=no hit count, 1=cached hit, 2=hits + real set */ }; struct facet_term_p { @@ -191,7 +191,6 @@ struct ZOOM_task_p { char *syntax; char *elementSetName; char *schema; - int recv_search_fired; } search; #define ZOOM_TASK_CONNECT 3 #define ZOOM_TASK_SCAN 4 diff --git a/src/zoom-record-cache.c b/src/zoom-record-cache.c index 17c4ac5..9e76d66 100644 --- a/src/zoom-record-cache.c +++ b/src/zoom-record-cache.c @@ -130,13 +130,18 @@ void ZOOM_record_cache_add(ZOOM_resultset r, Z_NamePlusRecord *npr, record_cache_add(r, npr, pos, syntax, elementSetName, schema, diag); #if HAVE_LIBMEMCACHED_MEMCACHED_H if (r->connection->mc_st && - !diag && npr->which == Z_NamePlusRecord_databaseRecord && - npr->u.databaseRecord->which == Z_External_octet) + !diag && npr->which == Z_NamePlusRecord_databaseRecord) { WRBUF k = wrbuf_alloc(); uint32_t flags = 0; memcached_return_t rc; time_t expiration = 36000; + ODR odr = odr_createmem(ODR_ENCODE); + char *rec_buf; + int rec_len; + + z_NamePlusRecord(odr, &npr, 0, 0); + rec_buf = odr_getbuf(odr, &rec_len, 0); wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key)); wrbuf_printf(k, ";%d;%s;%s;%s", pos, @@ -145,13 +150,13 @@ void ZOOM_record_cache_add(ZOOM_resultset r, Z_NamePlusRecord *npr, schema ? schema : ""); rc = memcached_set(r->connection->mc_st, wrbuf_buf(k),wrbuf_len(k), - npr->u.databaseRecord->u.octet_aligned->buf, - npr->u.databaseRecord->u.octet_aligned->len, + rec_buf, rec_len, expiration, flags); - yaz_log(YLOG_LOG, "Store record key=%s rc=%u %s", - wrbuf_cstr(k), (unsigned) rc, + yaz_log(YLOG_LOG, "Store record lkey=%s len=%d rc=%u %s", + wrbuf_cstr(k), rec_len, (unsigned) rc, memcached_last_error_message(r->connection->mc_st)); + odr_destroy(odr); wrbuf_destroy(k); } #endif @@ -183,7 +188,7 @@ ZOOM_record ZOOM_record_cache_lookup(ZOOM_resultset r, int pos, { WRBUF k = wrbuf_alloc(); size_t v_len; - char *v; + char *v_buf; uint32_t flags; memcached_return_t rc; @@ -193,33 +198,26 @@ ZOOM_record ZOOM_record_cache_lookup(ZOOM_resultset r, int pos, elementSetName ? elementSetName : "", schema ? schema : ""); - v = memcached_get(r->connection->mc_st, wrbuf_buf(k), wrbuf_len(k), - &v_len, &flags, &rc); + yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k)); + v_buf = memcached_get(r->connection->mc_st, wrbuf_buf(k), wrbuf_len(k), + &v_len, &flags, &rc); wrbuf_destroy(k); - if (v) + if (v_buf) { - yaz_log(YLOG_LOG, "Building record from memcached!! syntax=%s", - syntax); - Z_NamePlusRecord *npr = (Z_NamePlusRecord *) - odr_malloc(r->odr, sizeof(Z_NamePlusRecord)); - npr->databaseName = 0; - npr->which = Z_NamePlusRecord_databaseRecord; - npr->u.databaseRecord = (Z_External *) - odr_malloc(r->odr, sizeof(Z_External)); - npr->u.databaseRecord->descriptor = 0; - npr->u.databaseRecord->direct_reference = - syntax ? - yaz_string_to_oid_odr(yaz_oid_std(), CLASS_RECSYN, - syntax, r->odr) : 0; - npr->u.databaseRecord->indirect_reference = 0; - npr->u.databaseRecord->which = Z_External_octet; - npr->u.databaseRecord->u.octet_aligned = - odr_create_Odr_oct(r->odr, v, v_len); - free(v); - - if (v) + Z_NamePlusRecord *npr = 0; + + odr_setbuf(r->odr, v_buf, v_len, 0); + + z_NamePlusRecord(r->odr, &npr, 0, 0); + free(v_buf); + if (npr) + { + yaz_log(YLOG_LOG, "returned memcached copy"); return record_cache_add(r, npr, pos, syntax, elementSetName, schema, 0); + } + yaz_log(YLOG_WARN, "memcached_get npr failed v_len=%ld", + (long) v_len); } } #endif diff --git a/src/zoom-sru.c b/src/zoom-sru.c index 8f5feb0..bca1889 100644 --- a/src/zoom-sru.c +++ b/src/zoom-sru.c @@ -154,53 +154,52 @@ zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c) const char *schema = 0; Z_Query *z_query; Z_FacetList *facet_list = 0; + if (c->error) /* don't continue on error */ return zoom_complete; assert(c->tasks); - switch(c->tasks->which) - { - case ZOOM_TASK_SEARCH: - resultset = c->tasks->u.search.resultset; - if (!resultset->setname) - resultset->setname = xstrdup("default"); - ZOOM_options_set(resultset->options, "setname", resultset->setname); - start = &c->tasks->u.search.start; - count = &c->tasks->u.search.count; - facets = ZOOM_options_get(resultset->options, "facets"); - if (facets) - facet_list = yaz_pqf_parse_facet_list(c->odr_out, facets); - schema = c->tasks->u.search.schema; - break; + if (c->tasks->which != ZOOM_TASK_SEARCH) + return zoom_complete; + resultset = c->tasks->u.search.resultset; + if (!resultset->setname) + resultset->setname = xstrdup("default"); + ZOOM_options_set(resultset->options, "setname", resultset->setname); + start = &c->tasks->u.search.start; + count = &c->tasks->u.search.count; + facets = ZOOM_options_get(resultset->options, "facets"); + if (facets) + facet_list = yaz_pqf_parse_facet_list(c->odr_out, facets); + schema = c->tasks->u.search.schema; + + if (resultset->live_set) + { if (*start >= resultset->size) return zoom_complete; if (*start + *count > resultset->size) *count = resultset->size - *start; - - for (i = 0; i < *count; i++) + } + for (i = 0; i < *count; i++) + { + ZOOM_record rec = + ZOOM_record_cache_lookup(resultset, i + *start, + c->tasks->u.search.syntax, + c->tasks->u.search.elementSetName, + schema); + if (!rec) + break; + else { - ZOOM_record rec = - ZOOM_record_cache_lookup(resultset, i + *start, - c->tasks->u.search.syntax, - c->tasks->u.search.elementSetName, - schema); - if (!rec) - break; - else - { - ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD); - ZOOM_connection_put_event(c, event); - } + ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD); + ZOOM_connection_put_event(c, event); } - *start += i; - *count -= i; + } + *start += i; + *count -= i; - if (*count == 0) - return zoom_complete; - break; - default: + if (*count == 0 && resultset->live_set) return zoom_complete; - } + assert(resultset->query); sr = ZOOM_srw_get_pdu(c, Z_SRW_searchRetrieve_request); @@ -261,34 +260,26 @@ static zoom_ret handle_srw_response(ZOOM_connection c, int i; NMEM nmem; ZOOM_Event event; - int *start, *count; const char *syntax, *elementSetName, *schema; if (!c->tasks) return zoom_complete; - switch(c->tasks->which) - { - case ZOOM_TASK_SEARCH: - resultset = c->tasks->u.search.resultset; - start = &c->tasks->u.search.start; - count = &c->tasks->u.search.count; - syntax = c->tasks->u.search.syntax; - elementSetName = c->tasks->u.search.elementSetName; - schema = c->tasks->u.search.schema; - /* Required not for reporting client hit count multiple times into session */ - if (!c->tasks->u.search.recv_search_fired) { - yaz_log(YLOG_DEBUG, "posting ZOOM_EVENT_RECV_SEARCH"); - event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH); - ZOOM_connection_put_event(c, event); - c->tasks->u.search.recv_search_fired = 1; - } - if (res->facetList) - ZOOM_handle_facet_list(resultset, res->facetList); - break; - default: + if (c->tasks->which != ZOOM_TASK_SEARCH) return zoom_complete; + + resultset = c->tasks->u.search.resultset; + syntax = c->tasks->u.search.syntax; + elementSetName = c->tasks->u.search.elementSetName; + schema = c->tasks->u.search.schema; + + if (resultset->live_set == 0) + { + event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH); + ZOOM_connection_put_event(c, event); } + if (res->facetList) + ZOOM_handle_facet_list(resultset, res->facetList); resultset->size = 0; @@ -299,72 +290,89 @@ static zoom_ret handle_srw_response(ZOOM_connection c, if (res->num_diagnostics > 0) { + resultset->live_set = 2; set_SRU_error(c, &res->diagnostics[0]); } else { - if (res->numberOfRecords) { + if (res->numberOfRecords) + { resultset->size = *res->numberOfRecords; +#if HAVE_LIBMEMCACHED_MEMCACHED_H + if (c->mc_st && resultset->live_set == 0) + { + uint32_t flags = 0; + memcached_return_t rc; + time_t expiration = 36000; + char str[40]; + + sprintf(str, ODR_INT_PRINTF, resultset->size); + rc = memcached_set(c->mc_st, + wrbuf_buf(resultset->mc_key),wrbuf_len(resultset->mc_key), + str, strlen(str), expiration, flags); + yaz_log(YLOG_LOG, "Store SRU hit count key=%s value=%s rc=%u %s", + wrbuf_cstr(resultset->mc_key), str, (unsigned) rc, + memcached_last_error_message(c->mc_st)); + } +#endif } - if (res->suggestions) { - ZOOM_resultset_option_set(resultset, "suggestions", res->suggestions); - } - for (i = 0; inum_records; i++) + resultset->live_set = 2; + if (res->suggestions) + ZOOM_resultset_option_set(resultset, "suggestions", + res->suggestions); + for (i = 0; i < res->num_records; i++) { - int pos = *start + i; + int pos = c->tasks->u.search.start + i; Z_SRW_record *sru_rec; Z_SRW_diagnostic *diag = 0; int num_diag; - Z_NamePlusRecord *npr = (Z_NamePlusRecord *) - odr_malloc(c->odr_in, sizeof(Z_NamePlusRecord)); - /* only trust recordPosition if >= calculated position */ if (res->records[i].recordPosition && *res->records[i].recordPosition >= pos + 1) pos = *res->records[i].recordPosition - 1; - sru_rec = &res->records[i]; - - npr->databaseName = 0; - npr->which = Z_NamePlusRecord_databaseRecord; - npr->u.databaseRecord = (Z_External *) - odr_malloc(c->odr_in, sizeof(Z_External)); - npr->u.databaseRecord->descriptor = 0; - npr->u.databaseRecord->direct_reference = - odr_oiddup(c->odr_in, yaz_oid_recsyn_xml); - npr->u.databaseRecord->indirect_reference = 0; - npr->u.databaseRecord->which = Z_External_octet; - - npr->u.databaseRecord->u.octet_aligned = - odr_create_Odr_oct(c->odr_in, - sru_rec->recordData_buf, - sru_rec->recordData_len); - if (sru_rec->recordSchema - && !strcmp(sru_rec->recordSchema, - "info:srw/schema/1/diagnostics-v1.1")) + if (!ZOOM_record_cache_lookup(resultset, + pos, + syntax, elementSetName, schema)) { - sru_decode_surrogate_diagnostics(sru_rec->recordData_buf, - sru_rec->recordData_len, - &diag, &num_diag, - resultset->odr); + Z_NamePlusRecord *npr = (Z_NamePlusRecord *) + odr_malloc(c->odr_in, sizeof(Z_NamePlusRecord)); + sru_rec = &res->records[i]; + + npr->databaseName = 0; + npr->which = Z_NamePlusRecord_databaseRecord; + npr->u.databaseRecord = (Z_External *) + odr_malloc(c->odr_in, sizeof(Z_External)); + npr->u.databaseRecord->descriptor = 0; + npr->u.databaseRecord->direct_reference = + odr_oiddup(c->odr_in, yaz_oid_recsyn_xml); + npr->u.databaseRecord->indirect_reference = 0; + npr->u.databaseRecord->which = Z_External_octet; + + npr->u.databaseRecord->u.octet_aligned = + odr_create_Odr_oct(c->odr_in, + sru_rec->recordData_buf, + sru_rec->recordData_len); + if (sru_rec->recordSchema + && !strcmp(sru_rec->recordSchema, + "info:srw/schema/1/diagnostics-v1.1")) + { + sru_decode_surrogate_diagnostics(sru_rec->recordData_buf, + sru_rec->recordData_len, + &diag, &num_diag, + resultset->odr); + } + ZOOM_record_cache_add(resultset, npr, + pos, syntax, elementSetName, + schema, diag); } - ZOOM_record_cache_add(resultset, npr, pos, syntax, elementSetName, - schema, diag); } - *count -= i; - *start += i; - if (*count + *start > resultset->size) - *count = resultset->size - *start; - yaz_log(YLOG_DEBUG, "SRU result set size " ODR_INT_PRINTF " start %d count %d", resultset->size, *start, *count); - if (*count < 0) - *count = 0; nmem = odr_extract_mem(c->odr_in); nmem_transfer(odr_getmem(resultset->odr), nmem); nmem_destroy(nmem); - if (*count > 0) - return ZOOM_connection_srw_send_search(c); + return ZOOM_connection_srw_send_search(c); } return zoom_complete; } diff --git a/src/zoom-z3950.c b/src/zoom-z3950.c index 4ad090d..04f8fdf 100644 --- a/src/zoom-z3950.c +++ b/src/zoom-z3950.c @@ -651,7 +651,7 @@ zoom_ret ZOOM_connection_Z3950_send_search(ZOOM_connection c) assert(c->tasks->which == ZOOM_TASK_SEARCH); r = c->tasks->u.search.resultset; - if (r->live_set) + if (r->live_set == 2) return send_Z3950_present(c); apdu = zget_APDU(c->odr_out, Z_APDU_searchRequest); @@ -1258,11 +1258,13 @@ static void handle_Z3950_search_response(ZOOM_connection c, if (!c->tasks || c->tasks->which != ZOOM_TASK_SEARCH) return; - event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH); - ZOOM_connection_put_event(c, event); - resultset = c->tasks->u.search.resultset; + if (resultset->live_set == 0) + { + event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH); + ZOOM_connection_put_event(c, event); + } if (sr->resultSetStatus) { ZOOM_options_set_int(resultset->options, "resultSetStatus", @@ -1278,7 +1280,7 @@ static void handle_Z3950_search_response(ZOOM_connection c, handle_facet_result(c, resultset, sr->additionalSearchInfo); resultset->size = *sr->resultCount; - resultset->live_set = 1; + resultset->live_set = 2; #if HAVE_LIBMEMCACHED_MEMCACHED_H if (c->mc_st) @@ -1292,7 +1294,7 @@ static void handle_Z3950_search_response(ZOOM_connection c, rc = memcached_set(c->mc_st, wrbuf_buf(resultset->mc_key),wrbuf_len(resultset->mc_key), str, strlen(str), expiration, flags); - yaz_log(YLOG_LOG, "Store hit count key=%s value=%s rc=%u %s", + yaz_log(YLOG_LOG, "Store Z39.50 hit count key=%s value=%s rc=%u %s", wrbuf_cstr(resultset->mc_key), str, (unsigned) rc, memcached_last_error_message(c->mc_st)); }