From: Adam Dickmeiss Date: Wed, 29 Jan 2014 15:02:57 +0000 (+0100) Subject: Merge branch 'master' into yaz-728 X-Git-Tag: v5.0.13~31 X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=commitdiff_plain;h=9779edc6c34338368c661ef6105dcde1a7398b95;hp=5b0a40e3ecc7fe72f9423a7a6c5f162082f5b2eb Merge branch 'master' into yaz-728 --- diff --git a/configure.ac b/configure.ac index b30c472..60aca07 100644 --- a/configure.ac +++ b/configure.ac @@ -59,7 +59,35 @@ if test "$checkBoth" = "1"; then AC_CHECK_FUNC([accept], , [LIBS=$oldLibs]) fi AC_CHECK_FUNC([gethostbyname], ,[AC_CHECK_LIB(nsl, main, [LIBS="$LIBS -lnsl"])]) -dnl +dnl ------ libgcrypt +libgcryptpath=NONE +AC_ARG_WITH(libgcrypt, [ --with-libgcrypt=DIR use libgcrypt-config in DIR],[libgcryptpath=$withval]) +if test "$libgcryptpath" != "no"; then + if test "$libgcryptpath" = "NONE"; then + AC_PATH_PROG([libgcrypt],[libgcrypt-config],[NONE]) + else + libgcrypt=$libgcryptpath/libgcrypt-config + fi + AC_MSG_CHECKING([for libgcrypt]) + if "$libgcrypt" --version >/dev/null 2>&1; then + LIBS="$LIBS `$libgcrypt --libs`" + CFLAGS="$CFLAGS `$libgcrypt --cflags`" + libgcryptversion=`$libgcrypt --version` + AC_MSG_RESULT([$libgcryptversion]) + AC_DEFINE([HAVE_GCRYPT_H],[1],[Define to 1 if libgcrypt is present]) + else + AC_MSG_RESULT([no]) + fi +fi +dnl ------ memcached +memcached=default +AC_ARG_WITH([memcached], [ --with-memcached Memcached library], [memcached=$withval]) +if test "$memcached" != "no" -a -n "$libgcryptversion"; then + AC_CHECK_LIB([memcached],[memcached_get]) + if test "$ac_cv_lib_memcached_memcached_get" = "yes"; then + AC_CHECK_HEADERS([libmemcached/memcached.h]) + fi +fi dnl ------ GNU TLS AC_SUBST([SSL_CFLAGS]) AC_SUBST([SSL_LIBS]) diff --git a/doc/common b/doc/common index 3f5e2da..1278a48 160000 --- a/doc/common +++ b/doc/common @@ -1 +1 @@ -Subproject commit 3f5e2da375bd4a14d847b4d5f029ad76cf0c7339 +Subproject commit 1278a487dcec16a5e9406051269ef079b8cacadb diff --git a/doc/installation.xml b/doc/installation.xml index c9f216e..3e23ca4 100644 --- a/doc/installation.xml +++ b/doc/installation.xml @@ -308,6 +308,16 @@ + + --with-libgcrypt[=prefix] + + &yaz; will be linked with + Libgcrypt in the prefix if given. + If prefix is not given, the libraries exposed by the script + libgcrypt-config will be used if found. + + + diff --git a/include/yaz/wrbuf.h b/include/yaz/wrbuf.h index 4db046c..e7629d0 100644 --- a/include/yaz/wrbuf.h +++ b/include/yaz/wrbuf.h @@ -274,6 +274,27 @@ void wrbuf_iconv_json_write(WRBUF b, yaz_iconv_t cd, YAZ_EXPORT void wrbuf_iconv_json_puts(WRBUF b, yaz_iconv_t cd, const char *strz); +/** \brief writes SHA1 text to WRBUF + \param b result + \param cp char buffer + \param sz size of char buffer + \param hexit 1=hex mode; 0=binary + \returns 0 if successful + \returns -1 on error +*/ +YAZ_EXPORT +int wrbuf_sha1_write(WRBUF b, const char *cp, size_t sz, int hexit); + +/** \brief writes SHA1 text to WRBUF + \param b result + \param cp C-string + \param hexit 1=hex mode; 0=binary + \returns 0 if successful + \returns -1 on error +*/ +YAZ_EXPORT +int wrbuf_sha1_puts(WRBUF b, const char *cp, int hexit); + YAZ_END_CDECL #endif diff --git a/include/yaz/zoom.h b/include/yaz/zoom.h index 27ab47e..e460704 100644 --- a/include/yaz/zoom.h +++ b/include/yaz/zoom.h @@ -143,6 +143,7 @@ ZOOM_diag_str (int error); #define ZOOM_ERROR_ES_INVALID_ACTION 10015 #define ZOOM_ERROR_ES_INVALID_VERSION 10016 #define ZOOM_ERROR_ES_INVALID_SYNTAX 10017 +#define ZOOM_ERROR_MEMCACHED 10018 ZOOM_API(int) ZOOM_connection_last_event(ZOOM_connection cs); diff --git a/src/Makefile.am b/src/Makefile.am index ad34fa6..5ccb53f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -71,11 +71,11 @@ GEN_FILES = oid_std.c \ z-charneg.c \ ill-core.c item-req.c oclc-ill-req-ext.c -libyaz_la_SOURCES=base64.c version.c options.c log.c \ - $(GEN_FILES) \ +libyaz_la_SOURCES= $(GEN_FILES) \ + base64.c version.c options.c log.c \ cookie.c marcdisp.c \ marc_read_json.c marc_read_xml.c marc_read_iso2709.c marc_read_line.c \ - wrbuf.c oid_db.c errno.c \ + wrbuf.c wrbuf_sha1.c oid_db.c errno.c \ nmemsdup.c xmalloc.c readconf.c tpath.c nmem.c matchstr.c atoin.c \ siconv.c iconv-p.h utf8.c ucs4.c iso5428.c advancegreek.c \ odr_bool.c ber_bool.c ber_len.c ber_tag.c odr_util.c facet.c \ diff --git a/src/wrbuf_sha1.c b/src/wrbuf_sha1.c new file mode 100644 index 0000000..5627f6e --- /dev/null +++ b/src/wrbuf_sha1.c @@ -0,0 +1,65 @@ +/* This file is part of the YAZ toolkit. + * Copyright (C) Index Data + * See the file LICENSE for details. + */ + +/** + * \file wrbuf_sha1.c + * \brief Implements SHA1 creation over WRBUF + */ + +#if HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include +#if HAVE_GCRYPT_H +#include +#endif + +int wrbuf_sha1_write(WRBUF b, const char *cp, size_t sz, int hexit) +{ +#if HAVE_GCRYPT_H + gcry_error_t e; + gcry_md_hd_t hd; + const unsigned char *digest_buf; + int digest_len = gcry_md_get_algo_dlen(GCRY_MD_SHA1); + + e = gcry_md_open(&hd, GCRY_MD_SHA1, 0); + if (e) + return -1; + gcry_md_write(hd, cp, sz); + + digest_buf = gcry_md_read(hd, GCRY_MD_SHA1); + if (hexit) + { + int i; + for (i = 0; i < digest_len; i++) + wrbuf_printf(b, "%02x", digest_buf[i]); + } + else + wrbuf_write(b, (const char *) digest_buf, digest_len); + gcry_md_close(hd); + return 0; +#else + return -1; +#endif +} + +int wrbuf_sha1_puts(WRBUF b, const char *cp, int hexit) +{ + return wrbuf_sha1_write(b, cp, strlen(cp), hexit); +} + +/* + * Local variables: + * c-basic-offset: 4 + * c-file-style: "Stroustrup" + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ diff --git a/src/zoom-c.c b/src/zoom-c.c index c3d7f0c..b1874ea 100644 --- a/src/zoom-c.c +++ b/src/zoom-c.c @@ -96,8 +96,6 @@ int ZOOM_uri_to_code(const char *uri) return code; } - - void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo) { ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0); @@ -136,9 +134,6 @@ void ZOOM_connection_show_task(ZOOM_task task) case ZOOM_TASK_SEARCH: yaz_log(YLOG_LOG, "search p=%p", task); break; - case ZOOM_TASK_RETRIEVE: - yaz_log(YLOG_LOG, "retrieve p=%p", task); - break; case ZOOM_TASK_CONNECT: yaz_log(YLOG_LOG, "connect p=%p", task); break; @@ -201,12 +196,6 @@ void ZOOM_connection_remove_task(ZOOM_connection c) xfree(task->u.search.elementSetName); xfree(task->u.search.schema); break; - case ZOOM_TASK_RETRIEVE: - resultset_destroy(task->u.retrieve.resultset); - xfree(task->u.retrieve.syntax); - xfree(task->u.retrieve.elementSetName); - xfree(task->u.retrieve.schema); - break; case ZOOM_TASK_CONNECT: break; case ZOOM_TASK_SCAN: @@ -308,6 +297,10 @@ ZOOM_API(ZOOM_connection) c->no_redirects = 0; c->cookies = 0; c->saveAPDU_wrbuf = 0; + +#if HAVE_LIBMEMCACHED_MEMCACHED_H + c->mc_st = 0; +#endif return c; } @@ -552,6 +545,29 @@ ZOOM_API(void) yaz_cookies_destroy(c->cookies); c->cookies = yaz_cookies_create(); +#if HAVE_LIBMEMCACHED_MEMCACHED_H + if (c->mc_st) + { + memcached_free(c->mc_st); + c->mc_st = 0; + } +#endif + val = ZOOM_options_get(c->options, "memcached"); + if (val && *val) + { +#if HAVE_LIBMEMCACHED_MEMCACHED_H + c->mc_st = memcached(val, strlen(val)); + if (!c->mc_st) + { + ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, val); + return; + } +#else + ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled"); + return; +#endif + } + if (c->sru_mode == zoom_sru_error) { ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val); @@ -596,6 +612,11 @@ ZOOM_API(void) if (!c) return; yaz_log(c->log_api, "%p ZOOM_connection_destroy", c); + +#if HAVE_LIBMEMCACHED_MEMCACHED_H + if (c->mc_st) + memcached_free(c->mc_st); +#endif if (c->cs) cs_close(c->cs); @@ -677,7 +698,7 @@ ZOOM_resultset ZOOM_resultset_create(void) yaz_log(log_details0, "%p ZOOM_resultset_create", r); r->refcount = 1; r->size = 0; - r->odr = odr_createmem(ODR_ENCODE); + r->odr = odr_createmem(ODR_DECODE); r->piggyback = 1; r->setname = 0; r->step = 0; @@ -700,6 +721,8 @@ ZOOM_resultset ZOOM_resultset_create(void) } #endif resultset_use(1); + r->mc_key = 0; + r->live_set = 0; return r; } @@ -720,14 +743,14 @@ ZOOM_API(ZOOM_resultset) ZOOM_connection_search(ZOOM_connection c, ZOOM_query q) { ZOOM_resultset r = ZOOM_resultset_create(); - ZOOM_task task; const char *cp; + ZOOM_task task; int start, count; const char *syntax, *elementSetName, *schema; - yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q); r->r_sort_spec = ZOOM_query_get_sortspec(q); r->query = q; + ZOOM_query_addref(q); r->options = ZOOM_options_create_with_parent(c->options); @@ -750,6 +773,25 @@ ZOOM_API(ZOOM_resultset) r->connection = c; r->next = c->resultsets; c->resultsets = r; + +#if HAVE_LIBMEMCACHED_MEMCACHED_H + r->mc_key = wrbuf_alloc(); + wrbuf_puts(r->mc_key, "0;"); + wrbuf_puts(r->mc_key, c->host_port); + wrbuf_puts(r->mc_key, ";"); + if (c->user) + wrbuf_puts(r->mc_key, c->user); + wrbuf_puts(r->mc_key, ";"); + if (c->group) + wrbuf_puts(r->mc_key, c->group); + wrbuf_puts(r->mc_key, ";"); + if (c->password) + wrbuf_sha1_puts(r->mc_key, c->password, 1); + wrbuf_puts(r->mc_key, ";"); + wrbuf_sha1_puts(r->mc_key, ZOOM_query_get_query_string(q), 1); + wrbuf_puts(r->mc_key, ";"); +#endif + if (c->host_port && c->proto == PROTO_HTTP) { if (!c->cs) @@ -768,7 +810,6 @@ ZOOM_API(ZOOM_resultset) task->u.search.resultset = r; task->u.search.start = start; task->u.search.count = count; - task->u.search.recv_search_fired = 0; syntax = ZOOM_options_get(r->options, "preferredRecordSyntax"); task->u.search.syntax = syntax ? xstrdup(syntax) : 0; @@ -780,8 +821,6 @@ ZOOM_API(ZOOM_resultset) ZOOM_resultset_addref(r); - ZOOM_query_addref(q); - if (!c->async) { while (ZOOM_event(1, &c)) @@ -874,6 +913,7 @@ static void resultset_destroy(ZOOM_resultset r) #if SHPTR YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy); #endif + wrbuf_destroy(r->mc_key); resultset_use(-1); xfree(r); } @@ -934,19 +974,19 @@ static void ZOOM_resultset_retrieve(ZOOM_resultset r, c->reconnect_ok = 1; } } - task = ZOOM_connection_add_task(c, ZOOM_TASK_RETRIEVE); - task->u.retrieve.resultset = r; - task->u.retrieve.start = start; - task->u.retrieve.count = count; + task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH); + task->u.search.resultset = r; + task->u.search.start = start; + task->u.search.count = count; syntax = ZOOM_options_get(r->options, "preferredRecordSyntax"); - task->u.retrieve.syntax = syntax ? xstrdup(syntax) : 0; + task->u.search.syntax = syntax ? xstrdup(syntax) : 0; elementSetName = ZOOM_options_get(r->options, "elementSetName"); - task->u.retrieve.elementSetName = elementSetName + task->u.search.elementSetName = elementSetName ? xstrdup(elementSetName) : 0; cp = ZOOM_options_get(r->options, "schema"); - task->u.retrieve.schema = cp ? xstrdup(cp) : 0; + task->u.search.schema = cp ? xstrdup(cp) : 0; ZOOM_resultset_addref(r); @@ -977,30 +1017,30 @@ ZOOM_API(void) } ZOOM_API(size_t) - ZOOM_resultset_facets_size(ZOOM_resultset r) { + ZOOM_resultset_facets_size(ZOOM_resultset r) +{ return r->num_facets; } ZOOM_API(ZOOM_facet_field) - ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name) { + ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name) +{ int num = r->num_facets; ZOOM_facet_field *facets = r->facets; - int index; - for (index = 0; index < num; index++) { - if (!strcmp(facets[index]->facet_name, name)) { - return facets[index]; - } - } + int i; + for (i = 0; i < num; i++) + if (!strcmp(facets[i]->facet_name, name)) + return facets[i]; return 0; } ZOOM_API(ZOOM_facet_field) - ZOOM_resultset_get_facet_field_by_index(ZOOM_resultset r, int index) { + ZOOM_resultset_get_facet_field_by_index(ZOOM_resultset r, int idx) +{ int num = r->num_facets; ZOOM_facet_field *facets = r->facets; - if (index >= 0 && index < num) { - return facets[index]; - } + if (idx >= 0 && idx < num) + return facets[idx]; return 0; } @@ -1029,7 +1069,8 @@ ZOOM_API(size_t) } ZOOM_API(const char*) - ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq) { + ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq) +{ *freq = field->facet_terms[idx].frequency; return field->facet_terms[idx].term; } @@ -1442,13 +1483,7 @@ ZOOM_API(int) if (c->proto == PROTO_HTTP) ret = ZOOM_connection_srw_send_search(c); else - ret = ZOOM_connection_Z3950_send_search(c); - break; - case ZOOM_TASK_RETRIEVE: - if (c->proto == PROTO_HTTP) - ret = ZOOM_connection_srw_send_search(c); - else - ret = send_Z3950_present(c); + ret = ZOOM_connection_Z3950_search(c); break; case ZOOM_TASK_CONNECT: ret = do_connect(c); @@ -1898,6 +1933,8 @@ ZOOM_API(const char *) return "Extended Service. invalid version"; case ZOOM_ERROR_ES_INVALID_SYNTAX: return "Extended Service. invalid syntax"; + case ZOOM_ERROR_MEMCACHED: + return "Memcached"; default: return diagbib1_str(error); } diff --git a/src/zoom-p.h b/src/zoom-p.h index 77f39e9..2a64c5f 100644 --- a/src/zoom-p.h +++ b/src/zoom-p.h @@ -36,6 +36,9 @@ #include #include #include +#if HAVE_LIBMEMCACHED_MEMCACHED_H +#include +#endif #define SHPTR 1 @@ -108,6 +111,9 @@ struct ZOOM_connection_p { int log_details; int log_api; WRBUF saveAPDU_wrbuf; +#if HAVE_LIBMEMCACHED_MEMCACHED_H + memcached_st *mc_st; +#endif }; typedef struct ZOOM_record_cache_p *ZOOM_record_cache; @@ -136,6 +142,8 @@ struct ZOOM_resultset_p { ZOOM_facet_field *facets; int num_facets; char **facets_names; + WRBUF mc_key; + int live_set; /* 0=no hit count, 1=cached hit, 2=hits + real set */ }; struct facet_term_p { @@ -183,17 +191,7 @@ struct ZOOM_task_p { char *syntax; char *elementSetName; char *schema; - int recv_search_fired; } search; -#define ZOOM_TASK_RETRIEVE 2 - struct { - int start; - ZOOM_resultset resultset; - int count; - char *syntax; - char *elementSetName; - char *schema; - } retrieve; #define ZOOM_TASK_CONNECT 3 #define ZOOM_TASK_SCAN 4 struct { @@ -228,8 +226,7 @@ void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo); ZOOM_Event ZOOM_Event_create(int kind); void ZOOM_connection_put_event(ZOOM_connection c, ZOOM_Event event); -zoom_ret ZOOM_connection_Z3950_send_search(ZOOM_connection c); -zoom_ret send_Z3950_present(ZOOM_connection c); +zoom_ret ZOOM_connection_Z3950_search(ZOOM_connection c); zoom_ret ZOOM_connection_Z3950_send_scan(ZOOM_connection c); zoom_ret ZOOM_send_buf(ZOOM_connection c); zoom_ret send_Z3950_sort(ZOOM_connection c, ZOOM_resultset resultset); diff --git a/src/zoom-record-cache.c b/src/zoom-record-cache.c index 84f4dd2..9e76d66 100644 --- a/src/zoom-record-cache.c +++ b/src/zoom-record-cache.c @@ -56,11 +56,13 @@ static size_t record_hash(int pos) return pos % RECORD_HASH_SIZE; } -void ZOOM_record_cache_add(ZOOM_resultset r, Z_NamePlusRecord *npr, - int pos, - const char *syntax, const char *elementSetName, - const char *schema, - Z_SRW_diagnostic *diag) +static ZOOM_record record_cache_add(ZOOM_resultset r, + Z_NamePlusRecord *npr, + int pos, + const char *syntax, + const char *elementSetName, + const char *schema, + Z_SRW_diagnostic *diag) { ZOOM_record_cache rc = 0; @@ -94,7 +96,9 @@ void ZOOM_record_cache_add(ZOOM_resultset r, Z_NamePlusRecord *npr, rc->pos = pos; rc->next = r->record_hash[record_hash(pos)]; r->record_hash[record_hash(pos)] = rc; + } + rc->rec.npr = npr; rc->rec.schema = odr_strdup_null(r->odr, schema); rc->rec.diag_set = 0; @@ -114,8 +118,51 @@ void ZOOM_record_cache_add(ZOOM_resultset r, Z_NamePlusRecord *npr, rc->rec.diag_message = odr_strdup_null(r->odr, diag->message); rc->rec.diag_details = odr_strdup_null(r->odr, diag->details); } + return &rc->rec; +} + +void ZOOM_record_cache_add(ZOOM_resultset r, Z_NamePlusRecord *npr, + int pos, + const char *syntax, const char *elementSetName, + const char *schema, + Z_SRW_diagnostic *diag) +{ + record_cache_add(r, npr, pos, syntax, elementSetName, schema, diag); +#if HAVE_LIBMEMCACHED_MEMCACHED_H + if (r->connection->mc_st && + !diag && npr->which == Z_NamePlusRecord_databaseRecord) + { + WRBUF k = wrbuf_alloc(); + uint32_t flags = 0; + memcached_return_t rc; + time_t expiration = 36000; + ODR odr = odr_createmem(ODR_ENCODE); + char *rec_buf; + int rec_len; + + z_NamePlusRecord(odr, &npr, 0, 0); + rec_buf = odr_getbuf(odr, &rec_len, 0); + + wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key)); + wrbuf_printf(k, ";%d;%s;%s;%s", pos, + syntax ? syntax : "", + elementSetName ? elementSetName : "", + schema ? schema : ""); + rc = memcached_set(r->connection->mc_st, + wrbuf_buf(k),wrbuf_len(k), + rec_buf, rec_len, + expiration, flags); + + yaz_log(YLOG_LOG, "Store record lkey=%s len=%d rc=%u %s", + wrbuf_cstr(k), rec_len, (unsigned) rc, + memcached_last_error_message(r->connection->mc_st)); + odr_destroy(odr); + wrbuf_destroy(k); + } +#endif } + ZOOM_record ZOOM_record_cache_lookup(ZOOM_resultset r, int pos, const char *syntax, const char *elementSetName, @@ -136,6 +183,44 @@ ZOOM_record ZOOM_record_cache_lookup(ZOOM_resultset r, int pos, return &rc->rec; } } +#if HAVE_LIBMEMCACHED_MEMCACHED_H + if (r->connection && r->connection->mc_st) + { + WRBUF k = wrbuf_alloc(); + size_t v_len; + char *v_buf; + uint32_t flags; + memcached_return_t rc; + + wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key)); + wrbuf_printf(k, ";%d;%s;%s;%s", pos, + syntax ? syntax : "", + elementSetName ? elementSetName : "", + schema ? schema : ""); + + yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k)); + v_buf = memcached_get(r->connection->mc_st, wrbuf_buf(k), wrbuf_len(k), + &v_len, &flags, &rc); + wrbuf_destroy(k); + if (v_buf) + { + Z_NamePlusRecord *npr = 0; + + odr_setbuf(r->odr, v_buf, v_len, 0); + + z_NamePlusRecord(r->odr, &npr, 0, 0); + free(v_buf); + if (npr) + { + yaz_log(YLOG_LOG, "returned memcached copy"); + return record_cache_add(r, npr, pos, syntax, elementSetName, + schema, 0); + } + yaz_log(YLOG_WARN, "memcached_get npr failed v_len=%ld", + (long) v_len); + } + } +#endif return 0; } diff --git a/src/zoom-sru.c b/src/zoom-sru.c index 4e8807b..ad241cd 100644 --- a/src/zoom-sru.c +++ b/src/zoom-sru.c @@ -154,59 +154,77 @@ zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c) const char *schema = 0; Z_Query *z_query; Z_FacetList *facet_list = 0; + if (c->error) /* don't continue on error */ return zoom_complete; assert(c->tasks); - switch(c->tasks->which) + if (c->tasks->which != ZOOM_TASK_SEARCH) + return zoom_complete; + + resultset = c->tasks->u.search.resultset; + +#if HAVE_LIBMEMCACHED_MEMCACHED_H + /* TODO: add sorting */ + if (c->mc_st && resultset->live_set == 0) { - case ZOOM_TASK_SEARCH: - resultset = c->tasks->u.search.resultset; - if (!resultset->setname) - resultset->setname = xstrdup("default"); - ZOOM_options_set(resultset->options, "setname", resultset->setname); - start = &c->tasks->u.search.start; - count = &c->tasks->u.search.count; - facets = ZOOM_options_get(resultset->options, "facets"); - if (facets) - facet_list = yaz_pqf_parse_facet_list(c->odr_out, facets); - schema = c->tasks->u.search.schema; - break; - case ZOOM_TASK_RETRIEVE: - resultset = c->tasks->u.retrieve.resultset; - schema = c->tasks->u.retrieve.schema; - - start = &c->tasks->u.retrieve.start; - count = &c->tasks->u.retrieve.count; + size_t v_len; + uint32_t flags; + memcached_return_t rc; + char *v = memcached_get(c->mc_st, wrbuf_buf(resultset->mc_key), + wrbuf_len(resultset->mc_key), + &v_len, &flags, &rc); + if (v) + { + ZOOM_Event event; + WRBUF w = wrbuf_alloc(); + + wrbuf_write(w, v, v_len); + free(v); + resultset->size = odr_atoi(wrbuf_cstr(w)); + yaz_log(YLOG_LOG, "For key %s got value %s", + wrbuf_cstr(resultset->mc_key), wrbuf_cstr(w)); + + wrbuf_destroy(w); + event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH); + ZOOM_connection_put_event(c, event); + resultset->live_set = 1; + } + } +#endif + if (!resultset->setname) + resultset->setname = xstrdup("default"); + ZOOM_options_set(resultset->options, "setname", resultset->setname); + start = &c->tasks->u.search.start; + count = &c->tasks->u.search.count; + facets = ZOOM_options_get(resultset->options, "facets"); + if (facets) + facet_list = yaz_pqf_parse_facet_list(c->odr_out, facets); + schema = c->tasks->u.search.schema; + + if (resultset->live_set) + { if (*start >= resultset->size) return zoom_complete; if (*start + *count > resultset->size) *count = resultset->size - *start; + } + for (i = 0; i < *count; i++) + { + ZOOM_record rec = + ZOOM_record_cache_lookup(resultset, i + *start, + c->tasks->u.search.syntax, + c->tasks->u.search.elementSetName, + schema); + if (!rec) + break; + } + *start += i; + *count -= i; - for (i = 0; i < *count; i++) - { - ZOOM_record rec = - ZOOM_record_cache_lookup(resultset, i + *start, - c->tasks->u.retrieve.syntax, - c->tasks->u.retrieve.elementSetName, - schema); - if (!rec) - break; - else - { - ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD); - ZOOM_connection_put_event(c, event); - } - } - *start += i; - *count -= i; - - if (*count == 0) - return zoom_complete; - break; - default: + if (*count == 0 && resultset->live_set) return zoom_complete; - } + assert(resultset->query); sr = ZOOM_srw_get_pdu(c, Z_SRW_searchRetrieve_request); @@ -267,42 +285,26 @@ static zoom_ret handle_srw_response(ZOOM_connection c, int i; NMEM nmem; ZOOM_Event event; - int *start, *count; const char *syntax, *elementSetName, *schema; if (!c->tasks) return zoom_complete; - switch(c->tasks->which) - { - case ZOOM_TASK_SEARCH: - resultset = c->tasks->u.search.resultset; - start = &c->tasks->u.search.start; - count = &c->tasks->u.search.count; - syntax = c->tasks->u.search.syntax; - elementSetName = c->tasks->u.search.elementSetName; - schema = c->tasks->u.search.schema; - /* Required not for reporting client hit count multiple times into session */ - if (!c->tasks->u.search.recv_search_fired) { - yaz_log(YLOG_DEBUG, "posting ZOOM_EVENT_RECV_SEARCH"); - event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH); - ZOOM_connection_put_event(c, event); - c->tasks->u.search.recv_search_fired = 1; - } - if (res->facetList) - ZOOM_handle_facet_list(resultset, res->facetList); - break; - case ZOOM_TASK_RETRIEVE: - resultset = c->tasks->u.retrieve.resultset; - start = &c->tasks->u.retrieve.start; - count = &c->tasks->u.retrieve.count; - syntax = c->tasks->u.retrieve.syntax; - elementSetName = c->tasks->u.retrieve.elementSetName; - schema = c->tasks->u.retrieve.schema; - break; - default: + if (c->tasks->which != ZOOM_TASK_SEARCH) return zoom_complete; + + resultset = c->tasks->u.search.resultset; + syntax = c->tasks->u.search.syntax; + elementSetName = c->tasks->u.search.elementSetName; + schema = c->tasks->u.search.schema; + + if (resultset->live_set == 0) + { + event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH); + ZOOM_connection_put_event(c, event); } + if (res->facetList) + ZOOM_handle_facet_list(resultset, res->facetList); resultset->size = 0; @@ -313,72 +315,89 @@ static zoom_ret handle_srw_response(ZOOM_connection c, if (res->num_diagnostics > 0) { + resultset->live_set = 2; set_SRU_error(c, &res->diagnostics[0]); } else { - if (res->numberOfRecords) { + if (res->numberOfRecords) + { resultset->size = *res->numberOfRecords; +#if HAVE_LIBMEMCACHED_MEMCACHED_H + if (c->mc_st && resultset->live_set == 0) + { + uint32_t flags = 0; + memcached_return_t rc; + time_t expiration = 36000; + char str[40]; + + sprintf(str, ODR_INT_PRINTF, resultset->size); + rc = memcached_set(c->mc_st, + wrbuf_buf(resultset->mc_key),wrbuf_len(resultset->mc_key), + str, strlen(str), expiration, flags); + yaz_log(YLOG_LOG, "Store SRU hit count key=%s value=%s rc=%u %s", + wrbuf_cstr(resultset->mc_key), str, (unsigned) rc, + memcached_last_error_message(c->mc_st)); + } +#endif } - if (res->suggestions) { - ZOOM_resultset_option_set(resultset, "suggestions", res->suggestions); - } - for (i = 0; inum_records; i++) + resultset->live_set = 2; + if (res->suggestions) + ZOOM_resultset_option_set(resultset, "suggestions", + res->suggestions); + for (i = 0; i < res->num_records; i++) { - int pos = *start + i; + int pos = c->tasks->u.search.start + i; Z_SRW_record *sru_rec; Z_SRW_diagnostic *diag = 0; int num_diag; - Z_NamePlusRecord *npr = (Z_NamePlusRecord *) - odr_malloc(c->odr_in, sizeof(Z_NamePlusRecord)); - /* only trust recordPosition if >= calculated position */ if (res->records[i].recordPosition && *res->records[i].recordPosition >= pos + 1) pos = *res->records[i].recordPosition - 1; - sru_rec = &res->records[i]; - - npr->databaseName = 0; - npr->which = Z_NamePlusRecord_databaseRecord; - npr->u.databaseRecord = (Z_External *) - odr_malloc(c->odr_in, sizeof(Z_External)); - npr->u.databaseRecord->descriptor = 0; - npr->u.databaseRecord->direct_reference = - odr_oiddup(c->odr_in, yaz_oid_recsyn_xml); - npr->u.databaseRecord->indirect_reference = 0; - npr->u.databaseRecord->which = Z_External_octet; - - npr->u.databaseRecord->u.octet_aligned = - odr_create_Odr_oct(c->odr_in, - sru_rec->recordData_buf, - sru_rec->recordData_len); - if (sru_rec->recordSchema - && !strcmp(sru_rec->recordSchema, - "info:srw/schema/1/diagnostics-v1.1")) + if (!ZOOM_record_cache_lookup(resultset, + pos, + syntax, elementSetName, schema)) { - sru_decode_surrogate_diagnostics(sru_rec->recordData_buf, - sru_rec->recordData_len, - &diag, &num_diag, - resultset->odr); + Z_NamePlusRecord *npr = (Z_NamePlusRecord *) + odr_malloc(c->odr_in, sizeof(Z_NamePlusRecord)); + sru_rec = &res->records[i]; + + npr->databaseName = 0; + npr->which = Z_NamePlusRecord_databaseRecord; + npr->u.databaseRecord = (Z_External *) + odr_malloc(c->odr_in, sizeof(Z_External)); + npr->u.databaseRecord->descriptor = 0; + npr->u.databaseRecord->direct_reference = + odr_oiddup(c->odr_in, yaz_oid_recsyn_xml); + npr->u.databaseRecord->indirect_reference = 0; + npr->u.databaseRecord->which = Z_External_octet; + + npr->u.databaseRecord->u.octet_aligned = + odr_create_Odr_oct(c->odr_in, + sru_rec->recordData_buf, + sru_rec->recordData_len); + if (sru_rec->recordSchema + && !strcmp(sru_rec->recordSchema, + "info:srw/schema/1/diagnostics-v1.1")) + { + sru_decode_surrogate_diagnostics(sru_rec->recordData_buf, + sru_rec->recordData_len, + &diag, &num_diag, + resultset->odr); + } + ZOOM_record_cache_add(resultset, npr, + pos, syntax, elementSetName, + schema, diag); } - ZOOM_record_cache_add(resultset, npr, pos, syntax, elementSetName, - schema, diag); } - *count -= i; - *start += i; - if (*count + *start > resultset->size) - *count = resultset->size - *start; - yaz_log(YLOG_DEBUG, "SRU result set size " ODR_INT_PRINTF " start %d count %d", resultset->size, *start, *count); - if (*count < 0) - *count = 0; nmem = odr_extract_mem(c->odr_in); nmem_transfer(odr_getmem(resultset->odr), nmem); nmem_destroy(nmem); - if (*count > 0) - return ZOOM_connection_srw_send_search(c); + return ZOOM_connection_srw_send_search(c); } return zoom_complete; } diff --git a/src/zoom-z3950.c b/src/zoom-z3950.c index 80ed718..556e258 100644 --- a/src/zoom-z3950.c +++ b/src/zoom-z3950.c @@ -36,7 +36,8 @@ * if it could cause failure when a lookup fails, but that's hard. */ static Odr_oid *zoom_yaz_str_to_z3950oid(ZOOM_connection c, - oid_class oid_class, const char *str) { + oid_class oid_class, const char *str) +{ Odr_oid *res = yaz_string_to_oid_odr(yaz_oid_std(), oid_class, str, c->odr_out); if (res == 0) @@ -465,7 +466,7 @@ static Z_APDU *create_update_package(ZOOM_package p) static void otherInfo_attach(ZOOM_connection c, Z_APDU *a, ODR out) { int i; - for (i = 0; i<200; i++) + for (i = 0; i < 200; i++) { size_t len; Odr_oid *oid; @@ -633,14 +634,14 @@ zoom_ret ZOOM_connection_Z3950_send_init(ZOOM_connection c) return send_APDU(c, apdu); } -zoom_ret ZOOM_connection_Z3950_send_search(ZOOM_connection c) +static zoom_ret Z3950_send_search(ZOOM_connection c) { ZOOM_resultset r; int lslb, ssub, mspn; const char *syntax; const char *schema; - Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_searchRequest); - Z_SearchRequest *search_req = apdu->u.searchRequest; + Z_APDU *apdu; + Z_SearchRequest *search_req; const char *elementSetName; const char *smallSetElementSetName; const char *mediumSetElementSetName; @@ -648,13 +649,14 @@ zoom_ret ZOOM_connection_Z3950_send_search(ZOOM_connection c) assert(c->tasks); assert(c->tasks->which == ZOOM_TASK_SEARCH); - r = c->tasks->u.search.resultset; + apdu = zget_APDU(c->odr_out, Z_APDU_searchRequest); + search_req = apdu->u.searchRequest; + yaz_log(c->log_details, "%p ZOOM_connection_send_search set=%p", c, r); - elementSetName = - ZOOM_options_get(r->options, "elementSetName"); + elementSetName = c->tasks->u.search.elementSetName; smallSetElementSetName = ZOOM_options_get(r->options, "smallSetElementSetName"); mediumSetElementSetName = @@ -984,16 +986,21 @@ static int es_response_taskpackage_update(ZOOM_connection c, if (utp && utp->targetPart) { Z_IUTargetPart *targetPart = utp->targetPart; - switch ( *targetPart->updateStatus ) { + switch (*targetPart->updateStatus) + { case Z_IUTargetPart_success: - ZOOM_options_set(c->tasks->u.package->options,"updateStatus", "success"); + ZOOM_options_set(c->tasks->u.package->options, + "updateStatus", "success"); break; case Z_IUTargetPart_partial: - ZOOM_options_set(c->tasks->u.package->options,"updateStatus", "partial"); + ZOOM_options_set(c->tasks->u.package->options, + "updateStatus", "partial"); break; case Z_IUTargetPart_failure: - ZOOM_options_set(c->tasks->u.package->options,"updateStatus", "failure"); - if (targetPart->globalDiagnostics && targetPart->num_globalDiagnostics > 0) + ZOOM_options_set(c->tasks->u.package->options, + "updateStatus", "failure"); + if (targetPart->globalDiagnostics && + targetPart->num_globalDiagnostics > 0) response_diag(c, targetPart->globalDiagnostics[0]); break; } @@ -1009,8 +1016,8 @@ static int es_response_taskpackage(ZOOM_connection c, if (id) ZOOM_options_setl(c->tasks->u.package->options, "targetReference", (char*) id->buf, id->len); - - switch ( *taskPackage->taskStatus ) { + switch (*taskPackage->taskStatus) + { case Z_TaskPackage_pending: ZOOM_options_set(c->tasks->u.package->options,"taskStatus", "pending"); break; @@ -1022,25 +1029,27 @@ static int es_response_taskpackage(ZOOM_connection c, break; case Z_TaskPackage_aborted: ZOOM_options_set(c->tasks->u.package->options,"taskStatus", "aborted"); - if ( taskPackage->num_packageDiagnostics && taskPackage->packageDiagnostics ) + if (taskPackage->num_packageDiagnostics && + taskPackage->packageDiagnostics ) response_diag(c, taskPackage->packageDiagnostics[0]); break; } /* NOTE: Only Update implemented, no others. */ - if ( taskPackage->taskSpecificParameters->which == Z_External_update ) + if (taskPackage->taskSpecificParameters->which == Z_External_update) { - Z_IUUpdateTaskPackage *utp = taskPackage->taskSpecificParameters->u.update->u.taskPackage; + Z_IUUpdateTaskPackage *utp = + taskPackage->taskSpecificParameters->u.update->u.taskPackage; es_response_taskpackage_update(c, utp); } return 1; } - -static int handle_Z3950_es_response(ZOOM_connection c, - Z_ExtendedServicesResponse *res) +static void handle_Z3950_es_response(ZOOM_connection c, + Z_ExtendedServicesResponse *res) { - if (!c->tasks || c->tasks->which != ZOOM_TASK_PACKAGE) - return 0; + if (!c->tasks) + return; + assert(c->tasks->which == ZOOM_TASK_PACKAGE); switch (*res->operationStatus) { case Z_ExtendedServicesResponse_done: @@ -1068,14 +1077,14 @@ static int handle_Z3950_es_response(ZOOM_connection c, ZOOM_options_setl(c->tasks->u.package->options, "xmlUpdateDoc", (char*) doc->buf, doc->len); } - return 1; } -static char *get_term_cstr(ODR odr, Z_Term *term) { - - switch (term->which) { +static char *get_term_cstr(ODR odr, Z_Term *term) +{ + switch (term->which) + { case Z_Term_general: - return odr_strdupn(odr, (const char *) term->u.general->buf, (size_t) term->u.general->len); + return odr_strdupn(odr, term->u.general->buf, term->u.general->len); break; case Z_Term_characterString: return odr_strdup(odr, term->u.characterString); @@ -1083,22 +1092,27 @@ static char *get_term_cstr(ODR odr, Z_Term *term) { return 0; } -static ZOOM_facet_field get_zoom_facet_field(ODR odr, Z_FacetField *facet) { - int term_index; +static ZOOM_facet_field get_zoom_facet_field(ODR odr, Z_FacetField *facet) +{ + int i; struct yaz_facet_attr attr_values; ZOOM_facet_field facet_field = odr_malloc(odr, sizeof(*facet_field)); yaz_facet_attr_init(&attr_values); yaz_facet_attr_get_z_attributes(facet->attributes, &attr_values); facet_field->facet_name = odr_strdup(odr, attr_values.useattr); facet_field->num_terms = facet->num_terms; - yaz_log(YLOG_DEBUG, "ZOOM_facet_field %s %d terms %d", attr_values.useattr, attr_values.limit, facet->num_terms); - facet_field->facet_terms = odr_malloc(odr, facet_field->num_terms * sizeof(*facet_field->facet_terms)); - for (term_index = 0 ; term_index < facet->num_terms; term_index++) { - Z_FacetTerm *facetTerm = facet->terms[term_index]; - facet_field->facet_terms[term_index].frequency = *facetTerm->count; - facet_field->facet_terms[term_index].term = get_term_cstr(odr, facetTerm->term); + yaz_log(YLOG_DEBUG, "ZOOM_facet_field %s %d terms %d", + attr_values.useattr, attr_values.limit, facet->num_terms); + facet_field->facet_terms = + odr_malloc(odr, facet->num_terms * sizeof(*facet_field->facet_terms)); + for (i = 0 ; i < facet->num_terms; i++) + { + Z_FacetTerm *facetTerm = facet->terms[i]; + facet_field->facet_terms[i].frequency = *facetTerm->count; + facet_field->facet_terms[i].term = get_term_cstr(odr, facetTerm->term); yaz_log(YLOG_DEBUG, " term[%d] %s %d", - term_index, facet_field->facet_terms[term_index].term, facet_field->facet_terms[term_index].frequency); + i, facet_field->facet_terms[i].term, + facet_field->facet_terms[i].frequency); } return facet_field; } @@ -1109,8 +1123,8 @@ void ZOOM_handle_facet_list(ZOOM_resultset r, Z_FacetList *fl) int j; r->num_facets = fl->num; yaz_log(YLOG_DEBUG, "Facets found: %d", fl->num); - r->facets = odr_malloc(r->odr, r->num_facets * sizeof(*r->facets)); - r->facets_names = odr_malloc(r->odr, r->num_facets * sizeof(*r->facets_names)); + r->facets = odr_malloc(r->odr, fl->num * sizeof(*r->facets)); + r->facets_names = odr_malloc(r->odr, fl->num * sizeof(*r->facets_names)); for (j = 0; j < fl->num; j++) { r->facets[j] = get_zoom_facet_field(r->odr, fl->elements[j]); @@ -1144,8 +1158,7 @@ static void handle_queryExpressionTerm(ZOOM_options opt, const char *name, { case Z_Term_general: ZOOM_options_setl(opt, name, - (const char *)(term->u.general->buf), - term->u.general->len); + term->u.general->buf, term->u.general->len); break; case Z_Term_characterString: ZOOM_options_set(opt, name, term->u.characterString); @@ -1251,13 +1264,15 @@ static void handle_Z3950_search_response(ZOOM_connection c, ZOOM_Event event; if (!c->tasks || c->tasks->which != ZOOM_TASK_SEARCH) - return ; - - event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH); - ZOOM_connection_put_event(c, event); + return; resultset = c->tasks->u.search.resultset; + if (resultset->live_set == 0) + { + event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH); + ZOOM_connection_put_event(c, event); + } if (sr->resultSetStatus) { ZOOM_options_set_int(resultset->options, "resultSetStatus", @@ -1273,6 +1288,25 @@ static void handle_Z3950_search_response(ZOOM_connection c, handle_facet_result(c, resultset, sr->additionalSearchInfo); resultset->size = *sr->resultCount; + resultset->live_set = 2; + +#if HAVE_LIBMEMCACHED_MEMCACHED_H + if (c->mc_st) + { + uint32_t flags = 0; + memcached_return_t rc; + time_t expiration = 36000; + char str[40]; + + sprintf(str, ODR_INT_PRINTF, *sr->resultCount); + rc = memcached_set(c->mc_st, + wrbuf_buf(resultset->mc_key),wrbuf_len(resultset->mc_key), + str, strlen(str), expiration, flags); + yaz_log(YLOG_LOG, "Store Z39.50 hit count key=%s value=%s rc=%u %s", + wrbuf_cstr(resultset->mc_key), str, (unsigned) rc, + memcached_last_error_message(c->mc_st)); + } +#endif handle_Z3950_records(c, sr->records, 0); } @@ -1282,13 +1316,13 @@ static void handle_Z3950_sort_response(ZOOM_connection c, Z_SortResponse *res) response_diag(c, res->diagnostics[0]); } -static int handle_Z3950_scan_response(ZOOM_connection c, Z_ScanResponse *res) +static void handle_Z3950_scan_response(ZOOM_connection c, Z_ScanResponse *res) { NMEM nmem = odr_extract_mem(c->odr_in); ZOOM_scanset scan; if (!c->tasks || c->tasks->which != ZOOM_TASK_SCAN) - return 0; + return; scan = c->tasks->u.scan.scan; if (res->entries && res->entries->nonsurrogateDiagnostics) @@ -1306,7 +1340,6 @@ static int handle_Z3950_scan_response(ZOOM_connection c, Z_ScanResponse *res) ZOOM_options_set_int(scan->options, "number", *res->numberOfEntriesReturned); nmem_destroy(nmem); - return 1; } static void handle_Z3950_records(ZOOM_connection c, Z_Records *sr, @@ -1316,29 +1349,16 @@ static void handle_Z3950_records(ZOOM_connection c, Z_Records *sr, int *start, *count; const char *syntax = 0, *elementSetName = 0, *schema = 0; - if (!c->tasks) + if (!c->tasks || c->tasks->which != ZOOM_TASK_SEARCH) return ; - switch (c->tasks->which) - { - case ZOOM_TASK_SEARCH: - resultset = c->tasks->u.search.resultset; - start = &c->tasks->u.search.start; - count = &c->tasks->u.search.count; - syntax = c->tasks->u.search.syntax; - elementSetName = c->tasks->u.search.elementSetName; - schema = c->tasks->u.search.schema; - break; - case ZOOM_TASK_RETRIEVE: - resultset = c->tasks->u.retrieve.resultset; - start = &c->tasks->u.retrieve.start; - count = &c->tasks->u.retrieve.count; - syntax = c->tasks->u.retrieve.syntax; - elementSetName = c->tasks->u.retrieve.elementSetName; - schema = c->tasks->u.retrieve.schema; - break; - default: - return; - } + + resultset = c->tasks->u.search.resultset; + start = &c->tasks->u.search.start; + count = &c->tasks->u.search.count; + syntax = c->tasks->u.search.syntax; + elementSetName = c->tasks->u.search.elementSetName; + schema = c->tasks->u.search.schema; + if (sr && sr->which == Z_Records_NSD) response_default_diag(c, sr->u.nonSurrogateDiagnostic); else if (sr && sr->which == Z_Records_multipleNSD) @@ -1360,7 +1380,7 @@ static void handle_Z3950_records(ZOOM_connection c, Z_Records *sr, NMEM nmem = odr_extract_mem(c->odr_in); Z_NamePlusRecordList *p = sr->u.databaseOrSurDiagnostics; - for (i = 0; inum_records; i++) + for (i = 0; i < p->num_records; i++) { ZOOM_record_cache_add(resultset, p->records[i], i + *start, syntax, elementSetName, schema, 0); @@ -1419,7 +1439,6 @@ static void set_init_option(const char *name, void *clientData) ZOOM_connection_option_set(c, buf, "1"); } - zoom_ret send_Z3950_sort(ZOOM_connection c, ZOOM_resultset resultset) { if (c->error) @@ -1442,94 +1461,24 @@ zoom_ret send_Z3950_sort(ZOOM_connection c, ZOOM_resultset resultset) return zoom_complete; } -zoom_ret send_Z3950_present(ZOOM_connection c) +static zoom_ret Z3950_send_present(ZOOM_connection c) { - Z_APDU *apdu = 0; - Z_PresentRequest *req = 0; - int i = 0; - const char *syntax = 0; - const char *elementSetName = 0; - const char *schema = 0; - ZOOM_resultset resultset; - int *start, *count; - - if (!c->tasks) - { - yaz_log(c->log_details, "%p send_present no tasks", c); - return zoom_complete; - } - - switch (c->tasks->which) - { - case ZOOM_TASK_SEARCH: - resultset = c->tasks->u.search.resultset; - start = &c->tasks->u.search.start; - count = &c->tasks->u.search.count; - syntax = c->tasks->u.search.syntax; - elementSetName = c->tasks->u.search.elementSetName; - schema = c->tasks->u.search.schema; - break; - case ZOOM_TASK_RETRIEVE: - resultset = c->tasks->u.retrieve.resultset; - start = &c->tasks->u.retrieve.start; - count = &c->tasks->u.retrieve.count; - syntax = c->tasks->u.retrieve.syntax; - elementSetName = c->tasks->u.retrieve.elementSetName; - schema = c->tasks->u.retrieve.schema; - break; - default: - return zoom_complete; - } - yaz_log(c->log_details, "%p send_present start=%d count=%d", - c, *start, *count); - - if (*start < 0 || *count < 0 || *start + *count > resultset->size) - { - ZOOM_set_dset_error(c, YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, "Bib-1", - "", 0); - } - if (c->error) /* don't continue on error */ - return zoom_complete; - yaz_log(c->log_details, "send_present resultset=%p start=%d count=%d", - resultset, *start, *count); - - for (i = 0; i < *count; i++) - { - ZOOM_record rec = - ZOOM_record_cache_lookup(resultset, i + *start, - syntax, elementSetName, schema); - if (!rec) - break; - else - { - ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD); - ZOOM_connection_put_event(c, event); - } - } - *start += i; - *count -= i; + Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_presentRequest); + Z_PresentRequest *req = apdu->u.presentRequest; + ZOOM_resultset resultset = c->tasks->u.search.resultset; + const char *syntax = c->tasks->u.search.syntax; + const char *elementSetName = c->tasks->u.search.elementSetName; + const char *schema = c->tasks->u.search.schema; - if (*count == 0) - { - yaz_log(c->log_details, "%p send_present skip=%d no more to fetch", c, i); - return zoom_complete; - } + *req->resultSetStartPoint = c->tasks->u.search.start + 1; - apdu = zget_APDU(c->odr_out, Z_APDU_presentRequest); - req = apdu->u.presentRequest; - - if (i) - yaz_log(c->log_details, "%p send_present skip=%d", c, i); - - *req->resultSetStartPoint = *start + 1; - - if (resultset->step > 0 && resultset->step < *count) + if (resultset->step > 0 && resultset->step < c->tasks->u.search.count) *req->numberOfRecordsRequested = resultset->step; else - *req->numberOfRecordsRequested = *count; + *req->numberOfRecordsRequested = c->tasks->u.search.count; - if (*req->numberOfRecordsRequested + *start > resultset->size) - *req->numberOfRecordsRequested = resultset->size - *start; + if (*req->numberOfRecordsRequested + c->tasks->u.search.start > resultset->size) + *req->numberOfRecordsRequested = resultset->size - c->tasks->u.search.start; assert(*req->numberOfRecordsRequested > 0); if (syntax && *syntax) @@ -1596,6 +1545,96 @@ zoom_ret send_Z3950_present(ZOOM_connection c) return send_APDU(c, apdu); } +zoom_ret ZOOM_connection_Z3950_search(ZOOM_connection c) +{ + int i = 0; + const char *syntax = 0; + const char *elementSetName = 0; + const char *schema = 0; + ZOOM_resultset resultset; + int *start, *count; + + if (!c->tasks) + return zoom_complete; + assert(c->tasks->which == ZOOM_TASK_SEARCH); + resultset = c->tasks->u.search.resultset; + start = &c->tasks->u.search.start; + count = &c->tasks->u.search.count; + syntax = c->tasks->u.search.syntax; + elementSetName = c->tasks->u.search.elementSetName; + schema = c->tasks->u.search.schema; + + yaz_log(c->log_details, "%p send_present start=%d count=%d", + c, *start, *count); + +#if HAVE_LIBMEMCACHED_MEMCACHED_H + /* TODO: add sorting */ + if (c->mc_st && resultset->live_set == 0) + { + size_t v_len; + uint32_t flags; + memcached_return_t rc; + char *v = memcached_get(c->mc_st, wrbuf_buf(resultset->mc_key), + wrbuf_len(resultset->mc_key), + &v_len, &flags, &rc); + if (v) + { + ZOOM_Event event; + WRBUF w = wrbuf_alloc(); + + wrbuf_write(w, v, v_len); + free(v); + resultset->size = odr_atoi(wrbuf_cstr(w)); + + yaz_log(YLOG_LOG, "For key %s got value %s", + wrbuf_cstr(resultset->mc_key), wrbuf_cstr(w)); + + wrbuf_destroy(w); + event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH); + ZOOM_connection_put_event(c, event); + resultset->live_set = 1; + } + } +#endif + if (*start < 0 || *count < 0) + { + ZOOM_set_dset_error(c, YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, "Bib-1", + "start/count < 0", 0); + } + + if (resultset->live_set) + { + if (*start >= resultset->size) + return zoom_complete; + if (*start + *count > resultset->size) + *count = resultset->size - *start; + } + + if (c->error) /* don't continue on error */ + return zoom_complete; + yaz_log(c->log_details, "send_present resultset=%p start=%d count=%d", + resultset, *start, *count); + + for (i = 0; i < *count; i++) + { + ZOOM_record rec = + ZOOM_record_cache_lookup(resultset, i + *start, + syntax, elementSetName, schema); + if (!rec) + break; + } + *start += i; + *count -= i; + + if (*count == 0 && resultset->live_set) + return zoom_complete; + + if (resultset->live_set == 2) + return Z3950_send_present(c); + else + return Z3950_send_search(c); +} + static zoom_ret send_Z3950_sort_present(ZOOM_connection c) { zoom_ret r = zoom_complete; @@ -1603,7 +1642,7 @@ static zoom_ret send_Z3950_sort_present(ZOOM_connection c) if (c->tasks && c->tasks->which == ZOOM_TASK_SEARCH) r = send_Z3950_sort(c, c->tasks->u.search.resultset); if (r == zoom_complete) - r = send_Z3950_present(c); + r = ZOOM_connection_Z3950_search(c); return r; } @@ -1708,14 +1747,13 @@ void ZOOM_handle_Z3950_apdu(ZOOM_connection c, Z_APDU *apdu) case Z_APDU_presentResponse: yaz_log(c->log_api, "%p handle_Z3950_apdu Present response", c); handle_Z3950_present_response(c, apdu->u.presentResponse); - if (send_Z3950_present(c) == zoom_complete) + if (ZOOM_connection_Z3950_search(c) == zoom_complete) ZOOM_connection_remove_task(c); break; case Z_APDU_sortResponse: yaz_log(c->log_api, "%p handle_Z3950_apdu Sort response", c); handle_Z3950_sort_response(c, apdu->u.sortResponse); - if (send_Z3950_present(c) == zoom_complete) - ZOOM_connection_remove_task(c); + ZOOM_connection_remove_task(c); break; case Z_APDU_scanResponse: yaz_log(c->log_api, "%p handle_Z3950_apdu Scan response", c); diff --git a/test/test_wrbuf.c b/test/test_wrbuf.c index 644f52a..e7886c4 100644 --- a/test/test_wrbuf.c +++ b/test/test_wrbuf.c @@ -76,6 +76,15 @@ static void tstwrbuf(void) wrbuf_insert(wr, 5, "abc", 3); YAZ_CHECK(!strcmp(wrbuf_cstr(wr), "1234")); +#if HAVE_GCRYPT_H + { + const char *msg = "Hello world\n"; + wrbuf_rewind(wr); + wrbuf_sha1_write(wr, msg, strlen(msg), 1); + YAZ_CHECK(!strcmp(wrbuf_cstr(wr), + "33ab5639bfd8e7b95eb1d8d0b87781d4ffea4d5d")); + } +#endif wrbuf_destroy(wr); } diff --git a/win/makefile b/win/makefile index 661d78e..02e3ad1 100644 --- a/win/makefile +++ b/win/makefile @@ -457,6 +457,7 @@ MISC_OBJS= \ $(OBJDIR)\readconf.obj \ $(OBJDIR)\tpath.obj \ $(OBJDIR)\wrbuf.obj \ + $(OBJDIR)\wrbuf_sha1.obj \ $(OBJDIR)\xmalloc.obj \ $(OBJDIR)\matchstr.obj \ $(OBJDIR)\siconv.obj \