From: Adam Dickmeiss <adam@indexdata.dk>
Date: Thu, 23 Jan 2014 13:00:51 +0000 (+0100)
Subject: Merge branch 'master' into yaz-728
X-Git-Tag: v5.0.13~45
X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=commitdiff_plain;h=35ffe076512941377bb6f3cbdfa774a4ceaf9641;hp=c1f23597ea64485e2362d658f3653211202cc6a8

Merge branch 'master' into yaz-728

Conflicts:
	configure.ac
---

diff --git a/configure.ac b/configure.ac
index 8eb9e8c..c85286f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -59,7 +59,35 @@ if test "$checkBoth" = "1"; then
     AC_CHECK_FUNC([accept], , [LIBS=$oldLibs])
 fi
 AC_CHECK_FUNC([gethostbyname], ,[AC_CHECK_LIB(nsl, main, [LIBS="$LIBS -lnsl"])])
-dnl
+dnl ------ libgcrypt
+libgcryptpath=NONE
+AC_ARG_WITH(libgcrypt, [ --with-libgcrypt=DIR  use libgcrypt-config in DIR],[libgcryptpath=$withval])
+if test "$libgcryptpath" != "no"; then
+    if test "$libgcryptpath" = "NONE"; then
+	AC_PATH_PROG([libgcrypt],[libgcrypt-config],[NONE])
+    else
+	libgcrypt=$libgcryptpath/libgcrypt-config
+    fi
+    AC_MSG_CHECKING([for libgcrypt])
+    if "$libgcrypt" --version >/dev/null 2>&1; then
+	LIBS="$LIBS `$libgcrypt --libs`"
+	CFLAGS="$CFLAGS `$libgcrypt --cflags`"
+	libgcryptversion=`$libgcrypt --version`
+	AC_MSG_RESULT([$libgcryptversion])
+	AC_DEFINE([HAVE_GCRYPT_H],[1],[Define to 1 if libgcrypt is present])
+    else
+	AC_MSG_RESULT([no])
+    fi
+fi
+dnl ------ memcached
+memcached=default
+AC_ARG_WITH([memcached], [  --with-memcached        Memcached library], [memcached=$withval])
+if test "$memcached" != "no"; then
+    AC_CHECK_LIB([memcached],[memcached_get])
+    if test "$ac_cv_lib_memcached_memcached_get" = "yes"; then
+	AC_CHECK_HEADERS([libmemcached/memcached.h])
+    fi
+fi
 dnl ------ GNU TLS
 AC_SUBST([SSL_CFLAGS])
 AC_SUBST([SSL_LIBS])
diff --git a/include/yaz/zoom.h b/include/yaz/zoom.h
index 27ab47e..e460704 100644
--- a/include/yaz/zoom.h
+++ b/include/yaz/zoom.h
@@ -143,6 +143,7 @@ ZOOM_diag_str (int error);
 #define ZOOM_ERROR_ES_INVALID_ACTION 10015
 #define ZOOM_ERROR_ES_INVALID_VERSION 10016
 #define ZOOM_ERROR_ES_INVALID_SYNTAX 10017
+#define ZOOM_ERROR_MEMCACHED 10018
 
 ZOOM_API(int)
 ZOOM_connection_last_event(ZOOM_connection cs);
diff --git a/src/zoom-c.c b/src/zoom-c.c
index c3d7f0c..6e0a883 100644
--- a/src/zoom-c.c
+++ b/src/zoom-c.c
@@ -136,9 +136,6 @@ void ZOOM_connection_show_task(ZOOM_task task)
     case ZOOM_TASK_SEARCH:
         yaz_log(YLOG_LOG, "search p=%p", task);
         break;
-    case ZOOM_TASK_RETRIEVE:
-        yaz_log(YLOG_LOG, "retrieve p=%p", task);
-        break;
     case ZOOM_TASK_CONNECT:
         yaz_log(YLOG_LOG, "connect p=%p", task);
         break;
@@ -201,12 +198,6 @@ void ZOOM_connection_remove_task(ZOOM_connection c)
             xfree(task->u.search.elementSetName);
             xfree(task->u.search.schema);
             break;
-        case ZOOM_TASK_RETRIEVE:
-            resultset_destroy(task->u.retrieve.resultset);
-            xfree(task->u.retrieve.syntax);
-            xfree(task->u.retrieve.elementSetName);
-            xfree(task->u.retrieve.schema);
-            break;
         case ZOOM_TASK_CONNECT:
             break;
         case ZOOM_TASK_SCAN:
@@ -308,6 +299,10 @@ ZOOM_API(ZOOM_connection)
     c->no_redirects = 0;
     c->cookies = 0;
     c->saveAPDU_wrbuf = 0;
+
+#if HAVE_LIBMEMCACHED_MEMCACHED_H
+    c->mc_st = 0;
+#endif
     return c;
 }
 
@@ -552,6 +547,29 @@ ZOOM_API(void)
     yaz_cookies_destroy(c->cookies);
     c->cookies = yaz_cookies_create();
 
+#if HAVE_LIBMEMCACHED_MEMCACHED_H
+    if (c->mc_st)
+    {
+        memcached_free(c->mc_st);
+        c->mc_st = 0;
+    }
+#endif
+    val = ZOOM_options_get(c->options, "memcached");
+    if (val && *val)
+    {
+#if HAVE_LIBMEMCACHED_MEMCACHED_H
+        c->mc_st = memcached(val, strlen(val));
+        if (!c->mc_st)
+        {
+            ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, val);
+            return;
+        }
+#else
+        ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
+        return;
+#endif
+    }
+
     if (c->sru_mode == zoom_sru_error)
     {
         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
@@ -596,6 +614,11 @@ ZOOM_API(void)
     if (!c)
         return;
     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
+
+#if HAVE_LIBMEMCACHED_MEMCACHED_H
+    if (c->mc_st)
+        memcached_free(c->mc_st);
+#endif
     if (c->cs)
         cs_close(c->cs);
 
@@ -700,6 +723,8 @@ ZOOM_resultset ZOOM_resultset_create(void)
     }
 #endif
     resultset_use(1);
+    r->mc_key = 0;
+    r->live_set = 0;
     return r;
 }
 
@@ -720,14 +745,14 @@ ZOOM_API(ZOOM_resultset)
     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
 {
     ZOOM_resultset r = ZOOM_resultset_create();
-    ZOOM_task task;
     const char *cp;
+    ZOOM_task task;
     int start, count;
     const char *syntax, *elementSetName, *schema;
-
     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
     r->r_sort_spec = ZOOM_query_get_sortspec(q);
     r->query = q;
+    ZOOM_query_addref(q);
 
     r->options = ZOOM_options_create_with_parent(c->options);
 
@@ -750,6 +775,43 @@ ZOOM_API(ZOOM_resultset)
     r->connection = c;
     r->next = c->resultsets;
     c->resultsets = r;
+
+#if HAVE_LIBMEMCACHED_MEMCACHED_H
+    r->mc_key = wrbuf_alloc();
+    wrbuf_puts(r->mc_key, c->host_port);
+    wrbuf_puts(r->mc_key, ";");
+    wrbuf_puts(r->mc_key, ZOOM_query_get_query_string(q));
+    if (c->mc_st)
+    {
+        size_t v_len;
+        uint32_t flags;
+        memcached_return_t rc;
+        char *v = memcached_get(c->mc_st, wrbuf_buf(r->mc_key),
+                                wrbuf_len(r->mc_key), &v_len, &flags, &rc);
+        if (v)
+        {
+            ZOOM_Event event;
+            WRBUF w = wrbuf_alloc();
+
+            wrbuf_write(w, v, v_len);
+            free(v);
+            r->size = odr_atoi(wrbuf_cstr(w));
+
+            yaz_log(YLOG_LOG, "For key %s got value %s",
+                    wrbuf_cstr(r->mc_key), wrbuf_cstr(w));
+
+            wrbuf_destroy(w);
+            event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
+            ZOOM_connection_put_event(c, event);
+            return r;
+        }
+        else
+        {
+            yaz_log(YLOG_LOG, "For key %s got NO value", wrbuf_cstr(r->mc_key));
+        }
+    }
+#endif
+
     if (c->host_port && c->proto == PROTO_HTTP)
     {
         if (!c->cs)
@@ -780,8 +842,6 @@ ZOOM_API(ZOOM_resultset)
 
     ZOOM_resultset_addref(r);
 
-    ZOOM_query_addref(q);
-
     if (!c->async)
     {
         while (ZOOM_event(1, &c))
@@ -874,6 +934,7 @@ static void resultset_destroy(ZOOM_resultset r)
 #if SHPTR
         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
 #endif
+        wrbuf_destroy(r->mc_key);
         resultset_use(-1);
         xfree(r);
     }
@@ -934,19 +995,19 @@ static void ZOOM_resultset_retrieve(ZOOM_resultset r,
             c->reconnect_ok = 1;
         }
     }
-    task = ZOOM_connection_add_task(c, ZOOM_TASK_RETRIEVE);
-    task->u.retrieve.resultset = r;
-    task->u.retrieve.start = start;
-    task->u.retrieve.count = count;
+    task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
+    task->u.search.resultset = r;
+    task->u.search.start = start;
+    task->u.search.count = count;
 
     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
-    task->u.retrieve.syntax = syntax ? xstrdup(syntax) : 0;
+    task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
     elementSetName = ZOOM_options_get(r->options, "elementSetName");
-    task->u.retrieve.elementSetName = elementSetName
+    task->u.search.elementSetName = elementSetName
         ? xstrdup(elementSetName) : 0;
 
     cp = ZOOM_options_get(r->options, "schema");
-    task->u.retrieve.schema = cp ? xstrdup(cp) : 0;
+    task->u.search.schema = cp ? xstrdup(cp) : 0;
 
     ZOOM_resultset_addref(r);
 
@@ -1444,12 +1505,6 @@ ZOOM_API(int)
             else
                 ret = ZOOM_connection_Z3950_send_search(c);
             break;
-        case ZOOM_TASK_RETRIEVE:
-            if (c->proto == PROTO_HTTP)
-                ret = ZOOM_connection_srw_send_search(c);
-            else
-                ret = send_Z3950_present(c);
-            break;
         case ZOOM_TASK_CONNECT:
             ret = do_connect(c);
             break;
@@ -1898,6 +1953,8 @@ ZOOM_API(const char *)
         return "Extended Service. invalid version";
     case ZOOM_ERROR_ES_INVALID_SYNTAX:
         return "Extended Service. invalid syntax";
+    case ZOOM_ERROR_MEMCACHED:
+        return "Memcached";
     default:
         return diagbib1_str(error);
     }
diff --git a/src/zoom-p.h b/src/zoom-p.h
index 77f39e9..5d4ee61 100644
--- a/src/zoom-p.h
+++ b/src/zoom-p.h
@@ -36,6 +36,9 @@
 #include <yaz/srw.h>
 #include <yaz/cookie.h>
 #include <yaz/mutex.h>
+#if HAVE_LIBMEMCACHED_MEMCACHED_H
+#include <libmemcached/memcached.h>
+#endif
 
 #define SHPTR 1
 
@@ -108,6 +111,9 @@ struct ZOOM_connection_p {
     int log_details;
     int log_api;
     WRBUF saveAPDU_wrbuf;
+#if HAVE_LIBMEMCACHED_MEMCACHED_H
+    memcached_st *mc_st;
+#endif
 };
 
 typedef struct ZOOM_record_cache_p *ZOOM_record_cache;
@@ -136,6 +142,8 @@ struct ZOOM_resultset_p {
     ZOOM_facet_field *facets;
     int num_facets;
     char **facets_names;
+    WRBUF mc_key;
+    int live_set;
 };
 
 struct facet_term_p {
@@ -185,15 +193,6 @@ struct ZOOM_task_p {
             char *schema;
             int recv_search_fired;
         } search;
-#define ZOOM_TASK_RETRIEVE 2
-        struct {
-            int start;
-            ZOOM_resultset resultset;
-            int count;
-            char *syntax;
-            char *elementSetName;
-            char *schema;
-        } retrieve;
 #define ZOOM_TASK_CONNECT 3
 #define ZOOM_TASK_SCAN 4
         struct {
diff --git a/src/zoom-sru.c b/src/zoom-sru.c
index 4e8807b..8f5feb0 100644
--- a/src/zoom-sru.c
+++ b/src/zoom-sru.c
@@ -171,12 +171,6 @@ zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
             facet_list = yaz_pqf_parse_facet_list(c->odr_out, facets);
         schema = c->tasks->u.search.schema;
         break;
-    case ZOOM_TASK_RETRIEVE:
-        resultset = c->tasks->u.retrieve.resultset;
-        schema = c->tasks->u.retrieve.schema;
-
-        start = &c->tasks->u.retrieve.start;
-        count = &c->tasks->u.retrieve.count;
 
         if (*start >= resultset->size)
             return zoom_complete;
@@ -187,8 +181,8 @@ zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
         {
             ZOOM_record rec =
                 ZOOM_record_cache_lookup(resultset, i + *start,
-                                         c->tasks->u.retrieve.syntax,
-                                         c->tasks->u.retrieve.elementSetName,
+                                         c->tasks->u.search.syntax,
+                                         c->tasks->u.search.elementSetName,
                                          schema);
             if (!rec)
                 break;
@@ -292,14 +286,6 @@ static zoom_ret handle_srw_response(ZOOM_connection c,
         if (res->facetList)
             ZOOM_handle_facet_list(resultset, res->facetList);
         break;
-    case ZOOM_TASK_RETRIEVE:
-        resultset = c->tasks->u.retrieve.resultset;
-        start = &c->tasks->u.retrieve.start;
-        count = &c->tasks->u.retrieve.count;
-        syntax = c->tasks->u.retrieve.syntax;
-        elementSetName = c->tasks->u.retrieve.elementSetName;
-        schema = c->tasks->u.retrieve.schema;
-        break;
     default:
         return zoom_complete;
     }
diff --git a/src/zoom-z3950.c b/src/zoom-z3950.c
index 80ed718..df31381 100644
--- a/src/zoom-z3950.c
+++ b/src/zoom-z3950.c
@@ -639,17 +639,23 @@ zoom_ret ZOOM_connection_Z3950_send_search(ZOOM_connection c)
     int lslb, ssub, mspn;
     const char *syntax;
     const char *schema;
-    Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_searchRequest);
-    Z_SearchRequest *search_req = apdu->u.searchRequest;
+    Z_APDU *apdu;
+    Z_SearchRequest *search_req;
     const char *elementSetName;
     const char *smallSetElementSetName;
     const char *mediumSetElementSetName;
     const char *facets;
 
-    assert(c->tasks);
-    assert(c->tasks->which == ZOOM_TASK_SEARCH);
+    if (!c->tasks)
+        return zoom_complete;
 
+    assert(c->tasks->which == ZOOM_TASK_SEARCH);
     r = c->tasks->u.search.resultset;
+    if (r->live_set)
+        return send_Z3950_present(c);
+
+    apdu = zget_APDU(c->odr_out, Z_APDU_searchRequest);
+    search_req = apdu->u.searchRequest;
 
     yaz_log(c->log_details, "%p ZOOM_connection_send_search set=%p", c, r);
 
@@ -1035,12 +1041,12 @@ static int es_response_taskpackage(ZOOM_connection c,
     return 1;
 }
 
-
-static int handle_Z3950_es_response(ZOOM_connection c,
-                                    Z_ExtendedServicesResponse *res)
+static void handle_Z3950_es_response(ZOOM_connection c,
+                                     Z_ExtendedServicesResponse *res)
 {
-    if (!c->tasks || c->tasks->which != ZOOM_TASK_PACKAGE)
-        return 0;
+    if (!c->tasks)
+        return;
+    assert(c->tasks->which == ZOOM_TASK_PACKAGE);
     switch (*res->operationStatus)
     {
     case Z_ExtendedServicesResponse_done:
@@ -1068,7 +1074,6 @@ static int handle_Z3950_es_response(ZOOM_connection c,
         ZOOM_options_setl(c->tasks->u.package->options,
                           "xmlUpdateDoc", (char*) doc->buf, doc->len);
     }
-    return 1;
 }
 
 static char *get_term_cstr(ODR odr, Z_Term *term) {
@@ -1251,7 +1256,7 @@ static void handle_Z3950_search_response(ZOOM_connection c,
     ZOOM_Event event;
 
     if (!c->tasks || c->tasks->which != ZOOM_TASK_SEARCH)
-        return ;
+        return;
 
     event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
     ZOOM_connection_put_event(c, event);
@@ -1273,6 +1278,25 @@ static void handle_Z3950_search_response(ZOOM_connection c,
     handle_facet_result(c, resultset, sr->additionalSearchInfo);
 
     resultset->size = *sr->resultCount;
+    resultset->live_set = 1;
+
+#if HAVE_LIBMEMCACHED_MEMCACHED_H
+    if (c->mc_st)
+    {
+        uint32_t flags = 0;
+        memcached_return_t rc;
+        time_t expiration = 36000;
+        char str[40];
+
+        sprintf(str, ODR_INT_PRINTF, *sr->resultCount);
+        rc = memcached_set(c->mc_st,
+                           wrbuf_buf(resultset->mc_key),wrbuf_len(resultset->mc_key),
+                           str, strlen(str), expiration, flags);
+        yaz_log(YLOG_LOG, "Key=%s value=%s rc=%u %s",
+                wrbuf_cstr(resultset->mc_key), str, (unsigned) rc,
+                memcached_last_error_message(c->mc_st));
+    }
+#endif
     handle_Z3950_records(c, sr->records, 0);
 }
 
@@ -1282,13 +1306,13 @@ static void handle_Z3950_sort_response(ZOOM_connection c, Z_SortResponse *res)
         response_diag(c, res->diagnostics[0]);
 }
 
-static int handle_Z3950_scan_response(ZOOM_connection c, Z_ScanResponse *res)
+static void handle_Z3950_scan_response(ZOOM_connection c, Z_ScanResponse *res)
 {
     NMEM nmem = odr_extract_mem(c->odr_in);
     ZOOM_scanset scan;
 
     if (!c->tasks || c->tasks->which != ZOOM_TASK_SCAN)
-        return 0;
+        return;
     scan = c->tasks->u.scan.scan;
 
     if (res->entries && res->entries->nonsurrogateDiagnostics)
@@ -1306,7 +1330,6 @@ static int handle_Z3950_scan_response(ZOOM_connection c, Z_ScanResponse *res)
         ZOOM_options_set_int(scan->options, "number",
                              *res->numberOfEntriesReturned);
     nmem_destroy(nmem);
-    return 1;
 }
 
 static void handle_Z3950_records(ZOOM_connection c, Z_Records *sr,
@@ -1316,29 +1339,16 @@ static void handle_Z3950_records(ZOOM_connection c, Z_Records *sr,
     int *start, *count;
     const char *syntax = 0, *elementSetName = 0, *schema = 0;
 
-    if (!c->tasks)
+    if (!c->tasks || c->tasks->which != ZOOM_TASK_SEARCH)
         return ;
-    switch (c->tasks->which)
-    {
-    case ZOOM_TASK_SEARCH:
-        resultset = c->tasks->u.search.resultset;
-        start = &c->tasks->u.search.start;
-        count = &c->tasks->u.search.count;
-        syntax = c->tasks->u.search.syntax;
-        elementSetName = c->tasks->u.search.elementSetName;
-        schema =  c->tasks->u.search.schema;
-        break;
-    case ZOOM_TASK_RETRIEVE:
-        resultset = c->tasks->u.retrieve.resultset;
-        start = &c->tasks->u.retrieve.start;
-        count = &c->tasks->u.retrieve.count;
-        syntax = c->tasks->u.retrieve.syntax;
-        elementSetName = c->tasks->u.retrieve.elementSetName;
-        schema =  c->tasks->u.retrieve.schema;
-        break;
-    default:
-        return;
-    }
+
+    resultset = c->tasks->u.search.resultset;
+    start = &c->tasks->u.search.start;
+    count = &c->tasks->u.search.count;
+    syntax = c->tasks->u.search.syntax;
+    elementSetName = c->tasks->u.search.elementSetName;
+    schema =  c->tasks->u.search.schema;
+
     if (sr && sr->which == Z_Records_NSD)
         response_default_diag(c, sr->u.nonSurrogateDiagnostic);
     else if (sr && sr->which == Z_Records_multipleNSD)
@@ -1454,32 +1464,15 @@ zoom_ret send_Z3950_present(ZOOM_connection c)
     int *start, *count;
 
     if (!c->tasks)
-    {
-        yaz_log(c->log_details, "%p send_present no tasks", c);
         return zoom_complete;
-    }
+    assert(c->tasks->which == ZOOM_TASK_SEARCH);
+    resultset = c->tasks->u.search.resultset;
+    start = &c->tasks->u.search.start;
+    count = &c->tasks->u.search.count;
+    syntax = c->tasks->u.search.syntax;
+    elementSetName = c->tasks->u.search.elementSetName;
+    schema =  c->tasks->u.search.schema;
 
-    switch (c->tasks->which)
-    {
-    case ZOOM_TASK_SEARCH:
-        resultset = c->tasks->u.search.resultset;
-        start = &c->tasks->u.search.start;
-        count = &c->tasks->u.search.count;
-        syntax = c->tasks->u.search.syntax;
-        elementSetName = c->tasks->u.search.elementSetName;
-        schema =  c->tasks->u.search.schema;
-        break;
-    case ZOOM_TASK_RETRIEVE:
-        resultset = c->tasks->u.retrieve.resultset;
-        start = &c->tasks->u.retrieve.start;
-        count = &c->tasks->u.retrieve.count;
-        syntax = c->tasks->u.retrieve.syntax;
-        elementSetName = c->tasks->u.retrieve.elementSetName;
-        schema = c->tasks->u.retrieve.schema;
-        break;
-    default:
-        return zoom_complete;
-    }
     yaz_log(c->log_details, "%p send_present start=%d count=%d",
             c, *start, *count);