Redis functional including configure + code

No RPM/Deb work - yet.

diff --git a/configure.ac b/configure.ac
index 8204bf3..049b3ca 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -92,6 +92,31 @@ if test "$libgcryptpath" != "no"; then
        AC_MSG_RESULT([no])
     fi
 fi
+dnl ------ redis
+hiredis=default
+AC_SUBST([HIREDIS_LIBS])
+AC_ARG_WITH([redis], [  --with-redis        hiredis library], [hiredis=$withval])
+if test "$hiredis" != "no" -a "$pkgconfigpath" != "NONE"; then
+    AC_MSG_CHECKING([for redis])
+    if $pkgconfigpath --cflags hiredis >/dev/null 2>&1 ; then
+       if $pkgconfigpath --atleast-version 0.10 hiredis; then
+           AC_MSG_RESULT([yes])
+            CFLAGS="$CFLAGS `$pkgconfigpath --cflags hiredis`"
+            HIREDIS_LIBS="`$pkgconfigpath --libs hiredis`"
+           AC_DEFINE([HAVE_HIREDIS],[1],[Define to 1 if hiredis is enabled])
+       else
+           AC_MSG_RESULT([no. Version 0.10 required])
+            if test "$hiredis" != "default"; then
+               AC_MSG_ERROR([hiredis libraries missing])
+            fi
+       fi
+    else
+       AC_MSG_RESULT([no])
+        if test "$hiredis" != "default"; then
+           AC_MSG_ERROR([hiredis libraries missing])
+       fi
+    fi
+fi
 dnl ------ memcached
 memcached=default
 AC_SUBST([MEMCACHED_LIBS])

diff --git a/src/Makefile.am b/src/Makefile.am
index 07e125b..3f431f0 100644 (file)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -25,8 +25,9 @@ YAZCOMP=$(top_srcdir)/util/yaz-asncomp
 YAZCOMP_Z = $(YAZCOMP) -d $(srcdir)/z.tcl -i yaz -I$(top_srcdir)/include
 YAZCOMP_I = $(YAZCOMP) -d $(srcdir)/ill.tcl -i yaz -I$(top_srcdir)/include
 
-AM_CPPFLAGS=-I$(top_srcdir)/include $(XML2_CFLAGS) $(SSL_CFLAGS) 
-libyaz_la_LIBADD = $(SSL_LIBS) $(TCPD_LIBS) $(GCRYPT_LIBS) $(MEMCACHED_LIBS)
+AM_CPPFLAGS=-I$(top_srcdir)/include $(XML2_CFLAGS) $(SSL_CFLAGS)
+libyaz_la_LIBADD = $(SSL_LIBS) $(TCPD_LIBS) $(GCRYPT_LIBS) \
+       $(MEMCACHED_LIBS) $(HIREDIS_LIBS)
 libyaz_server_la_LIBADD = libyaz.la
 libyaz_icu_la_CPPFLAGS = $(AM_CPPFLAGS) $(ICU_CPPFLAGS) -I$(top_srcdir)/libstemmer_c/include
 libyaz_icu_la_LIBADD = libyaz.la $(ICU_LIBS)

diff --git a/src/zoom-memcached.c b/src/zoom-memcached.c
index 70d7f3a..828dca6 100644 (file)
--- a/src/zoom-memcached.c
+++ b/src/zoom-memcached.c
@@ -32,6 +32,9 @@ void ZOOM_memcached_init(ZOOM_connection c)
 #if HAVE_LIBMEMCACHED_MEMCACHED_H
     c->mc_st = 0;
 #endif
+#if HAVE_HIREDIS
+    c->redis_c = 0;
+#endif
 }
 
 void ZOOM_memcached_destroy(ZOOM_connection c)
@@ -40,6 +43,10 @@ void ZOOM_memcached_destroy(ZOOM_connection c)
     if (c->mc_st)
         memcached_free(c->mc_st);
 #endif
+#if HAVE_HIREDIS
+    if (c->redis_c)
+        redisFree(c->redis_c);
+#endif
 }
 
 #if HAVE_LIBMEMCACHED_MEMCACHED_H
@@ -95,6 +102,13 @@ static memcached_st *yaz_memcached_wrap(const char *conf)
 int ZOOM_memcached_configure(ZOOM_connection c)
 {
     const char *val;
+#if HAVE_HIREDIS
+    if (c->redis_c)
+    {
+        redisFree(c->redis_c);
+        c->redis_c = 0;
+    }
+#endif
 #if HAVE_LIBMEMCACHED_MEMCACHED_H
     if (c->mc_st)
     {
@@ -102,6 +116,26 @@ int ZOOM_memcached_configure(ZOOM_connection c)
         c->mc_st = 0;
     }
 #endif
+
+    val = ZOOM_options_get(c->options, "redis");
+    if (val && *val)
+    {
+#if HAVE_HIREDIS
+        struct timeval timeout = { 1, 500000 }; /* 1.5 seconds */
+
+        c->redis_c = redisConnectWithTimeout(val, 6379, timeout);
+        if (c->redis_c == 0 || c->redis_c->err)
+        {
+            ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED,
+                           "could not create redis");
+            return -1;
+        }
+        return 0; /* don't bother with memcached if redis is enabled */
+#else
+        ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
+        return -1;
+#endif
+    }
     val = ZOOM_options_get(c->options, "memcached");
     if (val && *val)
     {
@@ -122,7 +156,7 @@ int ZOOM_memcached_configure(ZOOM_connection c)
     return 0;
 }
 
-#if HAVE_LIBMEMCACHED_MEMCACHED_H
+#if HAVE_GCRYPT_H
 static void wrbuf_vary_puts(WRBUF w, const char *v)
 {
     if (v)
@@ -141,7 +175,7 @@ static void wrbuf_vary_puts(WRBUF w, const char *v)
 
 void ZOOM_memcached_resultset(ZOOM_resultset r, ZOOM_query q)
 {
-#if HAVE_LIBMEMCACHED_MEMCACHED_H
+#if HAVE_GCRYPT_H
     ZOOM_connection c = r->connection;
 
     r->mc_key = wrbuf_alloc();
@@ -170,6 +204,50 @@ void ZOOM_memcached_resultset(ZOOM_resultset r, ZOOM_query q)
 
 void ZOOM_memcached_search(ZOOM_connection c, ZOOM_resultset resultset)
 {
+#if HAVE_HIREDIS
+    if (c->redis_c && resultset->live_set == 0)
+    {
+        redisReply *reply;
+        const char *argv[2];
+
+        argv[0] = "GET";
+        argv[1] = wrbuf_cstr(resultset->mc_key);
+
+        reply = redisCommandArgv(c->redis_c, 2, argv, 0);
+        /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
+        if (reply && reply->type == REDIS_REPLY_STRING)
+        {
+            char *v = reply->str;
+            int v_len = reply->len;
+            ZOOM_Event event;
+            size_t lead_len = strlen(v) + 1;
+
+            resultset->size = odr_atoi(v);
+
+            yaz_log(YLOG_LOG, "For key %s got value %s lead_len=%d len=%d",
+                    wrbuf_cstr(resultset->mc_key), v, (int) lead_len,
+                    (int) v_len);
+            if (v_len > lead_len)
+            {
+                Z_OtherInformation *oi = 0;
+                int oi_len = v_len - lead_len;
+                odr_setbuf(resultset->odr, v + lead_len, oi_len, 0);
+                if (!z_OtherInformation(resultset->odr, &oi, 0, 0))
+                {
+                    yaz_log(YLOG_WARN, "oi decoding failed");
+                    freeReplyObject(reply);
+                    return;
+                }
+                ZOOM_handle_search_result(c, resultset, oi);
+                ZOOM_handle_facet_result(c, resultset, oi);
+            }
+            event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
+            ZOOM_connection_put_event(c, event);
+            resultset->live_set = 1;
+        }
+        freeReplyObject(reply);
+    }
+#endif
 #if HAVE_LIBMEMCACHED_MEMCACHED_H
     if (c->mc_st && resultset->live_set == 0)
     {
@@ -216,6 +294,44 @@ void ZOOM_memcached_search(ZOOM_connection c, ZOOM_resultset resultset)
 void ZOOM_memcached_hitcount(ZOOM_connection c, ZOOM_resultset resultset,
                              Z_OtherInformation *oi, const char *precision)
 {
+#if HAVE_HIREDIS
+    if (c->redis_c && resultset->live_set == 0)
+    {
+        char *str;
+        ODR odr = odr_createmem(ODR_ENCODE);
+        char *oi_buf = 0;
+        int oi_len = 0;
+        char *key;
+
+        str = odr_malloc(odr, 20 + strlen(precision));
+        /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
+        sprintf(str, ODR_INT_PRINTF ";%s", resultset->size, precision);
+        if (oi)
+        {
+            z_OtherInformation(odr, &oi, 0, 0);
+            oi_buf = odr_getbuf(odr, &oi_len, 0);
+        }
+        key = odr_malloc(odr, strlen(str) + 1 + oi_len);
+        strcpy(key, str);
+        if (oi_len)
+            memcpy(key + strlen(str) + 1, oi_buf, oi_len);
+
+        {
+            redisReply *reply;
+            const char *argv[3];
+            size_t argvlen[3];
+            argv[0] = "SET";
+            argvlen[0] = 3;
+            argv[1] = wrbuf_buf(resultset->mc_key);
+            argvlen[1] = wrbuf_len(resultset->mc_key);
+            argv[2] = key;
+            argvlen[2] = strlen(str) + 1 + oi_len;
+            reply = redisCommandArgv(c->redis_c, 3, argv, argvlen);
+            freeReplyObject(reply);
+        }
+        odr_destroy(odr);
+    }
+#endif
 #if HAVE_LIBMEMCACHED_MEMCACHED_H
     if (c->mc_st && resultset->live_set == 0)
     {
@@ -259,6 +375,57 @@ void ZOOM_memcached_add(ZOOM_resultset r, Z_NamePlusRecord *npr,
                         const char *schema,
                         Z_SRW_diagnostic *diag)
 {
+#if HAVE_HIREDIS
+    if (r->connection->redis_c &&
+        !diag && npr->which == Z_NamePlusRecord_databaseRecord)
+    {
+        WRBUF k = wrbuf_alloc();
+        WRBUF rec_sha1 = wrbuf_alloc();
+        ODR odr = odr_createmem(ODR_ENCODE);
+        char *rec_buf;
+        int rec_len;
+        const char *argv[3];
+        size_t argvlen[3];
+        redisReply *reply;
+
+        z_NamePlusRecord(odr, &npr, 0, 0);
+        rec_buf = odr_getbuf(odr, &rec_len, 0);
+
+        wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
+        wrbuf_printf(k, ";%d;%s;%s;%s", pos,
+                     syntax ? syntax : "",
+                     elementSetName ? elementSetName : "",
+                     schema ? schema : "");
+
+        wrbuf_sha1_write(rec_sha1, rec_buf, rec_len, 1);
+
+        argv[0] = "SET";
+        argvlen[0] = 3;
+        argv[1] = wrbuf_buf(k);
+        argvlen[1] = wrbuf_len(k);
+        argv[2] = wrbuf_buf(rec_sha1);
+        argvlen[2] = wrbuf_len(rec_sha1);
+
+        reply = redisCommandArgv(r->connection->redis_c, 3, argv, argvlen);
+        yaz_log(YLOG_LOG, "Store record key=%s val=%s",
+                wrbuf_cstr(k), wrbuf_cstr(rec_sha1));
+        freeReplyObject(reply);
+
+        argv[1] = wrbuf_buf(rec_sha1);
+        argvlen[1] = wrbuf_len(rec_sha1);
+        argv[2] = rec_buf;
+        argvlen[2] = rec_len;
+
+        reply = redisCommandArgv(r->connection->redis_c, 3, argv, argvlen);
+        yaz_log(YLOG_LOG, "Add record key=%s rec_len=%d",
+                wrbuf_cstr(rec_sha1), rec_len);
+        freeReplyObject(reply);
+
+        odr_destroy(odr);
+        wrbuf_destroy(k);
+        wrbuf_destroy(rec_sha1);
+    }
+#endif
 #if HAVE_LIBMEMCACHED_MEMCACHED_H
     if (r->connection->mc_st &&
         !diag && npr->which == Z_NamePlusRecord_databaseRecord)
@@ -313,6 +480,61 @@ Z_NamePlusRecord *ZOOM_memcached_lookup(ZOOM_resultset r, int pos,
                                         const char *elementSetName,
                                         const char *schema)
 {
+#if HAVE_HIREDIS
+    if (r->connection && r->connection->redis_c)
+    {
+        WRBUF k = wrbuf_alloc();
+        const char *argv[2];
+        size_t argvlen[2];
+        redisReply *reply1;
+
+        wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
+        wrbuf_printf(k, ";%d;%s;%s;%s", pos,
+                     syntax ? syntax : "",
+                     elementSetName ? elementSetName : "",
+                     schema ? schema : "");
+
+        yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
+        argv[0] = "GET";
+        argvlen[0] = 3;
+        argv[1] = wrbuf_buf(k);
+        argvlen[1] = wrbuf_len(k);
+        reply1 = redisCommandArgv(r->connection->redis_c, 2, argv, argvlen);
+
+        wrbuf_destroy(k);
+        if (reply1 && reply1->type == REDIS_REPLY_STRING)
+        {
+            redisReply *reply2;
+            char *sha1_buf = reply1->str;
+            int sha1_len = reply1->len;
+
+            yaz_log(YLOG_LOG, "Lookup record %.*s", (int) sha1_len, sha1_buf);
+
+            argv[0] = "GET";
+            argvlen[0] = 3;
+            argv[1] = sha1_buf;
+            argvlen[1] = sha1_len;
+
+            reply2 = redisCommandArgv(r->connection->redis_c, 2, argv, argvlen);
+            if (reply2 && reply2->type == REDIS_REPLY_STRING)
+            {
+                Z_NamePlusRecord *npr = 0;
+                char *v_buf = reply2->str;
+                int v_len = reply2->len;
+
+                odr_setbuf(r->odr, v_buf, v_len, 0);
+                z_NamePlusRecord(r->odr, &npr, 0, 0);
+                if (npr)
+                    yaz_log(YLOG_LOG, "returned redis copy");
+                freeReplyObject(reply2);
+                freeReplyObject(reply1);
+                return npr;
+            }
+            freeReplyObject(reply2);
+        }
+        freeReplyObject(reply1);
+    }
+#endif
 #if HAVE_LIBMEMCACHED_MEMCACHED_H
     if (r->connection && r->connection->mc_st)
     {

diff --git a/src/zoom-p.h b/src/zoom-p.h
index 5abaa2a..bb6db6f 100644 (file)
--- a/src/zoom-p.h
+++ b/src/zoom-p.h
@@ -39,6 +39,9 @@
 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 #include <libmemcached/memcached.h>
 #endif
+#if HAVE_HIREDIS
+#include <hiredis/hiredis.h>
+#endif
 
 #define SHPTR 1
 
@@ -115,6 +118,9 @@ struct ZOOM_connection_p {
 #if HAVE_LIBMEMCACHED_MEMCACHED_H
     memcached_st *mc_st;
 #endif
+#if HAVE_HIREDIS
+    redisContext *redis_c;
+#endif
 };
 
 typedef struct ZOOM_record_cache_p *ZOOM_record_cache;