src/zoom-memcached.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) Index Data
   3  * See the file LICENSE for details.
   4  */
   5 /**
   6  * \file zoom-memcached.c
   7  * \brief Implements query/record caching using memcached
   8  */
   9 #if HAVE_CONFIG_H
  10 #include <config.h>
  11 #endif
  12
  13 #include <assert.h>
  14 #include <string.h>
  15 #include <errno.h>
  16 #include "zoom-p.h"
  17
  18 #include <yaz/yaz-util.h>
  19 #include <yaz/xmalloc.h>
  20 #include <yaz/log.h>
  21 #include <yaz/diagbib1.h>
  22
  23 #if HAVE_MEMCACHED_RETURN_T
  24 #else
  25 typedef memcached_return memcached_return_t;
  26 #endif
  27
  28 void ZOOM_memcached_init(ZOOM_connection c)
  29 {
  30 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  31     c->mc_st = 0;
  32 #endif
  33 }
  34
  35 void ZOOM_memcached_destroy(ZOOM_connection c)
  36 {
  37 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  38     if (c->mc_st)
  39         memcached_free(c->mc_st);
  40 #endif
  41 }
  42
  43 int ZOOM_memcached_configure(ZOOM_connection c)
  44 {
  45     const char *val;
  46 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  47     if (c->mc_st)
  48     {
  49         memcached_free(c->mc_st);
  50         c->mc_st = 0;
  51     }
  52 #endif
  53     val = ZOOM_options_get(c->options, "memcached");
  54     if (val && *val)
  55     {
  56 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  57         memcached_return_t rc;
  58
  59         c->mc_st = memcached_create(0);
  60         rc = memcached_server_add(c->mc_st, val, 11211);
  61         yaz_log(YLOG_LOG, "memcached_server_add host=%s rc=%u %s",
  62                 val, (unsigned) rc, memcached_strerror(c->mc_st, rc));
  63         if (rc != MEMCACHED_SUCCESS)
  64         {
  65             ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, val);
  66             memcached_free(c->mc_st);
  67             c->mc_st = 0;
  68             return -1;
  69         }
  70         memcached_behavior_set(c->mc_st, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1);
  71 #else
  72         ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
  73         return -1;
  74 #endif
  75     }
  76     return 0;
  77 }
  78
  79 void ZOOM_memcached_resultset(ZOOM_resultset r, ZOOM_query q)
  80 {
  81 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  82     ZOOM_connection c = r->connection;
  83     r->mc_key = wrbuf_alloc();
  84     wrbuf_puts(r->mc_key, "0;");
  85     wrbuf_puts(r->mc_key, c->host_port);
  86     wrbuf_puts(r->mc_key, ";");
  87     if (c->user)
  88         wrbuf_puts(r->mc_key, c->user);
  89     wrbuf_puts(r->mc_key, ";");
  90     if (c->group)
  91         wrbuf_puts(r->mc_key, c->group);
  92     wrbuf_puts(r->mc_key, ";");
  93     if (c->password)
  94         wrbuf_sha1_puts(r->mc_key, c->password, 1);
  95     wrbuf_puts(r->mc_key, ";");
  96     {
  97         WRBUF w = wrbuf_alloc();
  98         ZOOM_query_get_hash(q, w);
  99         wrbuf_sha1_puts(r->mc_key, wrbuf_cstr(w), 1);
 100         wrbuf_destroy(w);
 101     }
 102     wrbuf_puts(r->mc_key, ";");
 103     if (r->req_facets)
 104         wrbuf_puts(r->mc_key, r->req_facets);
 105 #endif
 106 }
 107
 108 void ZOOM_memcached_search(ZOOM_connection c, ZOOM_resultset resultset)
 109 {
 110 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 111     if (c->mc_st && resultset->live_set == 0)
 112     {
 113         size_t v_len;
 114         uint32_t flags;
 115         memcached_return_t rc;
 116         char *v = memcached_get(c->mc_st, wrbuf_buf(resultset->mc_key),
 117                                 wrbuf_len(resultset->mc_key),
 118                                 &v_len, &flags, &rc);
 119         /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
 120         if (v)
 121         {
 122             ZOOM_Event event;
 123             size_t lead_len = strlen(v) + 1;
 124
 125             resultset->size = odr_atoi(v);
 126
 127             yaz_log(YLOG_LOG, "For key %s got value %s lead_len=%d len=%d",
 128                     wrbuf_cstr(resultset->mc_key), v, (int) lead_len,
 129                     (int) v_len);
 130             if (v_len > lead_len)
 131             {
 132                 Z_OtherInformation *oi = 0;
 133                 int oi_len = v_len - lead_len;
 134                 odr_setbuf(resultset->odr, v + lead_len, oi_len, 0);
 135                 if (!z_OtherInformation(resultset->odr, &oi, 0, 0))
 136                 {
 137                     yaz_log(YLOG_WARN, "oi decoding failed");
 138                     free(v);
 139                     return;
 140                 }
 141                 ZOOM_handle_search_result(c, resultset, oi);
 142                 ZOOM_handle_facet_result(c, resultset, oi);
 143             }
 144             free(v);
 145             event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
 146             ZOOM_connection_put_event(c, event);
 147             resultset->live_set = 1;
 148         }
 149     }
 150 #endif
 151 }
 152
 153 void ZOOM_memcached_hitcount(ZOOM_connection c, ZOOM_resultset resultset,
 154                              Z_OtherInformation *oi, const char *precision)
 155 {
 156 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 157     if (c->mc_st && resultset->live_set == 0)
 158     {
 159         uint32_t flags = 0;
 160         memcached_return_t rc;
 161         time_t expiration = 36000;
 162         char *str;
 163         ODR odr = odr_createmem(ODR_ENCODE);
 164         char *oi_buf = 0;
 165         int oi_len = 0;
 166         char *key;
 167
 168         str = odr_malloc(odr, 20 + strlen(precision));
 169         /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
 170         sprintf(str, ODR_INT_PRINTF ";%s", resultset->size, precision);
 171         if (oi)
 172         {
 173             z_OtherInformation(odr, &oi, 0, 0);
 174             oi_buf = odr_getbuf(odr, &oi_len, 0);
 175         }
 176         key = odr_malloc(odr, strlen(str) + 1 + oi_len);
 177         strcpy(key, str);
 178         if (oi_len)
 179             memcpy(key + strlen(str) + 1, oi_buf, oi_len);
 180
 181         rc = memcached_set(c->mc_st,
 182                            wrbuf_buf(resultset->mc_key),
 183                            wrbuf_len(resultset->mc_key),
 184                            key, strlen(str) + 1 + oi_len, expiration, flags);
 185         yaz_log(YLOG_LOG, "Store hit count key=%s value=%s oi_len=%d rc=%u %s",
 186                 wrbuf_cstr(resultset->mc_key), str, oi_len, (unsigned) rc,
 187                 memcached_strerror(c->mc_st, rc));
 188         odr_destroy(odr);
 189     }
 190 #endif
 191 }
 192
 193 void ZOOM_memcached_add(ZOOM_resultset r, Z_NamePlusRecord *npr,
 194                         int pos,
 195                         const char *syntax, const char *elementSetName,
 196                         const char *schema,
 197                         Z_SRW_diagnostic *diag)
 198 {
 199 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 200     if (r->connection->mc_st &&
 201         !diag && npr->which == Z_NamePlusRecord_databaseRecord)
 202     {
 203         WRBUF k = wrbuf_alloc();
 204         WRBUF rec_sha1 = wrbuf_alloc();
 205         uint32_t flags = 0;
 206         memcached_return_t rc;
 207         time_t expiration = 36000;
 208         ODR odr = odr_createmem(ODR_ENCODE);
 209         char *rec_buf;
 210         int rec_len;
 211
 212         z_NamePlusRecord(odr, &npr, 0, 0);
 213         rec_buf = odr_getbuf(odr, &rec_len, 0);
 214
 215         wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
 216         wrbuf_printf(k, ";%d;%s;%s;%s", pos,
 217                      syntax ? syntax : "",
 218                      elementSetName ? elementSetName : "",
 219                      schema ? schema : "");
 220
 221         wrbuf_sha1_write(rec_sha1, rec_buf, rec_len, 1);
 222
 223         rc = memcached_set(r->connection->mc_st,
 224                            wrbuf_buf(k), wrbuf_len(k),
 225                            wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
 226                            expiration, flags);
 227
 228         yaz_log(YLOG_LOG, "Store record key=%s val=%s rc=%u %s",
 229                 wrbuf_cstr(k), wrbuf_cstr(rec_sha1), (unsigned) rc,
 230                 memcached_strerror(r->connection->mc_st, rc));
 231
 232         rc = memcached_add(r->connection->mc_st,
 233                            wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
 234                            rec_buf, rec_len,
 235                            expiration, flags);
 236
 237         yaz_log(YLOG_LOG, "Add record key=%s rec_len=%d rc=%u %s",
 238                 wrbuf_cstr(rec_sha1), rec_len, (unsigned) rc,
 239                 memcached_strerror(r->connection->mc_st, rc));
 240
 241         odr_destroy(odr);
 242         wrbuf_destroy(k);
 243         wrbuf_destroy(rec_sha1);
 244     }
 245 #endif
 246 }
 247
 248 Z_NamePlusRecord *ZOOM_memcached_lookup(ZOOM_resultset r, int pos,
 249                                         const char *syntax,
 250                                         const char *elementSetName,
 251                                         const char *schema)
 252 {
 253 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 254     if (r->connection && r->connection->mc_st)
 255     {
 256         WRBUF k = wrbuf_alloc();
 257         char *sha1_buf;
 258         size_t sha1_len;
 259         uint32_t flags;
 260         memcached_return_t rc;
 261
 262         wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
 263         wrbuf_printf(k, ";%d;%s;%s;%s", pos,
 264                      syntax ? syntax : "",
 265                      elementSetName ? elementSetName : "",
 266                      schema ? schema : "");
 267
 268         yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
 269         sha1_buf = memcached_get(r->connection->mc_st,
 270                                  wrbuf_buf(k), wrbuf_len(k),
 271                                  &sha1_len, &flags, &rc);
 272
 273         wrbuf_destroy(k);
 274         if (sha1_buf)
 275         {
 276             size_t v_len;
 277             char *v_buf;
 278
 279             yaz_log(YLOG_LOG, "Lookup record %.*s", (int) sha1_len, sha1_buf);
 280             v_buf = memcached_get(r->connection->mc_st, sha1_buf, sha1_len,
 281                                   &v_len, &flags, &rc);
 282             free(sha1_buf);
 283             if (v_buf)
 284             {
 285                 Z_NamePlusRecord *npr = 0;
 286
 287                 odr_setbuf(r->odr, v_buf, v_len, 0);
 288                 z_NamePlusRecord(r->odr, &npr, 0, 0);
 289                 free(v_buf);
 290                 if (npr)
 291                     yaz_log(YLOG_LOG, "returned memcached copy");
 292                 return npr;
 293             }
 294         }
 295     }
 296 #endif
 297     return 0;
 298
 299 }
 300 /*
 301  * Local variables:
 302  * c-basic-offset: 4
 303  * c-file-style: "Stroustrup"
 304  * indent-tabs-mode: nil
 305  * End:
 306  * vim: shiftwidth=4 tabstop=8 expandtab
 307  */
 308