src/zoom-memcached.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) Index Data
   3  * See the file LICENSE for details.
   4  */
   5 /**
   6  * \file zoom-memcached.c
   7  * \brief Implements query/record caching using memcached
   8  */
   9 #if HAVE_CONFIG_H
  10 #include <config.h>
  11 #endif
  12
  13 #include <assert.h>
  14 #include <string.h>
  15 #include <errno.h>
  16 #include "zoom-p.h"
  17
  18 #include <yaz/yaz-util.h>
  19 #include <yaz/xmalloc.h>
  20 #include <yaz/log.h>
  21 #include <yaz/diagbib1.h>
  22
  23 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  24 #if HAVE_MEMCACHED_RETURN_T
  25 #else
  26 typedef memcached_return memcached_return_t;
  27 #endif
  28 #endif
  29
  30 void ZOOM_memcached_init(ZOOM_connection c)
  31 {
  32 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  33     c->mc_st = 0;
  34 #endif
  35 }
  36
  37 void ZOOM_memcached_destroy(ZOOM_connection c)
  38 {
  39 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  40     if (c->mc_st)
  41         memcached_free(c->mc_st);
  42 #endif
  43 }
  44
  45 int ZOOM_memcached_configure(ZOOM_connection c)
  46 {
  47     const char *val;
  48 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  49     if (c->mc_st)
  50     {
  51         memcached_free(c->mc_st);
  52         c->mc_st = 0;
  53     }
  54 #endif
  55     val = ZOOM_options_get(c->options, "memcached");
  56     if (val && *val)
  57     {
  58 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  59         memcached_return_t rc;
  60
  61         c->mc_st = memcached_create(0);
  62         rc = memcached_server_add(c->mc_st, val, 11211);
  63         yaz_log(YLOG_LOG, "memcached_server_add host=%s rc=%u %s",
  64                 val, (unsigned) rc, memcached_strerror(c->mc_st, rc));
  65         if (rc != MEMCACHED_SUCCESS)
  66         {
  67             ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, val);
  68             memcached_free(c->mc_st);
  69             c->mc_st = 0;
  70             return -1;
  71         }
  72         memcached_behavior_set(c->mc_st, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1);
  73 #else
  74         ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
  75         return -1;
  76 #endif
  77     }
  78     return 0;
  79 }
  80
  81 void ZOOM_memcached_resultset(ZOOM_resultset r, ZOOM_query q)
  82 {
  83 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  84     ZOOM_connection c = r->connection;
  85     r->mc_key = wrbuf_alloc();
  86     wrbuf_puts(r->mc_key, "0;");
  87     wrbuf_puts(r->mc_key, c->host_port);
  88     wrbuf_puts(r->mc_key, ";");
  89     if (c->user)
  90         wrbuf_puts(r->mc_key, c->user);
  91     wrbuf_puts(r->mc_key, ";");
  92     if (c->group)
  93         wrbuf_puts(r->mc_key, c->group);
  94     wrbuf_puts(r->mc_key, ";");
  95     if (c->password)
  96         wrbuf_sha1_puts(r->mc_key, c->password, 1);
  97     wrbuf_puts(r->mc_key, ";");
  98     {
  99         WRBUF w = wrbuf_alloc();
 100         ZOOM_query_get_hash(q, w);
 101         wrbuf_sha1_puts(r->mc_key, wrbuf_cstr(w), 1);
 102         wrbuf_destroy(w);
 103     }
 104     wrbuf_puts(r->mc_key, ";");
 105     if (r->req_facets)
 106         wrbuf_puts(r->mc_key, r->req_facets);
 107 #endif
 108 }
 109
 110 void ZOOM_memcached_search(ZOOM_connection c, ZOOM_resultset resultset)
 111 {
 112 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 113     if (c->mc_st && resultset->live_set == 0)
 114     {
 115         size_t v_len;
 116         uint32_t flags;
 117         memcached_return_t rc;
 118         char *v = memcached_get(c->mc_st, wrbuf_buf(resultset->mc_key),
 119                                 wrbuf_len(resultset->mc_key),
 120                                 &v_len, &flags, &rc);
 121         /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
 122         if (v)
 123         {
 124             ZOOM_Event event;
 125             size_t lead_len = strlen(v) + 1;
 126
 127             resultset->size = odr_atoi(v);
 128
 129             yaz_log(YLOG_LOG, "For key %s got value %s lead_len=%d len=%d",
 130                     wrbuf_cstr(resultset->mc_key), v, (int) lead_len,
 131                     (int) v_len);
 132             if (v_len > lead_len)
 133             {
 134                 Z_OtherInformation *oi = 0;
 135                 int oi_len = v_len - lead_len;
 136                 odr_setbuf(resultset->odr, v + lead_len, oi_len, 0);
 137                 if (!z_OtherInformation(resultset->odr, &oi, 0, 0))
 138                 {
 139                     yaz_log(YLOG_WARN, "oi decoding failed");
 140                     free(v);
 141                     return;
 142                 }
 143                 ZOOM_handle_search_result(c, resultset, oi);
 144                 ZOOM_handle_facet_result(c, resultset, oi);
 145             }
 146             free(v);
 147             event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
 148             ZOOM_connection_put_event(c, event);
 149             resultset->live_set = 1;
 150         }
 151     }
 152 #endif
 153 }
 154
 155 void ZOOM_memcached_hitcount(ZOOM_connection c, ZOOM_resultset resultset,
 156                              Z_OtherInformation *oi, const char *precision)
 157 {
 158 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 159     if (c->mc_st && resultset->live_set == 0)
 160     {
 161         uint32_t flags = 0;
 162         memcached_return_t rc;
 163         time_t expiration = 36000;
 164         char *str;
 165         ODR odr = odr_createmem(ODR_ENCODE);
 166         char *oi_buf = 0;
 167         int oi_len = 0;
 168         char *key;
 169
 170         str = odr_malloc(odr, 20 + strlen(precision));
 171         /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
 172         sprintf(str, ODR_INT_PRINTF ";%s", resultset->size, precision);
 173         if (oi)
 174         {
 175             z_OtherInformation(odr, &oi, 0, 0);
 176             oi_buf = odr_getbuf(odr, &oi_len, 0);
 177         }
 178         key = odr_malloc(odr, strlen(str) + 1 + oi_len);
 179         strcpy(key, str);
 180         if (oi_len)
 181             memcpy(key + strlen(str) + 1, oi_buf, oi_len);
 182
 183         rc = memcached_set(c->mc_st,
 184                            wrbuf_buf(resultset->mc_key),
 185                            wrbuf_len(resultset->mc_key),
 186                            key, strlen(str) + 1 + oi_len, expiration, flags);
 187         yaz_log(YLOG_LOG, "Store hit count key=%s value=%s oi_len=%d rc=%u %s",
 188                 wrbuf_cstr(resultset->mc_key), str, oi_len, (unsigned) rc,
 189                 memcached_strerror(c->mc_st, rc));
 190         odr_destroy(odr);
 191     }
 192 #endif
 193 }
 194
 195 void ZOOM_memcached_add(ZOOM_resultset r, Z_NamePlusRecord *npr,
 196                         int pos,
 197                         const char *syntax, const char *elementSetName,
 198                         const char *schema,
 199                         Z_SRW_diagnostic *diag)
 200 {
 201 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 202     if (r->connection->mc_st &&
 203         !diag && npr->which == Z_NamePlusRecord_databaseRecord)
 204     {
 205         WRBUF k = wrbuf_alloc();
 206         WRBUF rec_sha1 = wrbuf_alloc();
 207         uint32_t flags = 0;
 208         memcached_return_t rc;
 209         time_t expiration = 36000;
 210         ODR odr = odr_createmem(ODR_ENCODE);
 211         char *rec_buf;
 212         int rec_len;
 213
 214         z_NamePlusRecord(odr, &npr, 0, 0);
 215         rec_buf = odr_getbuf(odr, &rec_len, 0);
 216
 217         wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
 218         wrbuf_printf(k, ";%d;%s;%s;%s", pos,
 219                      syntax ? syntax : "",
 220                      elementSetName ? elementSetName : "",
 221                      schema ? schema : "");
 222
 223         wrbuf_sha1_write(rec_sha1, rec_buf, rec_len, 1);
 224
 225         rc = memcached_set(r->connection->mc_st,
 226                            wrbuf_buf(k), wrbuf_len(k),
 227                            wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
 228                            expiration, flags);
 229
 230         yaz_log(YLOG_LOG, "Store record key=%s val=%s rc=%u %s",
 231                 wrbuf_cstr(k), wrbuf_cstr(rec_sha1), (unsigned) rc,
 232                 memcached_strerror(r->connection->mc_st, rc));
 233
 234         rc = memcached_add(r->connection->mc_st,
 235                            wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
 236                            rec_buf, rec_len,
 237                            expiration, flags);
 238
 239         yaz_log(YLOG_LOG, "Add record key=%s rec_len=%d rc=%u %s",
 240                 wrbuf_cstr(rec_sha1), rec_len, (unsigned) rc,
 241                 memcached_strerror(r->connection->mc_st, rc));
 242
 243         odr_destroy(odr);
 244         wrbuf_destroy(k);
 245         wrbuf_destroy(rec_sha1);
 246     }
 247 #endif
 248 }
 249
 250 Z_NamePlusRecord *ZOOM_memcached_lookup(ZOOM_resultset r, int pos,
 251                                         const char *syntax,
 252                                         const char *elementSetName,
 253                                         const char *schema)
 254 {
 255 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 256     if (r->connection && r->connection->mc_st)
 257     {
 258         WRBUF k = wrbuf_alloc();
 259         char *sha1_buf;
 260         size_t sha1_len;
 261         uint32_t flags;
 262         memcached_return_t rc;
 263
 264         wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
 265         wrbuf_printf(k, ";%d;%s;%s;%s", pos,
 266                      syntax ? syntax : "",
 267                      elementSetName ? elementSetName : "",
 268                      schema ? schema : "");
 269
 270         yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
 271         sha1_buf = memcached_get(r->connection->mc_st,
 272                                  wrbuf_buf(k), wrbuf_len(k),
 273                                  &sha1_len, &flags, &rc);
 274
 275         wrbuf_destroy(k);
 276         if (sha1_buf)
 277         {
 278             size_t v_len;
 279             char *v_buf;
 280
 281             yaz_log(YLOG_LOG, "Lookup record %.*s", (int) sha1_len, sha1_buf);
 282             v_buf = memcached_get(r->connection->mc_st, sha1_buf, sha1_len,
 283                                   &v_len, &flags, &rc);
 284             free(sha1_buf);
 285             if (v_buf)
 286             {
 287                 Z_NamePlusRecord *npr = 0;
 288
 289                 odr_setbuf(r->odr, v_buf, v_len, 0);
 290                 z_NamePlusRecord(r->odr, &npr, 0, 0);
 291                 free(v_buf);
 292                 if (npr)
 293                     yaz_log(YLOG_LOG, "returned memcached copy");
 294                 return npr;
 295             }
 296         }
 297     }
 298 #endif
 299     return 0;
 300
 301 }
 302 /*
 303  * Local variables:
 304  * c-basic-offset: 4
 305  * c-file-style: "Stroustrup"
 306  * indent-tabs-mode: nil
 307  * End:
 308  * vim: shiftwidth=4 tabstop=8 expandtab
 309  */
 310