src/zoom-memcached.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) Index Data
   3  * See the file LICENSE for details.
   4  */
   5 /**
   6  * \file zoom-memcached.c
   7  * \brief Implements query/record caching using memcached
   8  */
   9 #if HAVE_CONFIG_H
  10 #include <config.h>
  11 #endif
  12
  13 #include <assert.h>
  14 #include <string.h>
  15 #include <errno.h>
  16 #include "zoom-p.h"
  17
  18 #include <yaz/yaz-util.h>
  19 #include <yaz/xmalloc.h>
  20 #include <yaz/log.h>
  21 #include <yaz/diagbib1.h>
  22
  23 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  24 #if HAVE_MEMCACHED_RETURN_T
  25 #else
  26 typedef memcached_return memcached_return_t;
  27 #endif
  28 #endif
  29
  30 void ZOOM_memcached_init(ZOOM_connection c)
  31 {
  32 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  33     c->mc_st = 0;
  34 #endif
  35 }
  36
  37 void ZOOM_memcached_destroy(ZOOM_connection c)
  38 {
  39 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  40     if (c->mc_st)
  41         memcached_free(c->mc_st);
  42 #endif
  43 }
  44
  45 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  46 /* memcached wrapper.. Because memcached function do not exist in older libs */
  47 static memcached_st *yaz_memcached_wrap(const char *conf)
  48 {
  49 #if HAVE_MEMCACHED_FUNC
  50     return memcached(conf, strlen(conf));
  51 #else
  52     char **darray;
  53     int i, num;
  54     memcached_st *mc = memcached_create(0);
  55     NMEM nmem = nmem_create();
  56     memcached_return_t rc;
  57
  58     nmem_strsplit_blank(nmem, conf, &darray, &num);
  59     for (i = 0; mc && i < num; i++)
  60     {
  61         if (!yaz_strncasecmp(darray[i], "--SERVER=", 9))
  62         {
  63             char *host = darray[i] + 9;
  64             char *port = strchr(host, ':');
  65             char *weight = strstr(host, "/?");
  66             if (port)
  67                 *port++ = '\0';
  68             if (weight)
  69             {
  70                 *weight = '\0';
  71                 weight += 2;
  72             }
  73             rc = memcached_server_add(mc, host, port ? atoi(port) : 11211);
  74             yaz_log(YLOG_LOG, "memcached_server_add host=%s rc=%u %s",
  75                     host, (unsigned) rc, memcached_strerror(mc, rc));
  76             if (rc != MEMCACHED_SUCCESS)
  77             {
  78                 memcached_free(mc);
  79                 mc = 0;
  80             }
  81         }
  82         else
  83         {
  84             /* bad directive */
  85             memcached_free(mc);
  86             mc = 0;
  87         }
  88     }
  89     nmem_destroy(nmem);
  90     return mc;
  91 #endif
  92 }
  93 #endif
  94
  95 int ZOOM_memcached_configure(ZOOM_connection c)
  96 {
  97     const char *val;
  98 #if HAVE_LIBMEMCACHED_MEMCACHED_H
  99     if (c->mc_st)
 100     {
 101         memcached_free(c->mc_st);
 102         c->mc_st = 0;
 103     }
 104 #endif
 105     val = ZOOM_options_get(c->options, "memcached");
 106     if (val && *val)
 107     {
 108 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 109         c->mc_st = yaz_memcached_wrap(val);
 110         if (!c->mc_st)
 111         {
 112             ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED,
 113                            "could not create memcached");
 114             return -1;
 115         }
 116         memcached_behavior_set(c->mc_st, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1);
 117 #else
 118         ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
 119         return -1;
 120 #endif
 121     }
 122     return 0;
 123 }
 124
 125 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 126 static void wrbuf_vary_puts(WRBUF w, const char *v)
 127 {
 128     if (v)
 129     {
 130         if (strlen(v) > 40)
 131         {
 132             wrbuf_sha1_puts(w, v, 1);
 133         }
 134         else
 135         {
 136             wrbuf_puts(w, v);
 137         }
 138     }
 139 }
 140 #endif
 141
 142 void ZOOM_memcached_resultset(ZOOM_resultset r, ZOOM_query q)
 143 {
 144 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 145     ZOOM_connection c = r->connection;
 146
 147     r->mc_key = wrbuf_alloc();
 148     wrbuf_puts(r->mc_key, "1;");
 149     wrbuf_vary_puts(r->mc_key, c->host_port);
 150     wrbuf_puts(r->mc_key, ";");
 151     wrbuf_vary_puts(r->mc_key, ZOOM_resultset_option_get(r, "extraArgs"));
 152     wrbuf_puts(r->mc_key, ";");
 153     wrbuf_vary_puts(r->mc_key, c->user);
 154     wrbuf_puts(r->mc_key, ";");
 155     wrbuf_vary_puts(r->mc_key, c->group);
 156     wrbuf_puts(r->mc_key, ";");
 157     if (c->password)
 158         wrbuf_sha1_puts(r->mc_key, c->password, 1);
 159     wrbuf_puts(r->mc_key, ";");
 160     {
 161         WRBUF w = wrbuf_alloc();
 162         ZOOM_query_get_hash(q, w);
 163         wrbuf_sha1_puts(r->mc_key, wrbuf_cstr(w), 1);
 164         wrbuf_destroy(w);
 165     }
 166     wrbuf_puts(r->mc_key, ";");
 167     wrbuf_vary_puts(r->mc_key, r->req_facets);
 168 #endif
 169 }
 170
 171 void ZOOM_memcached_search(ZOOM_connection c, ZOOM_resultset resultset)
 172 {
 173 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 174     if (c->mc_st && resultset->live_set == 0)
 175     {
 176         size_t v_len;
 177         uint32_t flags;
 178         memcached_return_t rc;
 179         char *v = memcached_get(c->mc_st, wrbuf_buf(resultset->mc_key),
 180                                 wrbuf_len(resultset->mc_key),
 181                                 &v_len, &flags, &rc);
 182         /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
 183         if (v)
 184         {
 185             ZOOM_Event event;
 186             size_t lead_len = strlen(v) + 1;
 187
 188             resultset->size = odr_atoi(v);
 189
 190             yaz_log(YLOG_LOG, "For key %s got value %s lead_len=%d len=%d",
 191                     wrbuf_cstr(resultset->mc_key), v, (int) lead_len,
 192                     (int) v_len);
 193             if (v_len > lead_len)
 194             {
 195                 Z_OtherInformation *oi = 0;
 196                 int oi_len = v_len - lead_len;
 197                 odr_setbuf(resultset->odr, v + lead_len, oi_len, 0);
 198                 if (!z_OtherInformation(resultset->odr, &oi, 0, 0))
 199                 {
 200                     yaz_log(YLOG_WARN, "oi decoding failed");
 201                     free(v);
 202                     return;
 203                 }
 204                 ZOOM_handle_search_result(c, resultset, oi);
 205                 ZOOM_handle_facet_result(c, resultset, oi);
 206             }
 207             free(v);
 208             event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
 209             ZOOM_connection_put_event(c, event);
 210             resultset->live_set = 1;
 211         }
 212     }
 213 #endif
 214 }
 215
 216 void ZOOM_memcached_hitcount(ZOOM_connection c, ZOOM_resultset resultset,
 217                              Z_OtherInformation *oi, const char *precision)
 218 {
 219 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 220     if (c->mc_st && resultset->live_set == 0)
 221     {
 222         uint32_t flags = 0;
 223         memcached_return_t rc;
 224         time_t expiration = 36000;
 225         char *str;
 226         ODR odr = odr_createmem(ODR_ENCODE);
 227         char *oi_buf = 0;
 228         int oi_len = 0;
 229         char *key;
 230
 231         str = odr_malloc(odr, 20 + strlen(precision));
 232         /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
 233         sprintf(str, ODR_INT_PRINTF ";%s", resultset->size, precision);
 234         if (oi)
 235         {
 236             z_OtherInformation(odr, &oi, 0, 0);
 237             oi_buf = odr_getbuf(odr, &oi_len, 0);
 238         }
 239         key = odr_malloc(odr, strlen(str) + 1 + oi_len);
 240         strcpy(key, str);
 241         if (oi_len)
 242             memcpy(key + strlen(str) + 1, oi_buf, oi_len);
 243
 244         rc = memcached_set(c->mc_st,
 245                            wrbuf_buf(resultset->mc_key),
 246                            wrbuf_len(resultset->mc_key),
 247                            key, strlen(str) + 1 + oi_len, expiration, flags);
 248         yaz_log(YLOG_LOG, "Store hit count key=%s value=%s oi_len=%d rc=%u %s",
 249                 wrbuf_cstr(resultset->mc_key), str, oi_len, (unsigned) rc,
 250                 memcached_strerror(c->mc_st, rc));
 251         odr_destroy(odr);
 252     }
 253 #endif
 254 }
 255
 256 void ZOOM_memcached_add(ZOOM_resultset r, Z_NamePlusRecord *npr,
 257                         int pos,
 258                         const char *syntax, const char *elementSetName,
 259                         const char *schema,
 260                         Z_SRW_diagnostic *diag)
 261 {
 262 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 263     if (r->connection->mc_st &&
 264         !diag && npr->which == Z_NamePlusRecord_databaseRecord)
 265     {
 266         WRBUF k = wrbuf_alloc();
 267         WRBUF rec_sha1 = wrbuf_alloc();
 268         uint32_t flags = 0;
 269         memcached_return_t rc;
 270         time_t expiration = 36000;
 271         ODR odr = odr_createmem(ODR_ENCODE);
 272         char *rec_buf;
 273         int rec_len;
 274
 275         z_NamePlusRecord(odr, &npr, 0, 0);
 276         rec_buf = odr_getbuf(odr, &rec_len, 0);
 277
 278         wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
 279         wrbuf_printf(k, ";%d;%s;%s;%s", pos,
 280                      syntax ? syntax : "",
 281                      elementSetName ? elementSetName : "",
 282                      schema ? schema : "");
 283
 284         wrbuf_sha1_write(rec_sha1, rec_buf, rec_len, 1);
 285
 286         rc = memcached_set(r->connection->mc_st,
 287                            wrbuf_buf(k), wrbuf_len(k),
 288                            wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
 289                            expiration, flags);
 290
 291         yaz_log(YLOG_LOG, "Store record key=%s val=%s rc=%u %s",
 292                 wrbuf_cstr(k), wrbuf_cstr(rec_sha1), (unsigned) rc,
 293                 memcached_strerror(r->connection->mc_st, rc));
 294
 295         rc = memcached_add(r->connection->mc_st,
 296                            wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
 297                            rec_buf, rec_len,
 298                            expiration, flags);
 299
 300         yaz_log(YLOG_LOG, "Add record key=%s rec_len=%d rc=%u %s",
 301                 wrbuf_cstr(rec_sha1), rec_len, (unsigned) rc,
 302                 memcached_strerror(r->connection->mc_st, rc));
 303
 304         odr_destroy(odr);
 305         wrbuf_destroy(k);
 306         wrbuf_destroy(rec_sha1);
 307     }
 308 #endif
 309 }
 310
 311 Z_NamePlusRecord *ZOOM_memcached_lookup(ZOOM_resultset r, int pos,
 312                                         const char *syntax,
 313                                         const char *elementSetName,
 314                                         const char *schema)
 315 {
 316 #if HAVE_LIBMEMCACHED_MEMCACHED_H
 317     if (r->connection && r->connection->mc_st)
 318     {
 319         WRBUF k = wrbuf_alloc();
 320         char *sha1_buf;
 321         size_t sha1_len;
 322         uint32_t flags;
 323         memcached_return_t rc;
 324
 325         wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
 326         wrbuf_printf(k, ";%d;%s;%s;%s", pos,
 327                      syntax ? syntax : "",
 328                      elementSetName ? elementSetName : "",
 329                      schema ? schema : "");
 330
 331         yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
 332         sha1_buf = memcached_get(r->connection->mc_st,
 333                                  wrbuf_buf(k), wrbuf_len(k),
 334                                  &sha1_len, &flags, &rc);
 335
 336         wrbuf_destroy(k);
 337         if (sha1_buf)
 338         {
 339             size_t v_len;
 340             char *v_buf;
 341
 342             yaz_log(YLOG_LOG, "Lookup record %.*s", (int) sha1_len, sha1_buf);
 343             v_buf = memcached_get(r->connection->mc_st, sha1_buf, sha1_len,
 344                                   &v_len, &flags, &rc);
 345             free(sha1_buf);
 346             if (v_buf)
 347             {
 348                 Z_NamePlusRecord *npr = 0;
 349
 350                 odr_setbuf(r->odr, v_buf, v_len, 0);
 351                 z_NamePlusRecord(r->odr, &npr, 0, 0);
 352                 free(v_buf);
 353                 if (npr)
 354                     yaz_log(YLOG_LOG, "returned memcached copy");
 355                 return npr;
 356             }
 357         }
 358     }
 359 #endif
 360     return 0;
 361
 362 }
 363 /*
 364  * Local variables:
 365  * c-basic-offset: 4
 366  * c-file-style: "Stroustrup"
 367  * indent-tabs-mode: nil
 368  * End:
 369  * vim: shiftwidth=4 tabstop=8 expandtab
 370  */
 371