1 /* This file is part of the YAZ toolkit.
2 * Copyright (C) Index Data
3 * See the file LICENSE for details.
6 * \file zoom-memcached.c
7 * \brief Implements query/record caching using memcached
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
21 #include <yaz/diagbib1.h>
24 #if LIBMEMCACHED_VERSION_HEX >= 0x01000000
25 #define HAVE_MEMCACHED_FUNC 1
27 #define HAVE_MEMCACHED_FUNC 0
31 void ZOOM_memcached_init(ZOOM_connection c)
41 void ZOOM_memcached_destroy(ZOOM_connection c)
45 memcached_free(c->mc_st);
49 redisFree(c->redis_c);
54 /* memcached wrapper.. Because memcached function do not exist in older libs */
55 static memcached_st *yaz_memcached_wrap(const char *conf)
57 #if HAVE_MEMCACHED_FUNC
58 return memcached(conf, strlen(conf));
62 memcached_st *mc = memcached_create(0);
63 NMEM nmem = nmem_create();
64 memcached_return_t rc;
66 nmem_strsplit_blank(nmem, conf, &darray, &num);
67 for (i = 0; mc && i < num; i++)
69 if (!yaz_strncasecmp(darray[i], "--SERVER=", 9))
71 char *host = darray[i] + 9;
72 char *port = strchr(host, ':');
73 char *weight = strstr(host, "/?");
81 rc = memcached_server_add(mc, host, port ? atoi(port) : 11211);
82 yaz_log(YLOG_LOG, "memcached_server_add host=%s rc=%u %s",
83 host, (unsigned) rc, memcached_strerror(mc, rc));
84 if (rc != MEMCACHED_SUCCESS)
104 static redisContext *create_redis(const char *conf)
108 NMEM nmem = nmem_create();
109 redisContext *context = 0;
111 nmem_strsplit_blank(nmem, conf, &darray, &num);
112 for (i = 0; i < num; i++)
114 if (!yaz_strncasecmp(darray[i], "--SERVER=", 9))
116 struct timeval timeout = { 1, 500000 }; /* 1.5 seconds */
117 char *host = darray[i] + 9;
118 char *port = strchr(host, ':');
121 context = redisConnectWithTimeout(host,
122 port ? atoi(port) : 6379,
131 int ZOOM_memcached_configure(ZOOM_connection c)
137 redisFree(c->redis_c);
141 #if HAVE_LIBMEMCACHED
144 memcached_free(c->mc_st);
149 val = ZOOM_options_get(c->options, "redis");
153 c->redis_c = create_redis(val);
154 if (c->redis_c == 0 || c->redis_c->err)
156 ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED,
157 "could not create redis");
160 return 0; /* don't bother with memcached if redis is enabled */
162 ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
166 val = ZOOM_options_get(c->options, "memcached");
169 #if HAVE_LIBMEMCACHED
170 c->mc_st = yaz_memcached_wrap(val);
173 ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED,
174 "could not create memcached");
177 memcached_behavior_set(c->mc_st, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1);
179 ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
187 static void wrbuf_vary_puts(WRBUF w, const char *v)
193 wrbuf_sha1_puts(w, v, 1);
203 void ZOOM_memcached_resultset(ZOOM_resultset r, ZOOM_query q)
206 ZOOM_connection c = r->connection;
208 r->mc_key = wrbuf_alloc();
209 wrbuf_puts(r->mc_key, "1;");
210 wrbuf_vary_puts(r->mc_key, c->host_port);
211 wrbuf_puts(r->mc_key, ";");
212 wrbuf_vary_puts(r->mc_key, ZOOM_resultset_option_get(r, "extraArgs"));
213 wrbuf_puts(r->mc_key, ";");
214 wrbuf_vary_puts(r->mc_key, c->user);
215 wrbuf_puts(r->mc_key, ";");
216 wrbuf_vary_puts(r->mc_key, c->group);
217 wrbuf_puts(r->mc_key, ";");
219 wrbuf_sha1_puts(r->mc_key, c->password, 1);
220 wrbuf_puts(r->mc_key, ";");
222 WRBUF w = wrbuf_alloc();
223 ZOOM_query_get_hash(q, w);
224 wrbuf_sha1_puts(r->mc_key, wrbuf_cstr(w), 1);
227 wrbuf_puts(r->mc_key, ";");
228 wrbuf_vary_puts(r->mc_key, r->req_facets);
232 void ZOOM_memcached_search(ZOOM_connection c, ZOOM_resultset resultset)
235 if (c->redis_c && resultset->live_set == 0)
241 argv[1] = wrbuf_cstr(resultset->mc_key);
243 reply = redisCommandArgv(c->redis_c, 2, argv, 0);
244 /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
245 if (reply && reply->type == REDIS_REPLY_STRING)
247 char *v = reply->str;
248 int v_len = reply->len;
250 size_t lead_len = strlen(v) + 1;
252 resultset->size = odr_atoi(v);
254 yaz_log(YLOG_LOG, "For key %s got value %s lead_len=%d len=%d",
255 wrbuf_cstr(resultset->mc_key), v, (int) lead_len,
257 if (v_len > lead_len)
259 Z_OtherInformation *oi = 0;
260 int oi_len = v_len - lead_len;
261 odr_setbuf(resultset->odr, v + lead_len, oi_len, 0);
262 if (!z_OtherInformation(resultset->odr, &oi, 0, 0))
264 yaz_log(YLOG_WARN, "oi decoding failed");
265 freeReplyObject(reply);
268 ZOOM_handle_search_result(c, resultset, oi);
269 ZOOM_handle_facet_result(c, resultset, oi);
271 event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
272 ZOOM_connection_put_event(c, event);
273 resultset->live_set = 1;
276 freeReplyObject(reply);
279 #if HAVE_LIBMEMCACHED
280 if (c->mc_st && resultset->live_set == 0)
284 memcached_return_t rc;
285 char *v = memcached_get(c->mc_st, wrbuf_buf(resultset->mc_key),
286 wrbuf_len(resultset->mc_key),
287 &v_len, &flags, &rc);
288 /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
292 size_t lead_len = strlen(v) + 1;
294 resultset->size = odr_atoi(v);
296 yaz_log(YLOG_LOG, "For key %s got value %s lead_len=%d len=%d",
297 wrbuf_cstr(resultset->mc_key), v, (int) lead_len,
299 if (v_len > lead_len)
301 Z_OtherInformation *oi = 0;
302 int oi_len = v_len - lead_len;
303 odr_setbuf(resultset->odr, v + lead_len, oi_len, 0);
304 if (!z_OtherInformation(resultset->odr, &oi, 0, 0))
306 yaz_log(YLOG_WARN, "oi decoding failed");
310 ZOOM_handle_search_result(c, resultset, oi);
311 ZOOM_handle_facet_result(c, resultset, oi);
314 event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
315 ZOOM_connection_put_event(c, event);
316 resultset->live_set = 1;
322 void ZOOM_memcached_hitcount(ZOOM_connection c, ZOOM_resultset resultset,
323 Z_OtherInformation *oi, const char *precision)
326 if (c->redis_c && resultset->live_set == 0)
329 ODR odr = odr_createmem(ODR_ENCODE);
334 str = odr_malloc(odr, 20 + strlen(precision));
335 /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
336 sprintf(str, ODR_INT_PRINTF ";%s", resultset->size, precision);
339 z_OtherInformation(odr, &oi, 0, 0);
340 oi_buf = odr_getbuf(odr, &oi_len, 0);
342 key = odr_malloc(odr, strlen(str) + 1 + oi_len);
345 memcpy(key + strlen(str) + 1, oi_buf, oi_len);
353 argv[1] = wrbuf_buf(resultset->mc_key);
354 argvlen[1] = wrbuf_len(resultset->mc_key);
356 argvlen[2] = strlen(str) + 1 + oi_len;
357 reply = redisCommandArgv(c->redis_c, 3, argv, argvlen);
358 freeReplyObject(reply);
363 #if HAVE_LIBMEMCACHED
364 if (c->mc_st && resultset->live_set == 0)
367 memcached_return_t rc;
368 time_t expiration = 36000;
370 ODR odr = odr_createmem(ODR_ENCODE);
375 str = odr_malloc(odr, 20 + strlen(precision));
376 /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
377 sprintf(str, ODR_INT_PRINTF ";%s", resultset->size, precision);
380 z_OtherInformation(odr, &oi, 0, 0);
381 oi_buf = odr_getbuf(odr, &oi_len, 0);
383 key = odr_malloc(odr, strlen(str) + 1 + oi_len);
386 memcpy(key + strlen(str) + 1, oi_buf, oi_len);
388 rc = memcached_set(c->mc_st,
389 wrbuf_buf(resultset->mc_key),
390 wrbuf_len(resultset->mc_key),
391 key, strlen(str) + 1 + oi_len, expiration, flags);
392 yaz_log(YLOG_LOG, "Store hit count key=%s value=%s oi_len=%d rc=%u %s",
393 wrbuf_cstr(resultset->mc_key), str, oi_len, (unsigned) rc,
394 memcached_strerror(c->mc_st, rc));
400 void ZOOM_memcached_add(ZOOM_resultset r, Z_NamePlusRecord *npr,
402 const char *syntax, const char *elementSetName,
404 Z_SRW_diagnostic *diag)
407 if (r->connection->redis_c &&
408 !diag && npr->which == Z_NamePlusRecord_databaseRecord)
410 WRBUF k = wrbuf_alloc();
411 WRBUF rec_sha1 = wrbuf_alloc();
412 ODR odr = odr_createmem(ODR_ENCODE);
419 z_NamePlusRecord(odr, &npr, 0, 0);
420 rec_buf = odr_getbuf(odr, &rec_len, 0);
422 wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
423 wrbuf_printf(k, ";%d;%s;%s;%s", pos,
424 syntax ? syntax : "",
425 elementSetName ? elementSetName : "",
426 schema ? schema : "");
428 wrbuf_sha1_write(rec_sha1, rec_buf, rec_len, 1);
432 argv[1] = wrbuf_buf(k);
433 argvlen[1] = wrbuf_len(k);
434 argv[2] = wrbuf_buf(rec_sha1);
435 argvlen[2] = wrbuf_len(rec_sha1);
437 reply = redisCommandArgv(r->connection->redis_c, 3, argv, argvlen);
438 yaz_log(YLOG_LOG, "Store record key=%s val=%s",
439 wrbuf_cstr(k), wrbuf_cstr(rec_sha1));
440 freeReplyObject(reply);
442 argv[1] = wrbuf_buf(rec_sha1);
443 argvlen[1] = wrbuf_len(rec_sha1);
445 argvlen[2] = rec_len;
447 reply = redisCommandArgv(r->connection->redis_c, 3, argv, argvlen);
448 yaz_log(YLOG_LOG, "Add record key=%s rec_len=%d",
449 wrbuf_cstr(rec_sha1), rec_len);
450 freeReplyObject(reply);
454 wrbuf_destroy(rec_sha1);
457 #if HAVE_LIBMEMCACHED
458 if (r->connection->mc_st &&
459 !diag && npr->which == Z_NamePlusRecord_databaseRecord)
461 WRBUF k = wrbuf_alloc();
462 WRBUF rec_sha1 = wrbuf_alloc();
464 memcached_return_t rc;
465 time_t expiration = 36000;
466 ODR odr = odr_createmem(ODR_ENCODE);
470 z_NamePlusRecord(odr, &npr, 0, 0);
471 rec_buf = odr_getbuf(odr, &rec_len, 0);
473 wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
474 wrbuf_printf(k, ";%d;%s;%s;%s", pos,
475 syntax ? syntax : "",
476 elementSetName ? elementSetName : "",
477 schema ? schema : "");
479 wrbuf_sha1_write(rec_sha1, rec_buf, rec_len, 1);
481 rc = memcached_set(r->connection->mc_st,
482 wrbuf_buf(k), wrbuf_len(k),
483 wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
486 yaz_log(YLOG_LOG, "Store record key=%s val=%s rc=%u %s",
487 wrbuf_cstr(k), wrbuf_cstr(rec_sha1), (unsigned) rc,
488 memcached_strerror(r->connection->mc_st, rc));
490 rc = memcached_add(r->connection->mc_st,
491 wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
495 yaz_log(YLOG_LOG, "Add record key=%s rec_len=%d rc=%u %s",
496 wrbuf_cstr(rec_sha1), rec_len, (unsigned) rc,
497 memcached_strerror(r->connection->mc_st, rc));
501 wrbuf_destroy(rec_sha1);
506 Z_NamePlusRecord *ZOOM_memcached_lookup(ZOOM_resultset r, int pos,
508 const char *elementSetName,
512 if (r->connection && r->connection->redis_c)
514 WRBUF k = wrbuf_alloc();
519 wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
520 wrbuf_printf(k, ";%d;%s;%s;%s", pos,
521 syntax ? syntax : "",
522 elementSetName ? elementSetName : "",
523 schema ? schema : "");
525 yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
528 argv[1] = wrbuf_buf(k);
529 argvlen[1] = wrbuf_len(k);
530 reply1 = redisCommandArgv(r->connection->redis_c, 2, argv, argvlen);
533 if (reply1 && reply1->type == REDIS_REPLY_STRING)
536 char *sha1_buf = reply1->str;
537 int sha1_len = reply1->len;
539 yaz_log(YLOG_LOG, "Lookup record %.*s", (int) sha1_len, sha1_buf);
544 argvlen[1] = sha1_len;
546 reply2 = redisCommandArgv(r->connection->redis_c, 2, argv, argvlen);
547 if (reply2 && reply2->type == REDIS_REPLY_STRING)
549 Z_NamePlusRecord *npr = 0;
550 char *v_buf = reply2->str;
551 int v_len = reply2->len;
553 odr_setbuf(r->odr, v_buf, v_len, 0);
554 z_NamePlusRecord(r->odr, &npr, 0, 0);
556 yaz_log(YLOG_LOG, "returned redis copy");
557 freeReplyObject(reply2);
558 freeReplyObject(reply1);
561 freeReplyObject(reply2);
563 freeReplyObject(reply1);
566 #if HAVE_LIBMEMCACHED
567 if (r->connection && r->connection->mc_st)
569 WRBUF k = wrbuf_alloc();
573 memcached_return_t rc;
575 wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
576 wrbuf_printf(k, ";%d;%s;%s;%s", pos,
577 syntax ? syntax : "",
578 elementSetName ? elementSetName : "",
579 schema ? schema : "");
581 yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
582 sha1_buf = memcached_get(r->connection->mc_st,
583 wrbuf_buf(k), wrbuf_len(k),
584 &sha1_len, &flags, &rc);
592 yaz_log(YLOG_LOG, "Lookup record %.*s", (int) sha1_len, sha1_buf);
593 v_buf = memcached_get(r->connection->mc_st, sha1_buf, sha1_len,
594 &v_len, &flags, &rc);
598 Z_NamePlusRecord *npr = 0;
600 odr_setbuf(r->odr, v_buf, v_len, 0);
601 z_NamePlusRecord(r->odr, &npr, 0, 0);
604 yaz_log(YLOG_LOG, "returned memcached copy");
616 * c-file-style: "Stroustrup"
617 * indent-tabs-mode: nil
619 * vim: shiftwidth=4 tabstop=8 expandtab