1 /* This file is part of the YAZ toolkit.
2 * Copyright (C) Index Data
3 * See the file LICENSE for details.
6 * \file zoom-memcached.c
7 * \brief Implements query/record caching using memcached
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
21 #include <yaz/diagbib1.h>
23 #if HAVE_LIBMEMCACHED_MEMCACHED_H
24 #if HAVE_MEMCACHED_RETURN_T
26 typedef memcached_return memcached_return_t;
30 void ZOOM_memcached_init(ZOOM_connection c)
32 #if HAVE_LIBMEMCACHED_MEMCACHED_H
40 void ZOOM_memcached_destroy(ZOOM_connection c)
42 #if HAVE_LIBMEMCACHED_MEMCACHED_H
44 memcached_free(c->mc_st);
48 redisFree(c->redis_c);
52 #if HAVE_LIBMEMCACHED_MEMCACHED_H
53 /* memcached wrapper.. Because memcached function do not exist in older libs */
54 static memcached_st *yaz_memcached_wrap(const char *conf)
56 #if HAVE_MEMCACHED_FUNC
57 return memcached(conf, strlen(conf));
61 memcached_st *mc = memcached_create(0);
62 NMEM nmem = nmem_create();
63 memcached_return_t rc;
65 nmem_strsplit_blank(nmem, conf, &darray, &num);
66 for (i = 0; mc && i < num; i++)
68 if (!yaz_strncasecmp(darray[i], "--SERVER=", 9))
70 char *host = darray[i] + 9;
71 char *port = strchr(host, ':');
72 char *weight = strstr(host, "/?");
80 rc = memcached_server_add(mc, host, port ? atoi(port) : 11211);
81 yaz_log(YLOG_LOG, "memcached_server_add host=%s rc=%u %s",
82 host, (unsigned) rc, memcached_strerror(mc, rc));
83 if (rc != MEMCACHED_SUCCESS)
102 int ZOOM_memcached_configure(ZOOM_connection c)
108 redisFree(c->redis_c);
112 #if HAVE_LIBMEMCACHED_MEMCACHED_H
115 memcached_free(c->mc_st);
120 val = ZOOM_options_get(c->options, "redis");
124 struct timeval timeout = { 1, 500000 }; /* 1.5 seconds */
126 c->redis_c = redisConnectWithTimeout(val, 6379, timeout);
127 if (c->redis_c == 0 || c->redis_c->err)
129 ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED,
130 "could not create redis");
133 return 0; /* don't bother with memcached if redis is enabled */
135 ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
139 val = ZOOM_options_get(c->options, "memcached");
142 #if HAVE_LIBMEMCACHED_MEMCACHED_H
143 c->mc_st = yaz_memcached_wrap(val);
146 ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED,
147 "could not create memcached");
150 memcached_behavior_set(c->mc_st, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1);
152 ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
160 static void wrbuf_vary_puts(WRBUF w, const char *v)
166 wrbuf_sha1_puts(w, v, 1);
176 void ZOOM_memcached_resultset(ZOOM_resultset r, ZOOM_query q)
179 ZOOM_connection c = r->connection;
181 r->mc_key = wrbuf_alloc();
182 wrbuf_puts(r->mc_key, "1;");
183 wrbuf_vary_puts(r->mc_key, c->host_port);
184 wrbuf_puts(r->mc_key, ";");
185 wrbuf_vary_puts(r->mc_key, ZOOM_resultset_option_get(r, "extraArgs"));
186 wrbuf_puts(r->mc_key, ";");
187 wrbuf_vary_puts(r->mc_key, c->user);
188 wrbuf_puts(r->mc_key, ";");
189 wrbuf_vary_puts(r->mc_key, c->group);
190 wrbuf_puts(r->mc_key, ";");
192 wrbuf_sha1_puts(r->mc_key, c->password, 1);
193 wrbuf_puts(r->mc_key, ";");
195 WRBUF w = wrbuf_alloc();
196 ZOOM_query_get_hash(q, w);
197 wrbuf_sha1_puts(r->mc_key, wrbuf_cstr(w), 1);
200 wrbuf_puts(r->mc_key, ";");
201 wrbuf_vary_puts(r->mc_key, r->req_facets);
205 void ZOOM_memcached_search(ZOOM_connection c, ZOOM_resultset resultset)
208 if (c->redis_c && resultset->live_set == 0)
214 argv[1] = wrbuf_cstr(resultset->mc_key);
216 reply = redisCommandArgv(c->redis_c, 2, argv, 0);
217 /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
218 if (reply && reply->type == REDIS_REPLY_STRING)
220 char *v = reply->str;
221 int v_len = reply->len;
223 size_t lead_len = strlen(v) + 1;
225 resultset->size = odr_atoi(v);
227 yaz_log(YLOG_LOG, "For key %s got value %s lead_len=%d len=%d",
228 wrbuf_cstr(resultset->mc_key), v, (int) lead_len,
230 if (v_len > lead_len)
232 Z_OtherInformation *oi = 0;
233 int oi_len = v_len - lead_len;
234 odr_setbuf(resultset->odr, v + lead_len, oi_len, 0);
235 if (!z_OtherInformation(resultset->odr, &oi, 0, 0))
237 yaz_log(YLOG_WARN, "oi decoding failed");
238 freeReplyObject(reply);
241 ZOOM_handle_search_result(c, resultset, oi);
242 ZOOM_handle_facet_result(c, resultset, oi);
244 event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
245 ZOOM_connection_put_event(c, event);
246 resultset->live_set = 1;
248 freeReplyObject(reply);
251 #if HAVE_LIBMEMCACHED_MEMCACHED_H
252 if (c->mc_st && resultset->live_set == 0)
256 memcached_return_t rc;
257 char *v = memcached_get(c->mc_st, wrbuf_buf(resultset->mc_key),
258 wrbuf_len(resultset->mc_key),
259 &v_len, &flags, &rc);
260 /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
264 size_t lead_len = strlen(v) + 1;
266 resultset->size = odr_atoi(v);
268 yaz_log(YLOG_LOG, "For key %s got value %s lead_len=%d len=%d",
269 wrbuf_cstr(resultset->mc_key), v, (int) lead_len,
271 if (v_len > lead_len)
273 Z_OtherInformation *oi = 0;
274 int oi_len = v_len - lead_len;
275 odr_setbuf(resultset->odr, v + lead_len, oi_len, 0);
276 if (!z_OtherInformation(resultset->odr, &oi, 0, 0))
278 yaz_log(YLOG_WARN, "oi decoding failed");
282 ZOOM_handle_search_result(c, resultset, oi);
283 ZOOM_handle_facet_result(c, resultset, oi);
286 event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
287 ZOOM_connection_put_event(c, event);
288 resultset->live_set = 1;
294 void ZOOM_memcached_hitcount(ZOOM_connection c, ZOOM_resultset resultset,
295 Z_OtherInformation *oi, const char *precision)
298 if (c->redis_c && resultset->live_set == 0)
301 ODR odr = odr_createmem(ODR_ENCODE);
306 str = odr_malloc(odr, 20 + strlen(precision));
307 /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
308 sprintf(str, ODR_INT_PRINTF ";%s", resultset->size, precision);
311 z_OtherInformation(odr, &oi, 0, 0);
312 oi_buf = odr_getbuf(odr, &oi_len, 0);
314 key = odr_malloc(odr, strlen(str) + 1 + oi_len);
317 memcpy(key + strlen(str) + 1, oi_buf, oi_len);
325 argv[1] = wrbuf_buf(resultset->mc_key);
326 argvlen[1] = wrbuf_len(resultset->mc_key);
328 argvlen[2] = strlen(str) + 1 + oi_len;
329 reply = redisCommandArgv(c->redis_c, 3, argv, argvlen);
330 freeReplyObject(reply);
335 #if HAVE_LIBMEMCACHED_MEMCACHED_H
336 if (c->mc_st && resultset->live_set == 0)
339 memcached_return_t rc;
340 time_t expiration = 36000;
342 ODR odr = odr_createmem(ODR_ENCODE);
347 str = odr_malloc(odr, 20 + strlen(precision));
348 /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
349 sprintf(str, ODR_INT_PRINTF ";%s", resultset->size, precision);
352 z_OtherInformation(odr, &oi, 0, 0);
353 oi_buf = odr_getbuf(odr, &oi_len, 0);
355 key = odr_malloc(odr, strlen(str) + 1 + oi_len);
358 memcpy(key + strlen(str) + 1, oi_buf, oi_len);
360 rc = memcached_set(c->mc_st,
361 wrbuf_buf(resultset->mc_key),
362 wrbuf_len(resultset->mc_key),
363 key, strlen(str) + 1 + oi_len, expiration, flags);
364 yaz_log(YLOG_LOG, "Store hit count key=%s value=%s oi_len=%d rc=%u %s",
365 wrbuf_cstr(resultset->mc_key), str, oi_len, (unsigned) rc,
366 memcached_strerror(c->mc_st, rc));
372 void ZOOM_memcached_add(ZOOM_resultset r, Z_NamePlusRecord *npr,
374 const char *syntax, const char *elementSetName,
376 Z_SRW_diagnostic *diag)
379 if (r->connection->redis_c &&
380 !diag && npr->which == Z_NamePlusRecord_databaseRecord)
382 WRBUF k = wrbuf_alloc();
383 WRBUF rec_sha1 = wrbuf_alloc();
384 ODR odr = odr_createmem(ODR_ENCODE);
391 z_NamePlusRecord(odr, &npr, 0, 0);
392 rec_buf = odr_getbuf(odr, &rec_len, 0);
394 wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
395 wrbuf_printf(k, ";%d;%s;%s;%s", pos,
396 syntax ? syntax : "",
397 elementSetName ? elementSetName : "",
398 schema ? schema : "");
400 wrbuf_sha1_write(rec_sha1, rec_buf, rec_len, 1);
404 argv[1] = wrbuf_buf(k);
405 argvlen[1] = wrbuf_len(k);
406 argv[2] = wrbuf_buf(rec_sha1);
407 argvlen[2] = wrbuf_len(rec_sha1);
409 reply = redisCommandArgv(r->connection->redis_c, 3, argv, argvlen);
410 yaz_log(YLOG_LOG, "Store record key=%s val=%s",
411 wrbuf_cstr(k), wrbuf_cstr(rec_sha1));
412 freeReplyObject(reply);
414 argv[1] = wrbuf_buf(rec_sha1);
415 argvlen[1] = wrbuf_len(rec_sha1);
417 argvlen[2] = rec_len;
419 reply = redisCommandArgv(r->connection->redis_c, 3, argv, argvlen);
420 yaz_log(YLOG_LOG, "Add record key=%s rec_len=%d",
421 wrbuf_cstr(rec_sha1), rec_len);
422 freeReplyObject(reply);
426 wrbuf_destroy(rec_sha1);
429 #if HAVE_LIBMEMCACHED_MEMCACHED_H
430 if (r->connection->mc_st &&
431 !diag && npr->which == Z_NamePlusRecord_databaseRecord)
433 WRBUF k = wrbuf_alloc();
434 WRBUF rec_sha1 = wrbuf_alloc();
436 memcached_return_t rc;
437 time_t expiration = 36000;
438 ODR odr = odr_createmem(ODR_ENCODE);
442 z_NamePlusRecord(odr, &npr, 0, 0);
443 rec_buf = odr_getbuf(odr, &rec_len, 0);
445 wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
446 wrbuf_printf(k, ";%d;%s;%s;%s", pos,
447 syntax ? syntax : "",
448 elementSetName ? elementSetName : "",
449 schema ? schema : "");
451 wrbuf_sha1_write(rec_sha1, rec_buf, rec_len, 1);
453 rc = memcached_set(r->connection->mc_st,
454 wrbuf_buf(k), wrbuf_len(k),
455 wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
458 yaz_log(YLOG_LOG, "Store record key=%s val=%s rc=%u %s",
459 wrbuf_cstr(k), wrbuf_cstr(rec_sha1), (unsigned) rc,
460 memcached_strerror(r->connection->mc_st, rc));
462 rc = memcached_add(r->connection->mc_st,
463 wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
467 yaz_log(YLOG_LOG, "Add record key=%s rec_len=%d rc=%u %s",
468 wrbuf_cstr(rec_sha1), rec_len, (unsigned) rc,
469 memcached_strerror(r->connection->mc_st, rc));
473 wrbuf_destroy(rec_sha1);
478 Z_NamePlusRecord *ZOOM_memcached_lookup(ZOOM_resultset r, int pos,
480 const char *elementSetName,
484 if (r->connection && r->connection->redis_c)
486 WRBUF k = wrbuf_alloc();
491 wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
492 wrbuf_printf(k, ";%d;%s;%s;%s", pos,
493 syntax ? syntax : "",
494 elementSetName ? elementSetName : "",
495 schema ? schema : "");
497 yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
500 argv[1] = wrbuf_buf(k);
501 argvlen[1] = wrbuf_len(k);
502 reply1 = redisCommandArgv(r->connection->redis_c, 2, argv, argvlen);
505 if (reply1 && reply1->type == REDIS_REPLY_STRING)
508 char *sha1_buf = reply1->str;
509 int sha1_len = reply1->len;
511 yaz_log(YLOG_LOG, "Lookup record %.*s", (int) sha1_len, sha1_buf);
516 argvlen[1] = sha1_len;
518 reply2 = redisCommandArgv(r->connection->redis_c, 2, argv, argvlen);
519 if (reply2 && reply2->type == REDIS_REPLY_STRING)
521 Z_NamePlusRecord *npr = 0;
522 char *v_buf = reply2->str;
523 int v_len = reply2->len;
525 odr_setbuf(r->odr, v_buf, v_len, 0);
526 z_NamePlusRecord(r->odr, &npr, 0, 0);
528 yaz_log(YLOG_LOG, "returned redis copy");
529 freeReplyObject(reply2);
530 freeReplyObject(reply1);
533 freeReplyObject(reply2);
535 freeReplyObject(reply1);
538 #if HAVE_LIBMEMCACHED_MEMCACHED_H
539 if (r->connection && r->connection->mc_st)
541 WRBUF k = wrbuf_alloc();
545 memcached_return_t rc;
547 wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
548 wrbuf_printf(k, ";%d;%s;%s;%s", pos,
549 syntax ? syntax : "",
550 elementSetName ? elementSetName : "",
551 schema ? schema : "");
553 yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
554 sha1_buf = memcached_get(r->connection->mc_st,
555 wrbuf_buf(k), wrbuf_len(k),
556 &sha1_len, &flags, &rc);
564 yaz_log(YLOG_LOG, "Lookup record %.*s", (int) sha1_len, sha1_buf);
565 v_buf = memcached_get(r->connection->mc_st, sha1_buf, sha1_len,
566 &v_len, &flags, &rc);
570 Z_NamePlusRecord *npr = 0;
572 odr_setbuf(r->odr, v_buf, v_len, 0);
573 z_NamePlusRecord(r->odr, &npr, 0, 0);
576 yaz_log(YLOG_LOG, "returned memcached copy");
588 * c-file-style: "Stroustrup"
589 * indent-tabs-mode: nil
591 * vim: shiftwidth=4 tabstop=8 expandtab