Fix null ptr reference for freeReplyObject YAZ-773
[yaz-moved-to-github.git] / src / zoom-memcached.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file zoom-memcached.c
7  * \brief Implements query/record caching using memcached
8  */
9 #if HAVE_CONFIG_H
10 #include <config.h>
11 #endif
12
13 #include <assert.h>
14 #include <string.h>
15 #include <errno.h>
16 #include "zoom-p.h"
17
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
20 #include <yaz/log.h>
21 #include <yaz/diagbib1.h>
22
23 #if HAVE_LIBMEMCACHED
24 #if LIBMEMCACHED_VERSION_HEX >= 0x01000000
25 #define HAVE_MEMCACHED_FUNC 1
26 #else
27 #define HAVE_MEMCACHED_FUNC 0
28 #endif
29 #endif
30
31 void ZOOM_memcached_init(ZOOM_connection c)
32 {
33 #if HAVE_LIBMEMCACHED
34     c->mc_st = 0;
35 #endif
36 #if HAVE_HIREDIS
37     c->redis_c = 0;
38 #endif
39 }
40
41 void ZOOM_memcached_destroy(ZOOM_connection c)
42 {
43 #if HAVE_LIBMEMCACHED
44     if (c->mc_st)
45         memcached_free(c->mc_st);
46 #endif
47 #if HAVE_HIREDIS
48     if (c->redis_c)
49         redisFree(c->redis_c);
50 #endif
51 }
52
53 #if HAVE_LIBMEMCACHED
54 /* memcached wrapper.. Because memcached function do not exist in older libs */
55 static memcached_st *yaz_memcached_wrap(const char *conf)
56 {
57 #if HAVE_MEMCACHED_FUNC
58     return memcached(conf, strlen(conf));
59 #else
60     char **darray;
61     int i, num;
62     memcached_st *mc = memcached_create(0);
63     NMEM nmem = nmem_create();
64     memcached_return_t rc;
65
66     nmem_strsplit_blank(nmem, conf, &darray, &num);
67     for (i = 0; mc && i < num; i++)
68     {
69         if (!yaz_strncasecmp(darray[i], "--SERVER=", 9))
70         {
71             char *host = darray[i] + 9;
72             char *port = strchr(host, ':');
73             char *weight = strstr(host, "/?");
74             if (port)
75                 *port++ = '\0';
76             if (weight)
77             {
78                 *weight = '\0';
79                 weight += 2;
80             }
81             rc = memcached_server_add(mc, host, port ? atoi(port) : 11211);
82             yaz_log(YLOG_LOG, "memcached_server_add host=%s rc=%u %s",
83                     host, (unsigned) rc, memcached_strerror(mc, rc));
84             if (rc != MEMCACHED_SUCCESS)
85             {
86                 memcached_free(mc);
87                 mc = 0;
88             }
89         }
90         else
91         {
92             /* bad directive */
93             memcached_free(mc);
94             mc = 0;
95         }
96     }
97     nmem_destroy(nmem);
98     return mc;
99 #endif
100 }
101 #endif
102
103 #if HAVE_HIREDIS
104 static redisContext *create_redis(const char *conf)
105 {
106     char **darray;
107     int i, num;
108     NMEM nmem = nmem_create();
109     redisContext *context = 0;
110
111     nmem_strsplit_blank(nmem, conf, &darray, &num);
112     for (i = 0; i < num; i++)
113     {
114         if (!yaz_strncasecmp(darray[i], "--SERVER=", 9))
115         {
116             struct timeval timeout = { 1, 500000 }; /* 1.5 seconds */
117             char *host = darray[i] + 9;
118             char *port = strchr(host, ':');
119             if (port)
120                 *port++ = '\0';
121             context = redisConnectWithTimeout(host,
122                                               port ? atoi(port) : 6379,
123                                               timeout);
124         }
125     }
126     nmem_destroy(nmem);
127     return context;
128 }
129 #endif
130
131 int ZOOM_memcached_configure(ZOOM_connection c)
132 {
133     const char *val;
134 #if HAVE_HIREDIS
135     if (c->redis_c)
136     {
137         redisFree(c->redis_c);
138         c->redis_c = 0;
139     }
140 #endif
141 #if HAVE_LIBMEMCACHED
142     if (c->mc_st)
143     {
144         memcached_free(c->mc_st);
145         c->mc_st = 0;
146     }
147 #endif
148
149     val = ZOOM_options_get(c->options, "redis");
150     if (val && *val)
151     {
152 #if HAVE_HIREDIS
153         c->redis_c = create_redis(val);
154         if (c->redis_c == 0 || c->redis_c->err)
155         {
156             ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED,
157                            "could not create redis");
158             return -1;
159         }
160         return 0; /* don't bother with memcached if redis is enabled */
161 #else
162         ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
163         return -1;
164 #endif
165     }
166     val = ZOOM_options_get(c->options, "memcached");
167     if (val && *val)
168     {
169 #if HAVE_LIBMEMCACHED
170         c->mc_st = yaz_memcached_wrap(val);
171         if (!c->mc_st)
172         {
173             ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED,
174                            "could not create memcached");
175             return -1;
176         }
177         memcached_behavior_set(c->mc_st, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1);
178 #else
179         ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
180         return -1;
181 #endif
182     }
183     return 0;
184 }
185
186 #if HAVE_GCRYPT_H
187 static void wrbuf_vary_puts(WRBUF w, const char *v)
188 {
189     if (v)
190     {
191         if (strlen(v) > 40)
192         {
193             wrbuf_sha1_puts(w, v, 1);
194         }
195         else
196         {
197             wrbuf_puts(w, v);
198         }
199     }
200 }
201 #endif
202
203 void ZOOM_memcached_resultset(ZOOM_resultset r, ZOOM_query q)
204 {
205 #if HAVE_GCRYPT_H
206     ZOOM_connection c = r->connection;
207
208     r->mc_key = wrbuf_alloc();
209     wrbuf_puts(r->mc_key, "1;");
210     wrbuf_vary_puts(r->mc_key, c->host_port);
211     wrbuf_puts(r->mc_key, ";");
212     wrbuf_vary_puts(r->mc_key, ZOOM_resultset_option_get(r, "extraArgs"));
213     wrbuf_puts(r->mc_key, ";");
214     wrbuf_vary_puts(r->mc_key, c->user);
215     wrbuf_puts(r->mc_key, ";");
216     wrbuf_vary_puts(r->mc_key, c->group);
217     wrbuf_puts(r->mc_key, ";");
218     if (c->password)
219         wrbuf_sha1_puts(r->mc_key, c->password, 1);
220     wrbuf_puts(r->mc_key, ";");
221     {
222         WRBUF w = wrbuf_alloc();
223         ZOOM_query_get_hash(q, w);
224         wrbuf_sha1_puts(r->mc_key, wrbuf_cstr(w), 1);
225         wrbuf_destroy(w);
226     }
227     wrbuf_puts(r->mc_key, ";");
228     wrbuf_vary_puts(r->mc_key, r->req_facets);
229 #endif
230 }
231
232 void ZOOM_memcached_search(ZOOM_connection c, ZOOM_resultset resultset)
233 {
234 #if HAVE_HIREDIS
235     if (c->redis_c && resultset->live_set == 0)
236     {
237         redisReply *reply;
238         const char *argv[2];
239
240         argv[0] = "GET";
241         argv[1] = wrbuf_cstr(resultset->mc_key);
242
243         reply = redisCommandArgv(c->redis_c, 2, argv, 0);
244         /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
245         if (reply && reply->type == REDIS_REPLY_STRING)
246         {
247             char *v = reply->str;
248             int v_len = reply->len;
249             ZOOM_Event event;
250             size_t lead_len = strlen(v) + 1;
251
252             resultset->size = odr_atoi(v);
253
254             yaz_log(YLOG_LOG, "For key %s got value %s lead_len=%d len=%d",
255                     wrbuf_cstr(resultset->mc_key), v, (int) lead_len,
256                     (int) v_len);
257             if (v_len > lead_len)
258             {
259                 Z_OtherInformation *oi = 0;
260                 int oi_len = v_len - lead_len;
261                 odr_setbuf(resultset->odr, v + lead_len, oi_len, 0);
262                 if (!z_OtherInformation(resultset->odr, &oi, 0, 0))
263                 {
264                     yaz_log(YLOG_WARN, "oi decoding failed");
265                     freeReplyObject(reply);
266                     return;
267                 }
268                 ZOOM_handle_search_result(c, resultset, oi);
269                 ZOOM_handle_facet_result(c, resultset, oi);
270             }
271             event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
272             ZOOM_connection_put_event(c, event);
273             resultset->live_set = 1;
274         }
275         if (reply)
276             freeReplyObject(reply);
277     }
278 #endif
279 #if HAVE_LIBMEMCACHED
280     if (c->mc_st && resultset->live_set == 0)
281     {
282         size_t v_len;
283         uint32_t flags;
284         memcached_return_t rc;
285         char *v = memcached_get(c->mc_st, wrbuf_buf(resultset->mc_key),
286                                 wrbuf_len(resultset->mc_key),
287                                 &v_len, &flags, &rc);
288         /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
289         if (v)
290         {
291             ZOOM_Event event;
292             size_t lead_len = strlen(v) + 1;
293
294             resultset->size = odr_atoi(v);
295
296             yaz_log(YLOG_LOG, "For key %s got value %s lead_len=%d len=%d",
297                     wrbuf_cstr(resultset->mc_key), v, (int) lead_len,
298                     (int) v_len);
299             if (v_len > lead_len)
300             {
301                 Z_OtherInformation *oi = 0;
302                 int oi_len = v_len - lead_len;
303                 odr_setbuf(resultset->odr, v + lead_len, oi_len, 0);
304                 if (!z_OtherInformation(resultset->odr, &oi, 0, 0))
305                 {
306                     yaz_log(YLOG_WARN, "oi decoding failed");
307                     free(v);
308                     return;
309                 }
310                 ZOOM_handle_search_result(c, resultset, oi);
311                 ZOOM_handle_facet_result(c, resultset, oi);
312             }
313             free(v);
314             event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
315             ZOOM_connection_put_event(c, event);
316             resultset->live_set = 1;
317         }
318     }
319 #endif
320 }
321
322 void ZOOM_memcached_hitcount(ZOOM_connection c, ZOOM_resultset resultset,
323                              Z_OtherInformation *oi, const char *precision)
324 {
325 #if HAVE_HIREDIS
326     if (c->redis_c && resultset->live_set == 0)
327     {
328         char *str;
329         ODR odr = odr_createmem(ODR_ENCODE);
330         char *oi_buf = 0;
331         int oi_len = 0;
332         char *key;
333
334         str = odr_malloc(odr, 20 + strlen(precision));
335         /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
336         sprintf(str, ODR_INT_PRINTF ";%s", resultset->size, precision);
337         if (oi)
338         {
339             z_OtherInformation(odr, &oi, 0, 0);
340             oi_buf = odr_getbuf(odr, &oi_len, 0);
341         }
342         key = odr_malloc(odr, strlen(str) + 1 + oi_len);
343         strcpy(key, str);
344         if (oi_len)
345             memcpy(key + strlen(str) + 1, oi_buf, oi_len);
346
347         {
348             redisReply *reply;
349             const char *argv[3];
350             size_t argvlen[3];
351             argv[0] = "SET";
352             argvlen[0] = 3;
353             argv[1] = wrbuf_buf(resultset->mc_key);
354             argvlen[1] = wrbuf_len(resultset->mc_key);
355             argv[2] = key;
356             argvlen[2] = strlen(str) + 1 + oi_len;
357             reply = redisCommandArgv(c->redis_c, 3, argv, argvlen);
358             freeReplyObject(reply);
359         }
360         odr_destroy(odr);
361     }
362 #endif
363 #if HAVE_LIBMEMCACHED
364     if (c->mc_st && resultset->live_set == 0)
365     {
366         uint32_t flags = 0;
367         memcached_return_t rc;
368         time_t expiration = 36000;
369         char *str;
370         ODR odr = odr_createmem(ODR_ENCODE);
371         char *oi_buf = 0;
372         int oi_len = 0;
373         char *key;
374
375         str = odr_malloc(odr, 20 + strlen(precision));
376         /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
377         sprintf(str, ODR_INT_PRINTF ";%s", resultset->size, precision);
378         if (oi)
379         {
380             z_OtherInformation(odr, &oi, 0, 0);
381             oi_buf = odr_getbuf(odr, &oi_len, 0);
382         }
383         key = odr_malloc(odr, strlen(str) + 1 + oi_len);
384         strcpy(key, str);
385         if (oi_len)
386             memcpy(key + strlen(str) + 1, oi_buf, oi_len);
387
388         rc = memcached_set(c->mc_st,
389                            wrbuf_buf(resultset->mc_key),
390                            wrbuf_len(resultset->mc_key),
391                            key, strlen(str) + 1 + oi_len, expiration, flags);
392         yaz_log(YLOG_LOG, "Store hit count key=%s value=%s oi_len=%d rc=%u %s",
393                 wrbuf_cstr(resultset->mc_key), str, oi_len, (unsigned) rc,
394                 memcached_strerror(c->mc_st, rc));
395         odr_destroy(odr);
396     }
397 #endif
398 }
399
400 void ZOOM_memcached_add(ZOOM_resultset r, Z_NamePlusRecord *npr,
401                         int pos,
402                         const char *syntax, const char *elementSetName,
403                         const char *schema,
404                         Z_SRW_diagnostic *diag)
405 {
406 #if HAVE_HIREDIS
407     if (r->connection->redis_c &&
408         !diag && npr->which == Z_NamePlusRecord_databaseRecord)
409     {
410         WRBUF k = wrbuf_alloc();
411         WRBUF rec_sha1 = wrbuf_alloc();
412         ODR odr = odr_createmem(ODR_ENCODE);
413         char *rec_buf;
414         int rec_len;
415         const char *argv[3];
416         size_t argvlen[3];
417         redisReply *reply;
418
419         z_NamePlusRecord(odr, &npr, 0, 0);
420         rec_buf = odr_getbuf(odr, &rec_len, 0);
421
422         wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
423         wrbuf_printf(k, ";%d;%s;%s;%s", pos,
424                      syntax ? syntax : "",
425                      elementSetName ? elementSetName : "",
426                      schema ? schema : "");
427
428         wrbuf_sha1_write(rec_sha1, rec_buf, rec_len, 1);
429
430         argv[0] = "SET";
431         argvlen[0] = 3;
432         argv[1] = wrbuf_buf(k);
433         argvlen[1] = wrbuf_len(k);
434         argv[2] = wrbuf_buf(rec_sha1);
435         argvlen[2] = wrbuf_len(rec_sha1);
436
437         reply = redisCommandArgv(r->connection->redis_c, 3, argv, argvlen);
438         yaz_log(YLOG_LOG, "Store record key=%s val=%s",
439                 wrbuf_cstr(k), wrbuf_cstr(rec_sha1));
440         freeReplyObject(reply);
441
442         argv[1] = wrbuf_buf(rec_sha1);
443         argvlen[1] = wrbuf_len(rec_sha1);
444         argv[2] = rec_buf;
445         argvlen[2] = rec_len;
446
447         reply = redisCommandArgv(r->connection->redis_c, 3, argv, argvlen);
448         yaz_log(YLOG_LOG, "Add record key=%s rec_len=%d",
449                 wrbuf_cstr(rec_sha1), rec_len);
450         freeReplyObject(reply);
451
452         odr_destroy(odr);
453         wrbuf_destroy(k);
454         wrbuf_destroy(rec_sha1);
455     }
456 #endif
457 #if HAVE_LIBMEMCACHED
458     if (r->connection->mc_st &&
459         !diag && npr->which == Z_NamePlusRecord_databaseRecord)
460     {
461         WRBUF k = wrbuf_alloc();
462         WRBUF rec_sha1 = wrbuf_alloc();
463         uint32_t flags = 0;
464         memcached_return_t rc;
465         time_t expiration = 36000;
466         ODR odr = odr_createmem(ODR_ENCODE);
467         char *rec_buf;
468         int rec_len;
469
470         z_NamePlusRecord(odr, &npr, 0, 0);
471         rec_buf = odr_getbuf(odr, &rec_len, 0);
472
473         wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
474         wrbuf_printf(k, ";%d;%s;%s;%s", pos,
475                      syntax ? syntax : "",
476                      elementSetName ? elementSetName : "",
477                      schema ? schema : "");
478
479         wrbuf_sha1_write(rec_sha1, rec_buf, rec_len, 1);
480
481         rc = memcached_set(r->connection->mc_st,
482                            wrbuf_buf(k), wrbuf_len(k),
483                            wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
484                            expiration, flags);
485
486         yaz_log(YLOG_LOG, "Store record key=%s val=%s rc=%u %s",
487                 wrbuf_cstr(k), wrbuf_cstr(rec_sha1), (unsigned) rc,
488                 memcached_strerror(r->connection->mc_st, rc));
489
490         rc = memcached_add(r->connection->mc_st,
491                            wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
492                            rec_buf, rec_len,
493                            expiration, flags);
494
495         yaz_log(YLOG_LOG, "Add record key=%s rec_len=%d rc=%u %s",
496                 wrbuf_cstr(rec_sha1), rec_len, (unsigned) rc,
497                 memcached_strerror(r->connection->mc_st, rc));
498
499         odr_destroy(odr);
500         wrbuf_destroy(k);
501         wrbuf_destroy(rec_sha1);
502     }
503 #endif
504 }
505
506 Z_NamePlusRecord *ZOOM_memcached_lookup(ZOOM_resultset r, int pos,
507                                         const char *syntax,
508                                         const char *elementSetName,
509                                         const char *schema)
510 {
511 #if HAVE_HIREDIS
512     if (r->connection && r->connection->redis_c)
513     {
514         WRBUF k = wrbuf_alloc();
515         const char *argv[2];
516         size_t argvlen[2];
517         redisReply *reply1;
518
519         wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
520         wrbuf_printf(k, ";%d;%s;%s;%s", pos,
521                      syntax ? syntax : "",
522                      elementSetName ? elementSetName : "",
523                      schema ? schema : "");
524
525         yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
526         argv[0] = "GET";
527         argvlen[0] = 3;
528         argv[1] = wrbuf_buf(k);
529         argvlen[1] = wrbuf_len(k);
530         reply1 = redisCommandArgv(r->connection->redis_c, 2, argv, argvlen);
531
532         wrbuf_destroy(k);
533         if (reply1 && reply1->type == REDIS_REPLY_STRING)
534         {
535             redisReply *reply2;
536             char *sha1_buf = reply1->str;
537             int sha1_len = reply1->len;
538
539             yaz_log(YLOG_LOG, "Lookup record %.*s", (int) sha1_len, sha1_buf);
540
541             argv[0] = "GET";
542             argvlen[0] = 3;
543             argv[1] = sha1_buf;
544             argvlen[1] = sha1_len;
545
546             reply2 = redisCommandArgv(r->connection->redis_c, 2, argv, argvlen);
547             if (reply2 && reply2->type == REDIS_REPLY_STRING)
548             {
549                 Z_NamePlusRecord *npr = 0;
550                 char *v_buf = reply2->str;
551                 int v_len = reply2->len;
552
553                 odr_setbuf(r->odr, v_buf, v_len, 0);
554                 z_NamePlusRecord(r->odr, &npr, 0, 0);
555                 if (npr)
556                     yaz_log(YLOG_LOG, "returned redis copy");
557                 freeReplyObject(reply2);
558                 freeReplyObject(reply1);
559                 return npr;
560             }
561             freeReplyObject(reply2);
562         }
563         freeReplyObject(reply1);
564     }
565 #endif
566 #if HAVE_LIBMEMCACHED
567     if (r->connection && r->connection->mc_st)
568     {
569         WRBUF k = wrbuf_alloc();
570         char *sha1_buf;
571         size_t sha1_len;
572         uint32_t flags;
573         memcached_return_t rc;
574
575         wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
576         wrbuf_printf(k, ";%d;%s;%s;%s", pos,
577                      syntax ? syntax : "",
578                      elementSetName ? elementSetName : "",
579                      schema ? schema : "");
580
581         yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
582         sha1_buf = memcached_get(r->connection->mc_st,
583                                  wrbuf_buf(k), wrbuf_len(k),
584                                  &sha1_len, &flags, &rc);
585
586         wrbuf_destroy(k);
587         if (sha1_buf)
588         {
589             size_t v_len;
590             char *v_buf;
591
592             yaz_log(YLOG_LOG, "Lookup record %.*s", (int) sha1_len, sha1_buf);
593             v_buf = memcached_get(r->connection->mc_st, sha1_buf, sha1_len,
594                                   &v_len, &flags, &rc);
595             free(sha1_buf);
596             if (v_buf)
597             {
598                 Z_NamePlusRecord *npr = 0;
599
600                 odr_setbuf(r->odr, v_buf, v_len, 0);
601                 z_NamePlusRecord(r->odr, &npr, 0, 0);
602                 free(v_buf);
603                 if (npr)
604                     yaz_log(YLOG_LOG, "returned memcached copy");
605                 return npr;
606             }
607         }
608     }
609 #endif
610     return 0;
611
612 }
613 /*
614  * Local variables:
615  * c-basic-offset: 4
616  * c-file-style: "Stroustrup"
617  * indent-tabs-mode: nil
618  * End:
619  * vim: shiftwidth=4 tabstop=8 expandtab
620  */
621