memcached: generate SHA1 for records and store blob once
[yaz-moved-to-github.git] / src / zoom-memcached.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file zoom-memcached.c
7  * \brief Implements query/record caching using memcached
8  */
9 #if HAVE_CONFIG_H
10 #include <config.h>
11 #endif
12
13 #include <assert.h>
14 #include <string.h>
15 #include <errno.h>
16 #include "zoom-p.h"
17
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
20 #include <yaz/log.h>
21 #include <yaz/diagbib1.h>
22
23 void ZOOM_memcached_init(ZOOM_connection c)
24 {
25 #if HAVE_LIBMEMCACHED_MEMCACHED_H
26     c->mc_st = 0;
27 #endif
28 }
29
30 void ZOOM_memcached_destroy(ZOOM_connection c)
31 {
32 #if HAVE_LIBMEMCACHED_MEMCACHED_H
33     if (c->mc_st)
34         memcached_free(c->mc_st);
35 #endif
36 }
37
38 int ZOOM_memcached_configure(ZOOM_connection c)
39 {
40     const char *val;
41 #if HAVE_LIBMEMCACHED_MEMCACHED_H
42     if (c->mc_st)
43     {
44         memcached_free(c->mc_st);
45         c->mc_st = 0;
46     }
47 #endif
48     val = ZOOM_options_get(c->options, "memcached");
49     if (val && *val)
50     {
51 #if HAVE_LIBMEMCACHED_MEMCACHED_H
52         c->mc_st = memcached(val, strlen(val));
53         if (!c->mc_st)
54         {
55             ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, val);
56             return -1;
57         }
58         memcached_behavior_set(c->mc_st, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1);
59 #else
60         ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled");
61         return -1;
62 #endif
63     }
64     return 0;
65 }
66
67 void ZOOM_memcached_resultset(ZOOM_resultset r, ZOOM_query q)
68 {
69 #if HAVE_LIBMEMCACHED_MEMCACHED_H
70     ZOOM_connection c = r->connection;
71     r->mc_key = wrbuf_alloc();
72     wrbuf_puts(r->mc_key, "0;");
73     wrbuf_puts(r->mc_key, c->host_port);
74     wrbuf_puts(r->mc_key, ";");
75     if (c->user)
76         wrbuf_puts(r->mc_key, c->user);
77     wrbuf_puts(r->mc_key, ";");
78     if (c->group)
79         wrbuf_puts(r->mc_key, c->group);
80     wrbuf_puts(r->mc_key, ";");
81     if (c->password)
82         wrbuf_sha1_puts(r->mc_key, c->password, 1);
83     wrbuf_puts(r->mc_key, ";");
84     {
85         WRBUF w = wrbuf_alloc();
86         ZOOM_query_get_hash(q, w);
87         wrbuf_sha1_puts(r->mc_key, wrbuf_cstr(w), 1);
88         wrbuf_destroy(w);
89     }
90     wrbuf_puts(r->mc_key, ";");
91     if (r->req_facets)
92         wrbuf_puts(r->mc_key, r->req_facets);
93 #endif
94 }
95
96 void ZOOM_memcached_search(ZOOM_connection c, ZOOM_resultset resultset)
97 {
98 #if HAVE_LIBMEMCACHED_MEMCACHED_H
99     if (c->mc_st && resultset->live_set == 0)
100     {
101         size_t v_len;
102         uint32_t flags;
103         memcached_return_t rc;
104         char *v = memcached_get(c->mc_st, wrbuf_buf(resultset->mc_key),
105                                 wrbuf_len(resultset->mc_key),
106                                 &v_len, &flags, &rc);
107         /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
108         if (v)
109         {
110             ZOOM_Event event;
111             size_t lead_len = strlen(v) + 1;
112
113             resultset->size = odr_atoi(v);
114
115             yaz_log(YLOG_LOG, "For key %s got value %s lead_len=%d len=%d",
116                     wrbuf_cstr(resultset->mc_key), v, (int) lead_len,
117                     (int) v_len);
118             if (v_len > lead_len)
119             {
120                 Z_OtherInformation *oi = 0;
121                 int oi_len = v_len - lead_len;
122                 odr_setbuf(resultset->odr, v + lead_len, oi_len, 0);
123                 if (!z_OtherInformation(resultset->odr, &oi, 0, 0))
124                 {
125                     yaz_log(YLOG_WARN, "oi decoding failed");
126                     free(v);
127                     return;
128                 }
129                 ZOOM_handle_search_result(c, resultset, oi);
130                 ZOOM_handle_facet_result(c, resultset, oi);
131             }
132             free(v);
133             event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
134             ZOOM_connection_put_event(c, event);
135             resultset->live_set = 1;
136         }
137     }
138 #endif
139 }
140
141 void ZOOM_memcached_hitcount(ZOOM_connection c, ZOOM_resultset resultset,
142                              Z_OtherInformation *oi, const char *precision)
143 {
144 #if HAVE_LIBMEMCACHED_MEMCACHED_H
145     if (c->mc_st && resultset->live_set == 0)
146     {
147         uint32_t flags = 0;
148         memcached_return_t rc;
149         time_t expiration = 36000;
150         char *str;
151         ODR odr = odr_createmem(ODR_ENCODE);
152         char *oi_buf = 0;
153         int oi_len = 0;
154         char *key;
155
156         str = odr_malloc(odr, 20 + strlen(precision));
157         /* count;precision (ASCII) + '\0' + BER buffer for otherInformation */
158         sprintf(str, ODR_INT_PRINTF ";%s", resultset->size, precision);
159         if (oi)
160         {
161             z_OtherInformation(odr, &oi, 0, 0);
162             oi_buf = odr_getbuf(odr, &oi_len, 0);
163         }
164         key = odr_malloc(odr, strlen(str) + 1 + oi_len);
165         strcpy(key, str);
166         if (oi_len)
167             memcpy(key + strlen(str) + 1, oi_buf, oi_len);
168
169         rc = memcached_set(c->mc_st,
170                            wrbuf_buf(resultset->mc_key),
171                            wrbuf_len(resultset->mc_key),
172                            key, strlen(str) + 1 + oi_len, expiration, flags);
173         yaz_log(YLOG_LOG, "Store hit count key=%s value=%s oi_len=%d rc=%u %s",
174                 wrbuf_cstr(resultset->mc_key), str, oi_len, (unsigned) rc,
175                 memcached_last_error_message(c->mc_st));
176         odr_destroy(odr);
177     }
178 #endif
179 }
180
181 void ZOOM_memcached_add(ZOOM_resultset r, Z_NamePlusRecord *npr,
182                         int pos,
183                         const char *syntax, const char *elementSetName,
184                         const char *schema,
185                         Z_SRW_diagnostic *diag)
186 {
187 #if HAVE_LIBMEMCACHED_MEMCACHED_H
188     if (r->connection->mc_st &&
189         !diag && npr->which == Z_NamePlusRecord_databaseRecord)
190     {
191         WRBUF k = wrbuf_alloc();
192         WRBUF rec_sha1 = wrbuf_alloc();
193         uint32_t flags = 0;
194         memcached_return_t rc;
195         time_t expiration = 36000;
196         ODR odr = odr_createmem(ODR_ENCODE);
197         char *rec_buf;
198         int rec_len;
199
200         z_NamePlusRecord(odr, &npr, 0, 0);
201         rec_buf = odr_getbuf(odr, &rec_len, 0);
202
203         wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
204         wrbuf_printf(k, ";%d;%s;%s;%s", pos,
205                      syntax ? syntax : "",
206                      elementSetName ? elementSetName : "",
207                      schema ? schema : "");
208
209         wrbuf_sha1_write(rec_sha1, rec_buf, rec_len, 1);
210
211         rc = memcached_set(r->connection->mc_st,
212                            wrbuf_buf(k), wrbuf_len(k),
213                            wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
214                            expiration, flags);
215
216         yaz_log(YLOG_LOG, "Store record key=%s val=%s rc=%u %s",
217                 wrbuf_cstr(k), wrbuf_cstr(rec_sha1), (unsigned) rc,
218                 memcached_last_error_message(r->connection->mc_st));
219
220         rc = memcached_add(r->connection->mc_st,
221                            wrbuf_buf(rec_sha1), wrbuf_len(rec_sha1),
222                            rec_buf, rec_len,
223                            expiration, flags);
224
225         yaz_log(YLOG_LOG, "Add record key=%s rec_len=%d rc=%u %s",
226                 wrbuf_cstr(rec_sha1), rec_len, (unsigned) rc,
227                 memcached_last_error_message(r->connection->mc_st));
228
229         odr_destroy(odr);
230         wrbuf_destroy(k);
231         wrbuf_destroy(rec_sha1);
232     }
233 #endif
234 }
235
236 Z_NamePlusRecord *ZOOM_memcached_lookup(ZOOM_resultset r, int pos,
237                                         const char *syntax,
238                                         const char *elementSetName,
239                                         const char *schema)
240 {
241 #if HAVE_LIBMEMCACHED_MEMCACHED_H
242     if (r->connection && r->connection->mc_st)
243     {
244         WRBUF k = wrbuf_alloc();
245         char *sha1_buf;
246         size_t sha1_len;
247         uint32_t flags;
248         memcached_return_t rc;
249
250         wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
251         wrbuf_printf(k, ";%d;%s;%s;%s", pos,
252                      syntax ? syntax : "",
253                      elementSetName ? elementSetName : "",
254                      schema ? schema : "");
255
256         yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
257         sha1_buf = memcached_get(r->connection->mc_st,
258                                  wrbuf_buf(k), wrbuf_len(k),
259                                  &sha1_len, &flags, &rc);
260
261         wrbuf_destroy(k);
262         if (sha1_buf)
263         {
264             size_t v_len;
265             char *v_buf;
266
267             yaz_log(YLOG_LOG, "Lookup record %.*s", (int) sha1_len, sha1_buf);
268             v_buf = memcached_get(r->connection->mc_st, sha1_buf, sha1_len,
269                                   &v_len, &flags, &rc);
270             free(sha1_buf);
271             if (v_buf)
272             {
273                 Z_NamePlusRecord *npr = 0;
274
275                 odr_setbuf(r->odr, v_buf, v_len, 0);
276                 z_NamePlusRecord(r->odr, &npr, 0, 0);
277                 free(v_buf);
278                 if (npr)
279                     yaz_log(YLOG_LOG, "returned memcached copy");
280                 return npr;
281             }
282         }
283     }
284 #endif
285     return 0;
286
287 }
288 /*
289  * Local variables:
290  * c-basic-offset: 4
291  * c-file-style: "Stroustrup"
292  * indent-tabs-mode: nil
293  * End:
294  * vim: shiftwidth=4 tabstop=8 expandtab
295  */
296