From: Adam Dickmeiss Date: Wed, 29 Jan 2014 16:15:02 +0000 (+0100) Subject: Refactor: all memcached stuff to zoom-memcached.c X-Git-Tag: v5.0.13~30 X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=commitdiff_plain;h=dd016baf50d8cd02a1a32d1fd10b5544cae6ff96 Refactor: all memcached stuff to zoom-memcached.c --- diff --git a/src/Makefile.am b/src/Makefile.am index 5ccb53f..92961ae 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -89,8 +89,8 @@ libyaz_la_SOURCES= $(GEN_FILES) \ zget.c yaz-ccl.c diag-entry.c diag-entry.h \ logrpn.c \ otherinfo.c pquery.c sortspec.c charneg.c initopt.c init_diag.c \ - zoom-c.c zoom-z3950.c zoom-sru.c zoom-query.c zoom-record-cache.c \ - zoom-event.c \ + zoom-c.c zoom-memcached.c zoom-z3950.c zoom-sru.c zoom-query.c \ + zoom-record-cache.c zoom-event.c \ record_render.c zoom-socket.c zoom-opt.c zoom-p.h sru_facet.c sru-p.h \ grs1disp.c zgdu.c soap.c srw.c srwutil.c uri.c solr.c diag_map.c \ opac_to_xml.c xml_add.c xml_match.c xml_to_opac.c \ diff --git a/src/zoom-c.c b/src/zoom-c.c index b1874ea..28eb798 100644 --- a/src/zoom-c.c +++ b/src/zoom-c.c @@ -298,9 +298,7 @@ ZOOM_API(ZOOM_connection) c->cookies = 0; c->saveAPDU_wrbuf = 0; -#if HAVE_LIBMEMCACHED_MEMCACHED_H - c->mc_st = 0; -#endif + ZOOM_memcached_init(c); return c; } @@ -545,29 +543,11 @@ ZOOM_API(void) yaz_cookies_destroy(c->cookies); c->cookies = yaz_cookies_create(); -#if HAVE_LIBMEMCACHED_MEMCACHED_H - if (c->mc_st) + if (ZOOM_memcached_configure(c)) { - memcached_free(c->mc_st); - c->mc_st = 0; - } -#endif - val = ZOOM_options_get(c->options, "memcached"); - if (val && *val) - { -#if HAVE_LIBMEMCACHED_MEMCACHED_H - c->mc_st = memcached(val, strlen(val)); - if (!c->mc_st) - { - ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, val); - return; - } -#else - ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled"); + ZOOM_connection_remove_tasks(c); return; -#endif } - if (c->sru_mode == zoom_sru_error) { ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val); @@ -613,10 +593,7 @@ ZOOM_API(void) return; yaz_log(c->log_api, "%p ZOOM_connection_destroy", c); -#if HAVE_LIBMEMCACHED_MEMCACHED_H - if (c->mc_st) - memcached_free(c->mc_st); -#endif + ZOOM_memcached_destroy(c); if (c->cs) cs_close(c->cs); @@ -774,23 +751,7 @@ ZOOM_API(ZOOM_resultset) r->next = c->resultsets; c->resultsets = r; -#if HAVE_LIBMEMCACHED_MEMCACHED_H - r->mc_key = wrbuf_alloc(); - wrbuf_puts(r->mc_key, "0;"); - wrbuf_puts(r->mc_key, c->host_port); - wrbuf_puts(r->mc_key, ";"); - if (c->user) - wrbuf_puts(r->mc_key, c->user); - wrbuf_puts(r->mc_key, ";"); - if (c->group) - wrbuf_puts(r->mc_key, c->group); - wrbuf_puts(r->mc_key, ";"); - if (c->password) - wrbuf_sha1_puts(r->mc_key, c->password, 1); - wrbuf_puts(r->mc_key, ";"); - wrbuf_sha1_puts(r->mc_key, ZOOM_query_get_query_string(q), 1); - wrbuf_puts(r->mc_key, ";"); -#endif + ZOOM_memcached_resultset(r, q); if (c->host_port && c->proto == PROTO_HTTP) { diff --git a/src/zoom-memcached.c b/src/zoom-memcached.c new file mode 100644 index 0000000..250010c --- /dev/null +++ b/src/zoom-memcached.c @@ -0,0 +1,230 @@ +/* This file is part of the YAZ toolkit. + * Copyright (C) Index Data + * See the file LICENSE for details. + */ +/** + * \file zoom-memcached.c + * \brief Implements query/record caching using memcached + */ +#if HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include "zoom-p.h" + +#include +#include +#include +#include + +void ZOOM_memcached_init(ZOOM_connection c) +{ +#if HAVE_LIBMEMCACHED_MEMCACHED_H + c->mc_st = 0; +#endif +} + +void ZOOM_memcached_destroy(ZOOM_connection c) +{ +#if HAVE_LIBMEMCACHED_MEMCACHED_H + if (c->mc_st) + memcached_free(c->mc_st); +#endif +} + +int ZOOM_memcached_configure(ZOOM_connection c) +{ + const char *val; +#if HAVE_LIBMEMCACHED_MEMCACHED_H + if (c->mc_st) + { + memcached_free(c->mc_st); + c->mc_st = 0; + } +#endif + val = ZOOM_options_get(c->options, "memcached"); + if (val && *val) + { +#if HAVE_LIBMEMCACHED_MEMCACHED_H + c->mc_st = memcached(val, strlen(val)); + if (!c->mc_st) + { + ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, val); + return -1; + } +#else + ZOOM_set_error(c, ZOOM_ERROR_MEMCACHED, "not enabled"); + return -1; +#endif + } + return 0; +} + +void ZOOM_memcached_resultset(ZOOM_resultset r, ZOOM_query q) +{ +#if HAVE_LIBMEMCACHED_MEMCACHED_H + ZOOM_connection c = r->connection; + r->mc_key = wrbuf_alloc(); + wrbuf_puts(r->mc_key, "0;"); + wrbuf_puts(r->mc_key, c->host_port); + wrbuf_puts(r->mc_key, ";"); + if (c->user) + wrbuf_puts(r->mc_key, c->user); + wrbuf_puts(r->mc_key, ";"); + if (c->group) + wrbuf_puts(r->mc_key, c->group); + wrbuf_puts(r->mc_key, ";"); + if (c->password) + wrbuf_sha1_puts(r->mc_key, c->password, 1); + wrbuf_puts(r->mc_key, ";"); + wrbuf_sha1_puts(r->mc_key, ZOOM_query_get_query_string(q), 1); + wrbuf_puts(r->mc_key, ";"); +#endif +} + +void ZOOM_memcached_search(ZOOM_connection c, ZOOM_resultset resultset) +{ +#if HAVE_LIBMEMCACHED_MEMCACHED_H + /* TODO: add sorting */ + if (c->mc_st && resultset->live_set == 0) + { + size_t v_len; + uint32_t flags; + memcached_return_t rc; + char *v = memcached_get(c->mc_st, wrbuf_buf(resultset->mc_key), + wrbuf_len(resultset->mc_key), + &v_len, &flags, &rc); + if (v) + { + ZOOM_Event event; + WRBUF w = wrbuf_alloc(); + + wrbuf_write(w, v, v_len); + free(v); + resultset->size = odr_atoi(wrbuf_cstr(w)); + + yaz_log(YLOG_LOG, "For key %s got value %s", + wrbuf_cstr(resultset->mc_key), wrbuf_cstr(w)); + + wrbuf_destroy(w); + event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH); + ZOOM_connection_put_event(c, event); + resultset->live_set = 1; + } + } +#endif +} + +void ZOOM_memcached_hitcount(ZOOM_connection c, ZOOM_resultset resultset) +{ +#if HAVE_LIBMEMCACHED_MEMCACHED_H + if (c->mc_st && resultset->live_set == 0) + { + uint32_t flags = 0; + memcached_return_t rc; + time_t expiration = 36000; + char str[40]; + + sprintf(str, ODR_INT_PRINTF, resultset->size); + rc = memcached_set(c->mc_st, + wrbuf_buf(resultset->mc_key),wrbuf_len(resultset->mc_key), + str, strlen(str), expiration, flags); + yaz_log(YLOG_LOG, "Store SRU hit count key=%s value=%s rc=%u %s", + wrbuf_cstr(resultset->mc_key), str, (unsigned) rc, + memcached_last_error_message(c->mc_st)); + } +#endif +} + +void ZOOM_memcached_add(ZOOM_resultset r, Z_NamePlusRecord *npr, + int pos, + const char *syntax, const char *elementSetName, + const char *schema, + Z_SRW_diagnostic *diag) +{ +#if HAVE_LIBMEMCACHED_MEMCACHED_H + if (r->connection->mc_st && + !diag && npr->which == Z_NamePlusRecord_databaseRecord) + { + WRBUF k = wrbuf_alloc(); + uint32_t flags = 0; + memcached_return_t rc; + time_t expiration = 36000; + ODR odr = odr_createmem(ODR_ENCODE); + char *rec_buf; + int rec_len; + + z_NamePlusRecord(odr, &npr, 0, 0); + rec_buf = odr_getbuf(odr, &rec_len, 0); + + wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key)); + wrbuf_printf(k, ";%d;%s;%s;%s", pos, + syntax ? syntax : "", + elementSetName ? elementSetName : "", + schema ? schema : ""); + rc = memcached_set(r->connection->mc_st, + wrbuf_buf(k),wrbuf_len(k), + rec_buf, rec_len, + expiration, flags); + + yaz_log(YLOG_LOG, "Store record lkey=%s len=%d rc=%u %s", + wrbuf_cstr(k), rec_len, (unsigned) rc, + memcached_last_error_message(r->connection->mc_st)); + odr_destroy(odr); + wrbuf_destroy(k); + } +#endif +} + +Z_NamePlusRecord *ZOOM_memcached_lookup(ZOOM_resultset r, int pos, + const char *syntax, + const char *elementSetName, + const char *schema) +{ +#if HAVE_LIBMEMCACHED_MEMCACHED_H + if (r->connection && r->connection->mc_st) + { + WRBUF k = wrbuf_alloc(); + size_t v_len; + char *v_buf; + uint32_t flags; + memcached_return_t rc; + + wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key)); + wrbuf_printf(k, ";%d;%s;%s;%s", pos, + syntax ? syntax : "", + elementSetName ? elementSetName : "", + schema ? schema : ""); + + yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k)); + v_buf = memcached_get(r->connection->mc_st, wrbuf_buf(k), wrbuf_len(k), + &v_len, &flags, &rc); + wrbuf_destroy(k); + if (v_buf) + { + Z_NamePlusRecord *npr = 0; + + odr_setbuf(r->odr, v_buf, v_len, 0); + z_NamePlusRecord(r->odr, &npr, 0, 0); + free(v_buf); + if (npr) + yaz_log(YLOG_LOG, "returned memcached copy"); + return npr; + } + } +#endif + return 0; + +} +/* + * Local variables: + * c-basic-offset: 4 + * c-file-style: "Stroustrup" + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/src/zoom-p.h b/src/zoom-p.h index 2a64c5f..f443594 100644 --- a/src/zoom-p.h +++ b/src/zoom-p.h @@ -270,6 +270,22 @@ void ZOOM_Event_destroy(ZOOM_Event event); zoom_ret ZOOM_send_GDU(ZOOM_connection c, Z_GDU *gdu); void ZOOM_handle_facet_list(ZOOM_resultset r, Z_FacetList *fl); +void ZOOM_memcached_init(ZOOM_connection c); +int ZOOM_memcached_configure(ZOOM_connection c); +void ZOOM_memcached_destroy(ZOOM_connection c); +void ZOOM_memcached_resultset(ZOOM_resultset r, ZOOM_query q); +void ZOOM_memcached_search(ZOOM_connection c, ZOOM_resultset r); +void ZOOM_memcached_hitcount(ZOOM_connection c, ZOOM_resultset result); +void ZOOM_memcached_add(ZOOM_resultset r, Z_NamePlusRecord *npr, + int pos, + const char *syntax, const char *elementSetName, + const char *schema, + Z_SRW_diagnostic *diag); +Z_NamePlusRecord *ZOOM_memcached_lookup(ZOOM_resultset r, int pos, + const char *syntax, + const char *elementSetName, + const char *schema); + /* * Local variables: * c-basic-offset: 4 diff --git a/src/zoom-record-cache.c b/src/zoom-record-cache.c index 9e76d66..b999e59 100644 --- a/src/zoom-record-cache.c +++ b/src/zoom-record-cache.c @@ -128,47 +128,16 @@ void ZOOM_record_cache_add(ZOOM_resultset r, Z_NamePlusRecord *npr, Z_SRW_diagnostic *diag) { record_cache_add(r, npr, pos, syntax, elementSetName, schema, diag); -#if HAVE_LIBMEMCACHED_MEMCACHED_H - if (r->connection->mc_st && - !diag && npr->which == Z_NamePlusRecord_databaseRecord) - { - WRBUF k = wrbuf_alloc(); - uint32_t flags = 0; - memcached_return_t rc; - time_t expiration = 36000; - ODR odr = odr_createmem(ODR_ENCODE); - char *rec_buf; - int rec_len; - - z_NamePlusRecord(odr, &npr, 0, 0); - rec_buf = odr_getbuf(odr, &rec_len, 0); - - wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key)); - wrbuf_printf(k, ";%d;%s;%s;%s", pos, - syntax ? syntax : "", - elementSetName ? elementSetName : "", - schema ? schema : ""); - rc = memcached_set(r->connection->mc_st, - wrbuf_buf(k),wrbuf_len(k), - rec_buf, rec_len, - expiration, flags); - - yaz_log(YLOG_LOG, "Store record lkey=%s len=%d rc=%u %s", - wrbuf_cstr(k), rec_len, (unsigned) rc, - memcached_last_error_message(r->connection->mc_st)); - odr_destroy(odr); - wrbuf_destroy(k); - } -#endif + ZOOM_memcached_add(r, npr, pos, syntax, elementSetName, schema, diag); } - ZOOM_record ZOOM_record_cache_lookup(ZOOM_resultset r, int pos, const char *syntax, const char *elementSetName, const char *schema) { ZOOM_record_cache rc; + Z_NamePlusRecord *npr; for (rc = r->record_hash[record_hash(pos)]; rc; rc = rc->next) { @@ -183,44 +152,10 @@ ZOOM_record ZOOM_record_cache_lookup(ZOOM_resultset r, int pos, return &rc->rec; } } -#if HAVE_LIBMEMCACHED_MEMCACHED_H - if (r->connection && r->connection->mc_st) - { - WRBUF k = wrbuf_alloc(); - size_t v_len; - char *v_buf; - uint32_t flags; - memcached_return_t rc; - - wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key)); - wrbuf_printf(k, ";%d;%s;%s;%s", pos, - syntax ? syntax : "", - elementSetName ? elementSetName : "", - schema ? schema : ""); - - yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k)); - v_buf = memcached_get(r->connection->mc_st, wrbuf_buf(k), wrbuf_len(k), - &v_len, &flags, &rc); - wrbuf_destroy(k); - if (v_buf) - { - Z_NamePlusRecord *npr = 0; - - odr_setbuf(r->odr, v_buf, v_len, 0); - - z_NamePlusRecord(r->odr, &npr, 0, 0); - free(v_buf); - if (npr) - { - yaz_log(YLOG_LOG, "returned memcached copy"); - return record_cache_add(r, npr, pos, syntax, elementSetName, - schema, 0); - } - yaz_log(YLOG_WARN, "memcached_get npr failed v_len=%ld", - (long) v_len); - } - } -#endif + npr = ZOOM_memcached_lookup(r, pos, syntax, elementSetName, schema); + if (npr) + return record_cache_add(r, npr, pos, syntax, elementSetName, + schema, 0); return 0; } diff --git a/src/zoom-sru.c b/src/zoom-sru.c index ad241cd..142051f 100644 --- a/src/zoom-sru.c +++ b/src/zoom-sru.c @@ -163,35 +163,8 @@ zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c) resultset = c->tasks->u.search.resultset; -#if HAVE_LIBMEMCACHED_MEMCACHED_H - /* TODO: add sorting */ - if (c->mc_st && resultset->live_set == 0) - { - size_t v_len; - uint32_t flags; - memcached_return_t rc; - char *v = memcached_get(c->mc_st, wrbuf_buf(resultset->mc_key), - wrbuf_len(resultset->mc_key), - &v_len, &flags, &rc); - if (v) - { - ZOOM_Event event; - WRBUF w = wrbuf_alloc(); - - wrbuf_write(w, v, v_len); - free(v); - resultset->size = odr_atoi(wrbuf_cstr(w)); + ZOOM_memcached_search(c, resultset); - yaz_log(YLOG_LOG, "For key %s got value %s", - wrbuf_cstr(resultset->mc_key), wrbuf_cstr(w)); - - wrbuf_destroy(w); - event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH); - ZOOM_connection_put_event(c, event); - resultset->live_set = 1; - } - } -#endif if (!resultset->setname) resultset->setname = xstrdup("default"); ZOOM_options_set(resultset->options, "setname", resultset->setname); @@ -323,23 +296,7 @@ static zoom_ret handle_srw_response(ZOOM_connection c, if (res->numberOfRecords) { resultset->size = *res->numberOfRecords; -#if HAVE_LIBMEMCACHED_MEMCACHED_H - if (c->mc_st && resultset->live_set == 0) - { - uint32_t flags = 0; - memcached_return_t rc; - time_t expiration = 36000; - char str[40]; - - sprintf(str, ODR_INT_PRINTF, resultset->size); - rc = memcached_set(c->mc_st, - wrbuf_buf(resultset->mc_key),wrbuf_len(resultset->mc_key), - str, strlen(str), expiration, flags); - yaz_log(YLOG_LOG, "Store SRU hit count key=%s value=%s rc=%u %s", - wrbuf_cstr(resultset->mc_key), str, (unsigned) rc, - memcached_last_error_message(c->mc_st)); - } -#endif + ZOOM_memcached_hitcount(c, resultset); } resultset->live_set = 2; if (res->suggestions) diff --git a/src/zoom-z3950.c b/src/zoom-z3950.c index 556e258..e39ff8a 100644 --- a/src/zoom-z3950.c +++ b/src/zoom-z3950.c @@ -1288,25 +1288,9 @@ static void handle_Z3950_search_response(ZOOM_connection c, handle_facet_result(c, resultset, sr->additionalSearchInfo); resultset->size = *sr->resultCount; - resultset->live_set = 2; - -#if HAVE_LIBMEMCACHED_MEMCACHED_H - if (c->mc_st) - { - uint32_t flags = 0; - memcached_return_t rc; - time_t expiration = 36000; - char str[40]; - sprintf(str, ODR_INT_PRINTF, *sr->resultCount); - rc = memcached_set(c->mc_st, - wrbuf_buf(resultset->mc_key),wrbuf_len(resultset->mc_key), - str, strlen(str), expiration, flags); - yaz_log(YLOG_LOG, "Store Z39.50 hit count key=%s value=%s rc=%u %s", - wrbuf_cstr(resultset->mc_key), str, (unsigned) rc, - memcached_last_error_message(c->mc_st)); - } -#endif + ZOOM_memcached_hitcount(c, resultset); + resultset->live_set = 2; handle_Z3950_records(c, sr->records, 0); } @@ -1567,35 +1551,8 @@ zoom_ret ZOOM_connection_Z3950_search(ZOOM_connection c) yaz_log(c->log_details, "%p send_present start=%d count=%d", c, *start, *count); -#if HAVE_LIBMEMCACHED_MEMCACHED_H - /* TODO: add sorting */ - if (c->mc_st && resultset->live_set == 0) - { - size_t v_len; - uint32_t flags; - memcached_return_t rc; - char *v = memcached_get(c->mc_st, wrbuf_buf(resultset->mc_key), - wrbuf_len(resultset->mc_key), - &v_len, &flags, &rc); - if (v) - { - ZOOM_Event event; - WRBUF w = wrbuf_alloc(); - - wrbuf_write(w, v, v_len); - free(v); - resultset->size = odr_atoi(wrbuf_cstr(w)); - - yaz_log(YLOG_LOG, "For key %s got value %s", - wrbuf_cstr(resultset->mc_key), wrbuf_cstr(w)); + ZOOM_memcached_search(c, resultset); - wrbuf_destroy(w); - event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH); - ZOOM_connection_put_event(c, event); - resultset->live_set = 1; - } - } -#endif if (*start < 0 || *count < 0) { ZOOM_set_dset_error(c, YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, "Bib-1", diff --git a/win/makefile b/win/makefile index 02e3ad1..b1b346f 100644 --- a/win/makefile +++ b/win/makefile @@ -498,6 +498,7 @@ MISC_OBJS= \ $(OBJDIR)\srwutil.obj \ $(OBJDIR)\zoom-c.obj \ $(OBJDIR)\zoom-event.obj \ + $(OBJDIR)\zoom-memcached.obj \ $(OBJDIR)\zoom-record-cache.obj \ $(OBJDIR)\zoom-z3950.obj \ $(OBJDIR)\zoom-sru.obj \