From dbcf2c3410ed74d919efdc62ffbd00e716284d8e Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Mon, 23 Apr 2007 21:05:23 +0000 Subject: [PATCH] Refactor stuff from logic.c: connection stuff in connection.[ch], client stuff to client.[ch]. host definitions in host.h. Global parameters to parameters.h., --- src/.cvsignore | 1 + src/Makefile.am | 9 +- src/client.c | 704 ++++++++++++++++++++++++ src/client.h | 93 ++++ src/connection.c | 433 +++++++++++++++ src/connection.h | 56 ++ src/database.c | 3 +- src/getaddrinfo.c | 4 +- src/host.h | 42 ++ src/http_command.c | 9 +- src/logic.c | 1560 ++++++++++++---------------------------------------- src/parameters.h | 56 ++ src/pazpar2.c | 4 +- src/pazpar2.h | 87 +-- src/relevance.h | 7 +- 15 files changed, 1775 insertions(+), 1293 deletions(-) create mode 100644 src/client.c create mode 100644 src/client.h create mode 100644 src/connection.c create mode 100644 src/connection.h create mode 100644 src/host.h create mode 100644 src/parameters.h diff --git a/src/.cvsignore b/src/.cvsignore index fb93fcd..3957405 100644 --- a/src/.cvsignore +++ b/src/.cvsignore @@ -10,3 +10,4 @@ stamp-h1 test_config test_relevance test_sel_thread +test_record diff --git a/src/Makefile.am b/src/Makefile.am index 2892fe6..a3c24c2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,4 +1,4 @@ -# $Id: Makefile.am,v 1.14 2007-04-23 12:33:00 marc Exp $ +# $Id: Makefile.am,v 1.15 2007-04-23 21:05:23 adam Exp $ bin_PROGRAMS = pazpar2 check_PROGRAMS = test_config test_record test_relevance test_sel_thread @@ -6,7 +6,9 @@ TESTS = $(check_PROGRAMS) noinst_LIBRARIES = libpazpar2.a -MAINTAINERCLEANFILES = Makefile.in cconfig.h +MAINTAINERCLEANFILES = Makefile.in cconfig.h + +CONFIG_CLEAN_FILES=*.log AM_CFLAGS = $(YAZINC) @@ -16,7 +18,8 @@ libpazpar2_a_SOURCES = config.c config.h eventl.c eventl.h \ record.h record.c reclists.c reclists.h \ relevance.c relevance.h termlists.c termlists.h \ util.c util.h zeerex.c zeerex.h database.c database.h \ - settings.h settings.c sel_thread.c sel_thread.h getaddrinfo.c + settings.h settings.c sel_thread.c sel_thread.h getaddrinfo.c \ + client.c client.h connection.c connection.h host.h parameters.h pazpar2_SOURCES = pazpar2.c pazpar2_LDADD = libpazpar2.a $(YAZLIB) diff --git a/src/client.c b/src/client.c new file mode 100644 index 0000000..e9bdbb9 --- /dev/null +++ b/src/client.c @@ -0,0 +1,704 @@ +/* $Id: client.c,v 1.1 2007-04-23 21:05:23 adam Exp $ + Copyright (c) 2006-2007, Index Data. + +This file is part of Pazpar2. + +Pazpar2 is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Pazpar2; see the file LICENSE. If not, write to the +Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA +02111-1307, USA. + */ + +/** \file client.c + \brief Z39.50 client +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if YAZ_VERSIONL >= 0x020163 +#include +#endif + +#if HAVE_CONFIG_H +#include "cconfig.h" +#endif + +#define USE_TIMING 0 +#if USE_TIMING +#include +#endif + +#include + +#include "pazpar2.h" + +#include "client.h" +#include "connection.h" +#include "settings.h" + +// Represents client state for a connection to one search target +struct client { + struct session_database *database; + struct connection *connection; + struct session *session; + char *pquery; // Current search + int hits; + int records; + int setno; + int requestid; // ID of current outstanding request + int diagnostic; + enum client_state state; + struct client *next; +}; + +static const char *client_states[] = { + "Client_Connecting", + "Client_Connected", + "Client_Idle", + "Client_Initializing", + "Client_Searching", + "Client_Presenting", + "Client_Error", + "Client_Failed", + "Client_Disconnected", + "Client_Stopped" +}; + +static struct client *client_freelist = 0; + +static int send_apdu(struct client *c, Z_APDU *a) +{ + return connection_send_apdu(client_get_connection(c), a); +} + + +const char *client_get_state_str(struct client *cl) +{ + return client_states[cl->state]; +} + +enum client_state client_get_state(struct client *cl) +{ + return cl->state; +} + +void client_set_state(struct client *cl, enum client_state st) +{ + cl->state = st; +} + +// Close connection and set state to error +void client_fatal(struct client *cl) +{ + yaz_log(YLOG_WARN, "Fatal error from %s", client_get_url(cl)); + connection_destroy(cl->connection); + cl->state = Client_Error; +} + +struct connection *client_get_connection(struct client *cl) +{ + return cl->connection; +} + +struct session_database *client_get_database(struct client *cl) +{ + return cl->database; +} + +struct session *client_get_session(struct client *cl) +{ + return cl->session; +} + +const char *client_get_pquery(struct client *cl) +{ + return cl->pquery; +} + +void client_set_requestid(struct client *cl, int id) +{ + cl->requestid = id; +} + +void client_send_present(struct client *cl) +{ + struct session_database *sdb = client_get_database(cl); + Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_presentRequest); + int toget; + int start = cl->records + 1; + char *recsyn; + + toget = global_parameters.chunk; + if (toget > global_parameters.toget - cl->records) + toget = global_parameters.toget - cl->records; + if (toget > cl->hits - cl->records) + toget = cl->hits - cl->records; + + yaz_log(YLOG_DEBUG, "Trying to present %d records\n", toget); + + a->u.presentRequest->resultSetStartPoint = &start; + a->u.presentRequest->numberOfRecordsRequested = &toget; + + a->u.presentRequest->resultSetId = "Default"; + + if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX))) + { +#if YAZ_VERSIONL >= 0x020163 + a->u.presentRequest->preferredRecordSyntax = + yaz_string_to_oid_odr(yaz_oid_std(), + CLASS_RECSYN, recsyn, + global_parameters.odr_out); +#else + a->u.presentRequest->preferredRecordSyntax = + yaz_str_to_z3950oid(global_parameters.odr_out, + CLASS_RECSYN, recsyn); +#endif + } + + if (send_apdu(cl, a) >= 0) + cl->state = Client_Presenting; + else + cl->state = Client_Error; + odr_reset(global_parameters.odr_out); +} + + +void client_send_search(struct client *cl) +{ + struct session *se = client_get_session(cl); + struct session_database *sdb = client_get_database(cl); + Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_searchRequest); + int ndb; + char **databaselist; + Z_Query *zquery; + int ssub = 0, lslb = 100000, mspn = 10; + char *recsyn = 0; + char *piggyback = 0; + char *queryenc = 0; + yaz_iconv_t iconv = 0; + + yaz_log(YLOG_DEBUG, "Sending search to %s", sdb->database->url); + + // constructing RPN query + a->u.searchRequest->query = zquery = odr_malloc(global_parameters.odr_out, + sizeof(Z_Query)); + zquery->which = Z_Query_type_1; + zquery->u.type_1 = p_query_rpn(global_parameters.odr_out, + client_get_pquery(cl)); + + // converting to target encoding + if ((queryenc = session_setting_oneval(sdb, PZ_QUERYENCODING))){ + iconv = yaz_iconv_open(queryenc, "UTF-8"); + if (iconv){ + yaz_query_charset_convert_rpnquery(zquery->u.type_1, + global_parameters.odr_out, + iconv); + yaz_iconv_close(iconv); + } else + yaz_log(YLOG_WARN, "Query encoding failed %s %s", + client_get_database(cl)->database->url, queryenc); + } + + for (ndb = 0; sdb->database->databases[ndb]; ndb++) + ; + databaselist = odr_malloc(global_parameters.odr_out, sizeof(char*) * ndb); + for (ndb = 0; sdb->database->databases[ndb]; ndb++) + databaselist[ndb] = sdb->database->databases[ndb]; + + if (!(piggyback = session_setting_oneval(sdb, PZ_PIGGYBACK)) || *piggyback == '1') + { + if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX))) + { +#if YAZ_VERSIONL >= 0x020163 + a->u.searchRequest->preferredRecordSyntax = + yaz_string_to_oid_odr(yaz_oid_std(), + CLASS_RECSYN, recsyn, + global_parameters.odr_out); +#else + a->u.searchRequest->preferredRecordSyntax = + yaz_str_to_z3950oid(global_parameters.odr_out, + CLASS_RECSYN, recsyn); +#endif + } + a->u.searchRequest->smallSetUpperBound = &ssub; + a->u.searchRequest->largeSetLowerBound = &lslb; + a->u.searchRequest->mediumSetPresentNumber = &mspn; + } + a->u.searchRequest->resultSetName = "Default"; + a->u.searchRequest->databaseNames = databaselist; + a->u.searchRequest->num_databaseNames = ndb; + + + { //scope for sending and logging queries + WRBUF wbquery = wrbuf_alloc(); + yaz_query_to_wrbuf(wbquery, a->u.searchRequest->query); + + + if (send_apdu(cl, a) >= 0) + { + client_set_state(cl, Client_Searching); + client_set_requestid(cl, se->requestid); + yaz_log(YLOG_LOG, "SearchRequest %s %s %s", + client_get_database(cl)->database->url, + queryenc ? queryenc : "UTF-8", + wrbuf_cstr(wbquery)); + } + else { + client_set_state(cl, Client_Error); + yaz_log(YLOG_WARN, "Failed SearchRequest %s %s %s", + client_get_database(cl)->database->url, + queryenc ? queryenc : "UTF-8", + wrbuf_cstr(wbquery)); + } + + wrbuf_destroy(wbquery); + } + + odr_reset(global_parameters.odr_out); +} + +void client_init_response(struct client *cl, Z_APDU *a) +{ + Z_InitResponse *r = a->u.initResponse; + + yaz_log(YLOG_DEBUG, "Init response %s", cl->database->database->url); + + if (*r->result) + { + cl->state = Client_Idle; + } + else + cl->state = Client_Failed; // FIXME need to do something to the connection +} + + +static void ingest_records(struct client *cl, Z_Records *r) +{ +#if USE_TIMING + yaz_timing_t t = yaz_timing_create(); +#endif + struct record *rec; + struct session *s = client_get_session(cl); + Z_NamePlusRecordList *rlist; + int i; + + if (r->which != Z_Records_DBOSD) + return; + rlist = r->u.databaseOrSurDiagnostics; + for (i = 0; i < rlist->num_records; i++) + { + Z_NamePlusRecord *npr = rlist->records[i]; + + cl->records++; + if (npr->which != Z_NamePlusRecord_databaseRecord) + { + yaz_log(YLOG_WARN, + "Unexpected record type, probably diagnostic %s", + cl->database->database->url); + continue; + } + + rec = ingest_record(cl, npr->u.databaseRecord, cl->records); + if (!rec) + continue; + } + if (rlist->num_records) + session_alert_watch(s, SESSION_WATCH_RECORDS); + +#if USE_TIMING + yaz_timing_stop(t); + yaz_log(YLOG_LOG, "ingest_records %6.5f %3.2f %3.2f", + yaz_timing_get_real(t), yaz_timing_get_user(t), + yaz_timing_get_sys(t)); + yaz_timing_destroy(&t); +#endif +} + + +void client_search_response(struct client *cl, Z_APDU *a) +{ + struct session *se = cl->session; + Z_SearchResponse *r = a->u.searchResponse; + + yaz_log(YLOG_DEBUG, "Search response %s (status=%d)", + cl->database->database->url, *r->searchStatus); + + if (*r->searchStatus) + { + cl->hits = *r->resultCount; + se->total_hits += cl->hits; + if (r->presentStatus && !*r->presentStatus && r->records) + { + yaz_log(YLOG_DEBUG, "Records in search response %s", + cl->database->database->url); + ingest_records(cl, r->records); + } + cl->state = Client_Idle; + } + else + { /*"FAILED"*/ + cl->hits = 0; + cl->state = Client_Error; + if (r->records) { + Z_Records *recs = r->records; + if (recs->which == Z_Records_NSD) + { + yaz_log(YLOG_WARN, + "Search response: Non-surrogate diagnostic %s", + cl->database->database->url); + cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition; + cl->state = Client_Error; + } + } + } +} + +void client_present_response(struct client *cl, Z_APDU *a) +{ + Z_PresentResponse *r = a->u.presentResponse; + + if (r->records) { + Z_Records *recs = r->records; + if (recs->which == Z_Records_NSD) + { + yaz_log(YLOG_WARN, "Non-surrogate diagnostic %s", + cl->database->database->url); + cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition; + cl->state = Client_Error; + } + } + + if (!*r->presentStatus && cl->state != Client_Error) + { + yaz_log(YLOG_DEBUG, "Good Present response %s", + cl->database->database->url); + ingest_records(cl, r->records); + cl->state = Client_Idle; + } + else if (*r->presentStatus) + { + yaz_log(YLOG_WARN, "Bad Present response %s", + cl->database->database->url); + cl->state = Client_Error; + } +} + +void client_close_response(struct client *cl, Z_APDU *a) +{ + struct connection *co = cl->connection; + /* Z_Close *r = a->u.close; */ + + yaz_log(YLOG_WARN, "Close response %s", cl->database->database->url); + + cl->state = Client_Failed; + connection_destroy(co); +} + +int client_is_our_response(struct client *cl) +{ + struct session *se = client_get_session(cl); + + if (cl && (cl->requestid == se->requestid || + cl->state == Client_Initializing)) + return 1; + return 0; +} + +// Set authentication token in init if one is set for the client +// TODO: Extend this to handle other schemes than open (should be simple) +static void init_authentication(struct client *cl, Z_InitRequest *req) +{ + struct session_database *sdb = client_get_database(cl); + char *auth = session_setting_oneval(sdb, PZ_AUTHENTICATION); + + if (auth) + { + Z_IdAuthentication *idAuth = odr_malloc(global_parameters.odr_out, + sizeof(*idAuth)); + idAuth->which = Z_IdAuthentication_open; + idAuth->u.open = auth; + req->idAuthentication = idAuth; + } +} + +static void client_init_request(struct client *cl) +{ + Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_initRequest); + + a->u.initRequest->implementationId = global_parameters.implementationId; + a->u.initRequest->implementationName = global_parameters.implementationName; + a->u.initRequest->implementationVersion = + global_parameters.implementationVersion; + ODR_MASK_SET(a->u.initRequest->options, Z_Options_search); + ODR_MASK_SET(a->u.initRequest->options, Z_Options_present); + ODR_MASK_SET(a->u.initRequest->options, Z_Options_namedResultSets); + + ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_1); + ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_2); + ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_3); + + init_authentication(cl, a->u.initRequest); + + /* add virtual host if tunneling through Z39.50 proxy */ + + if (0 < strlen(global_parameters.zproxy_override) + && 0 < strlen(client_get_database(cl)->database->url)) + { +#if YAZ_VERSIONL >= 0x020163 + yaz_oi_set_string_oid(&a->u.initRequest->otherInfo, + global_parameters.odr_out, + yaz_oid_userinfo_proxy, + 1, client_get_database(cl)->database->url); +#else + yaz_oi_set_string_oidval(&a->u.initRequest->otherInfo, + global_parameters.odr_out, VAL_PROXY, + 1, client_get_database(cl->database)->url); +#endif + } + + if (send_apdu(cl, a) >= 0) + client_set_state(cl, Client_Initializing); + else + client_set_state(cl, Client_Error); + odr_reset(global_parameters.odr_out); +} + +void client_continue(struct client *cl) +{ + if (cl->state == Client_Connected) { + client_init_request(cl); + } + + if (cl->state == Client_Idle) + { + struct session *se = client_get_session(cl); + if (cl->requestid != se->requestid && cl->pquery) { + client_send_search(cl); + } + else if (cl->hits > 0 && cl->records < global_parameters.toget && + cl->records < cl->hits) { + client_send_present(cl); + } + } +} + +struct client *client_create(void) +{ + struct client *r; + if (client_freelist) + { + r = client_freelist; + client_freelist = client_freelist->next; + } + else + r = xmalloc(sizeof(struct client)); + r->pquery = 0; + r->database = 0; + r->connection = 0; + r->session = 0; + r->hits = 0; + r->records = 0; + r->setno = 0; + r->requestid = -1; + r->diagnostic = 0; + r->state = Client_Disconnected; + r->next = 0; + return r; +} + +void client_destroy(struct client *c) +{ + struct session *se = c->session; + if (c == se->clients) + se->clients = c->next; + else + { + struct client *cc; + for (cc = se->clients; cc && cc->next != c; cc = cc->next) + ; + if (cc) + cc->next = c->next; + } + if (c->connection) + connection_release(c->connection); + c->next = client_freelist; + client_freelist = c; +} + +void client_set_connection(struct client *cl, struct connection *con) +{ + cl->connection = con; +} + +void client_disconnect(struct client *cl) +{ + if (cl->state != Client_Idle) + cl->state = Client_Disconnected; + client_set_connection(cl, 0); +} + +// Extract terms from query into null-terminated termlist +static void extract_terms(NMEM nmem, struct ccl_rpn_node *query, char **termlist) +{ + int num = 0; + + pull_terms(nmem, query, termlist, &num); + termlist[num] = 0; +} + +// Initialize CCL map for a target +static CCL_bibset prepare_cclmap(struct client *cl) +{ + struct session_database *sdb = client_get_database(cl); + struct setting *s; + CCL_bibset res; + + if (!sdb->settings) + return 0; + res = ccl_qual_mk(); + for (s = sdb->settings[PZ_CCLMAP]; s; s = s->next) + { + char *p = strchr(s->name + 3, ':'); + if (!p) + { + yaz_log(YLOG_WARN, "Malformed cclmap name: %s", s->name); + ccl_qual_rm(&res); + return 0; + } + p++; + ccl_qual_fitem(res, s->value, p); + } + return res; +} + +// Parse the query given the settings specific to this client +int client_parse_query(struct client *cl, const char *query) +{ + struct session *se = client_get_session(cl); + struct ccl_rpn_node *cn; + int cerror, cpos; + CCL_bibset ccl_map = prepare_cclmap(cl); + + if (!ccl_map) + return -1; + cn = ccl_find_str(ccl_map, query, &cerror, &cpos); + ccl_qual_rm(&ccl_map); + if (!cn) + { + cl->state = Client_Error; + yaz_log(YLOG_WARN, "Failed to parse query for %s", + client_get_database(cl)->database->url); + return -1; + } + wrbuf_rewind(se->wrbuf); + ccl_pquery(se->wrbuf, cn); + wrbuf_putc(se->wrbuf, '\0'); + if (cl->pquery) + xfree(cl->pquery); + cl->pquery = xstrdup(wrbuf_buf(se->wrbuf)); + + if (!se->relevance) + { + // Initialize relevance structure with query terms + char *p[512]; + extract_terms(se->nmem, cn, p); + se->relevance = relevance_create(se->nmem, (const char **) p, + se->expected_maxrecs); + } + + ccl_rpn_delete(cn); + return 0; +} + +void client_set_session(struct client *cl, struct session *se) +{ + cl->session = se; + cl->next = se->clients; + se->clients = cl; +} + +int client_is_active(struct client *cl) +{ + if (cl->connection && (cl->state == Client_Connecting || + cl->state == Client_Initializing || + cl->state == Client_Searching || + cl->state == Client_Presenting)) + return 1; + return 0; +} + +struct client *client_next_in_session(struct client *cl) +{ + if (cl) + return cl->next; + return 0; + +} + +int client_get_hits(struct client *cl) +{ + return cl->hits; +} + +int client_get_num_records(struct client *cl) +{ + return cl->records; +} + +int client_get_diagnostic(struct client *cl) +{ + return cl->diagnostic; +} + +void client_set_database(struct client *cl, struct session_database *db) +{ + cl->database = db; +} + +struct host *client_get_host(struct client *cl) +{ + return client_get_database(cl)->database->host; +} + +const char *client_get_url(struct client *cl) +{ + return client_get_database(cl)->database->url; +} + +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ diff --git a/src/client.h b/src/client.h new file mode 100644 index 0000000..eb2abf1 --- /dev/null +++ b/src/client.h @@ -0,0 +1,93 @@ +/* $Id: client.h,v 1.1 2007-04-23 21:05:23 adam Exp $ + Copyright (c) 2006-2007, Index Data. + +This file is part of Pazpar2. + +Pazpar2 is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Pazpar2; see the file LICENSE. If not, write to the +Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA +02111-1307, USA. + */ + +/** \file client.h + \brief Z39.50 client +*/ + +#ifndef CLIENT_H +#define CLIENT_H + +struct client; +struct connection; + +enum client_state +{ + Client_Connecting, + Client_Connected, + Client_Idle, + Client_Initializing, + Client_Searching, + Client_Presenting, + Client_Error, + Client_Failed, + Client_Disconnected, + Client_Stopped +}; + +const char *client_get_state_str(struct client *cl); +enum client_state client_get_state(struct client *cl); +void client_set_state(struct client *cl, enum client_state st); +void client_fatal(struct client *cl); +struct connection *client_get_connection(struct client *cl); +struct session_database *client_get_database(struct client *cl); +void client_set_database(struct client *cl, struct session_database *db); +struct session *client_get_session(struct client *cl); +const char *client_get_pquery(struct client *cl); + +void client_set_requestid(struct client *cl, int id); + +void client_init_response(struct client *cl, Z_APDU *a); +void client_search_response(struct client *cl, Z_APDU *a); +void client_present_response(struct client *cl, Z_APDU *a); +void client_close_response(struct client *cl, Z_APDU *a); + +int client_is_our_response(struct client *cl); + +void client_continue(struct client *cl); + +struct client *client_create(void); +void client_destroy(struct client *c); + +void client_set_connection(struct client *cl, struct connection *con); +void client_disconnect(struct client *cl); +int client_prep_connection(struct client *cl); +void client_set_session(struct client *cl, struct session *se); +int client_is_active(struct client *cl); +struct client *client_next_in_session(struct client *cl); + +int client_parse_query(struct client *cl, const char *query); +int client_get_hits(struct client *cl); +int client_get_num_records(struct client *cl); +int client_get_diagnostic(struct client *cl); +void client_set_database(struct client *cl, struct session_database *db); +struct host *client_get_host(struct client *cl); +const char *client_get_url(struct client *cl); + +#endif + +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ diff --git a/src/connection.c b/src/connection.c new file mode 100644 index 0000000..fc7bf9f --- /dev/null +++ b/src/connection.c @@ -0,0 +1,433 @@ +/* $Id: connection.c,v 1.1 2007-04-23 21:05:23 adam Exp $ + Copyright (c) 2006-2007, Index Data. + +This file is part of Pazpar2. + +Pazpar2 is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Pazpar2; see the file LICENSE. If not, write to the +Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA +02111-1307, USA. + */ + +/** \file connection.c + \brief Z39.50 connection (low-level client) +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if HAVE_CONFIG_H +#include "cconfig.h" +#endif + +#include +#include +#include +#include "connection.h" +#include "eventl.h" +#include "pazpar2.h" +#include "host.h" +#include "client.h" +#include "parameters.h" + + +// Represents a physical, reusable connection to a remote Z39.50 host +struct connection { + IOCHAN iochan; + COMSTACK link; + struct host *host; + struct client *client; + char *ibuf; + int ibufsize; + enum { + Conn_Resolving, + Conn_Connecting, + Conn_Open, + Conn_Waiting, + } state; + struct connection *next; +}; + +static struct connection *connection_freelist = 0; + +void host_remove_connection(struct host *h, struct connection *con) +{ + struct connection **conp = &h->connections; + assert(con); + while (*conp) + { + if (*conp == con) + { + *conp = (*conp)->next; + return; + } + conp = &(*conp)->next; + } + assert(*conp == 0); +} + +// Close connection and recycle structure +void connection_destroy(struct connection *co) +{ + struct host *h = co->host; + + if (co->link) + { + cs_close(co->link); + iochan_destroy(co->iochan); + } + + yaz_log(YLOG_DEBUG, "Connection destroy %s", co->host->hostport); + + host_remove_connection(h, co); + if (co->client) + { + client_disconnect(co->client); + } + co->next = connection_freelist; + connection_freelist = co; +} + +// Creates a new connection for client, associated with the host of +// client's database +struct connection *connection_create(struct client *cl) +{ + struct connection *new; + struct host *host = client_get_host(cl); + + if ((new = connection_freelist)) + connection_freelist = new->next; + else + { + new = xmalloc(sizeof (struct connection)); + new->ibuf = 0; + new->ibufsize = 0; + } + new->host = host; + new->next = new->host->connections; + new->host->connections = new; + new->client = cl; + client_set_connection(cl, new); + new->link = 0; + new->state = Conn_Resolving; + if (host->ipport) + connection_connect(new); + return new; +} + +static void connection_handler(IOCHAN i, int event) +{ + struct connection *co = iochan_getdata(i); + struct client *cl = co->client; + struct session *se = 0; + + if (cl) + se = client_get_session(cl); + else + { + yaz_log(YLOG_WARN, "Destroying orphan connection"); + connection_destroy(co); + return; + } + + if (co->state == Conn_Connecting && event & EVENT_OUTPUT) + { + int errcode; + socklen_t errlen = sizeof(errcode); + + if (getsockopt(cs_fileno(co->link), SOL_SOCKET, SO_ERROR, &errcode, + &errlen) < 0 || errcode != 0) + { + client_fatal(cl); + return; + } + else + { + yaz_log(YLOG_DEBUG, "Connect OK"); + co->state = Conn_Open; + if (cl) + client_set_state(cl, Client_Connected); + } + } + + else if (event & EVENT_INPUT) + { + int len = cs_get(co->link, &co->ibuf, &co->ibufsize); + + if (len < 0) + { + yaz_log(YLOG_WARN|YLOG_ERRNO, "Error reading from %s", + client_get_url(cl)); + connection_destroy(co); + return; + } + else if (len == 0) + { + yaz_log(YLOG_WARN, "EOF reading from %s", client_get_url(cl)); + connection_destroy(co); + return; + } + else if (len > 1) // We discard input if we have no connection + { + co->state = Conn_Open; + + if (client_is_our_response(cl)) + { + Z_APDU *a; + + odr_reset(global_parameters.odr_in); + odr_setbuf(global_parameters.odr_in, co->ibuf, len, 0); + if (!z_APDU(global_parameters.odr_in, &a, 0, 0)) + { + client_fatal(cl); + return; + } + switch (a->which) + { + case Z_APDU_initResponse: + client_init_response(cl, a); + break; + case Z_APDU_searchResponse: + client_search_response(cl, a); + break; + case Z_APDU_presentResponse: + client_present_response(cl, a); + break; + case Z_APDU_close: + client_close_response(cl, a); + break; + default: + yaz_log(YLOG_WARN, + "Unexpected Z39.50 response from %s", + client_get_url(cl)); + client_fatal(cl); + return; + } + // We aren't expecting staggered output from target + // if (cs_more(t->link)) + // iochan_setevent(i, EVENT_INPUT); + } + else // we throw away response and go to idle mode + { + yaz_log(YLOG_DEBUG, "Ignoring result of expired operation"); + client_set_state(cl, Client_Idle); + } + } + /* if len==1 we do nothing but wait for more input */ + } + client_continue(cl); +} + +// Disassociate connection from client +void connection_release(struct connection *co) +{ + struct client *cl = co->client; + + yaz_log(YLOG_DEBUG, "Connection release %s", co->host->hostport); + if (!cl) + return; + client_set_connection(cl, 0); + co->client = 0; +} + +void connect_resolver_host(struct host *host) +{ + struct connection *con = host->connections; + while (con) + { + if (con->state == Conn_Resolving) + { + if (!host->ipport) /* unresolved */ + { + connection_destroy(con); + /* start all over .. at some point it will be NULL */ + con = host->connections; + } + else if (!con->client) + { + yaz_log(YLOG_WARN, "connect_unresolved_host : ophan client"); + connection_destroy(con); + /* start all over .. at some point it will be NULL */ + con = host->connections; + } + else + { + connection_connect(con); + con = con->next; + } + } + } +} + +int connection_send_apdu(struct connection *co, Z_APDU *a) +{ + char *buf; + int len, r; + + if (!z_APDU(global_parameters.odr_out, &a, 0, 0)) + { + odr_perror(global_parameters.odr_out, "Encoding APDU"); + abort(); + } + buf = odr_getbuf(global_parameters.odr_out, &len, 0); + r = cs_put(co->link, buf, len); + if (r < 0) + { + yaz_log(YLOG_WARN, "cs_put: %s", cs_errmsg(cs_errno(co->link))); + return -1; + } + else if (r == 1) + { + fprintf(stderr, "cs_put incomplete (ParaZ does not handle that)\n"); + exit(1); + } + odr_reset(global_parameters.odr_out); /* release the APDU structure */ + co->state = Conn_Waiting; + iochan_setflags(co->iochan, EVENT_INPUT); + return 0; +} + +struct host *connection_get_host(struct connection *con) +{ + return con->host; +} + +int connection_connect(struct connection *con) +{ + COMSTACK link = 0; + struct host *host = connection_get_host(con); + void *addr; + int res; + + assert(host->ipport); + assert(con); + + if (!(link = cs_create(tcpip_type, 0, PROTO_Z3950))) + { + yaz_log(YLOG_FATAL|YLOG_ERRNO, "Failed to create comstack"); + return -1; + } + + if (0 == strlen(global_parameters.zproxy_override)){ + /* no Z39.50 proxy needed - direct connect */ + yaz_log(YLOG_DEBUG, "Connection create %s", connection_get_url(con)); + + if (!(addr = cs_straddr(link, host->ipport))) + { + yaz_log(YLOG_WARN|YLOG_ERRNO, + "Lookup of IP address %s failed", host->ipport); + return -1; + } + + } else { + /* Z39.50 proxy connect */ + yaz_log(YLOG_DEBUG, "Connection create %s proxy %s", + connection_get_url(con), global_parameters.zproxy_override); + + if (!(addr = cs_straddr(link, global_parameters.zproxy_override))) + { + yaz_log(YLOG_WARN|YLOG_ERRNO, + "Lookup of IP address %s failed", + global_parameters.zproxy_override); + return -1; + } + } + + res = cs_connect(link, addr); + if (res < 0) + { + yaz_log(YLOG_WARN|YLOG_ERRNO, "cs_connect %s", + connection_get_url(con)); + return -1; + } + con->link = link; + con->state = Conn_Connecting; + con->iochan = iochan_create(cs_fileno(link), connection_handler, 0); + iochan_setdata(con->iochan, con); + pazpar2_add_channel(con->iochan); + + /* this fragment is bad DRY: from client_prep_connection */ + client_set_state(con->client, Client_Connecting); + iochan_setflag(con->iochan, EVENT_OUTPUT); + return 0; +} + +const char *connection_get_url(struct connection *co) +{ + return client_get_url(co->client); +} + +// Ensure that client has a connection associated +int client_prep_connection(struct client *cl) +{ + struct connection *co; + struct session *se = client_get_session(cl); + struct host *host = client_get_host(cl); + + co = client_get_connection(cl); + + yaz_log(YLOG_DEBUG, "Client prep %s", client_get_url(cl)); + + if (!co) + { + // See if someone else has an idle connection + // We should look at timestamps here to select the longest-idle connection + for (co = host->connections; co; co = co->next) + if (co->state == Conn_Open && (!co->client || client_get_session(co->client) != se)) + break; + if (co) + { + connection_release(co); + client_set_connection(cl, co); + co->client = cl; + } + else + co = connection_create(cl); + } + if (co) + { + if (co->state == Conn_Connecting) + { + client_set_state(cl, Client_Connecting); + iochan_setflag(co->iochan, EVENT_OUTPUT); + } + else if (co->state == Conn_Open) + { + if (client_get_state(cl) == Client_Error + || client_get_state(cl) == Client_Disconnected) + client_set_state(cl, Client_Idle); + iochan_setflag(co->iochan, EVENT_OUTPUT); + } + return 1; + } + else + return 0; +} + + + +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ diff --git a/src/connection.h b/src/connection.h new file mode 100644 index 0000000..4b8a561 --- /dev/null +++ b/src/connection.h @@ -0,0 +1,56 @@ +/* $Id: connection.h,v 1.1 2007-04-23 21:05:23 adam Exp $ + Copyright (c) 2006-2007, Index Data. + +This file is part of Pazpar2. + +Pazpar2 is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Pazpar2; see the file LICENSE. If not, write to the +Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA +02111-1307, USA. + */ + +/** \file connection.h + \brief Z39.50 connection (low-level client) +*/ + +#ifndef CONNECTION_H +#define CONNECTION_H + +#include +#include "eventl.h" + +struct client; +struct connection; +struct host; +struct session; + +void connection_destroy(struct connection *co); +struct connection *connection_create(struct client *cl); +void connect_resolver_host(struct host *host); +int connection_send_apdu(struct connection *co, Z_APDU *a); +struct host *connection_get_host(struct connection *con); +int connection_connect(struct connection *con); +struct connection *connection_get_available(struct connection *con_list, + struct session *se); +int connection_prep_connection(struct connection *co, struct session *se); +const char *connection_get_url(struct connection *co); + +#endif + +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ diff --git a/src/database.c b/src/database.c index f43cc75..342cbc9 100644 --- a/src/database.c +++ b/src/database.c @@ -1,4 +1,4 @@ -/* $Id: database.c,v 1.24 2007-04-22 15:00:56 adam Exp $ +/* $Id: database.c,v 1.25 2007-04-23 21:05:23 adam Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -29,6 +29,7 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include #include "pazpar2.h" +#include "host.h" #include "config.h" #include "settings.h" #include "http.h" diff --git a/src/getaddrinfo.c b/src/getaddrinfo.c index 8f0c7d9..c400e07 100644 --- a/src/getaddrinfo.c +++ b/src/getaddrinfo.c @@ -1,4 +1,4 @@ -/* $Id: getaddrinfo.c,v 1.4 2007-04-23 08:06:21 adam Exp $ +/* $Id: getaddrinfo.c,v 1.5 2007-04-23 21:05:23 adam Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -39,6 +39,8 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include #include "pazpar2.h" +#include "connection.h" +#include "host.h" struct work { char *hostport; /* hostport to be resolved in separate thread */ diff --git a/src/host.h b/src/host.h new file mode 100644 index 0000000..e2aa043 --- /dev/null +++ b/src/host.h @@ -0,0 +1,42 @@ +/* $Id: host.h,v 1.1 2007-04-23 21:05:23 adam Exp $ + Copyright (c) 2006-2007, Index Data. + +This file is part of Pazpar2. + +Pazpar2 is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Pazpar2; see the file LICENSE. If not, write to the +Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA +02111-1307, USA. + */ + +#ifndef HOST_H +#define HOST_H + +/** \brief Represents a host (irrespective of databases) */ +struct host { + char *hostport; + char *ipport; + struct connection *connections; // All connections to this + struct host *next; +}; + + +#endif + +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ diff --git a/src/http_command.c b/src/http_command.c index f44fdf3..35bed6b 100644 --- a/src/http_command.c +++ b/src/http_command.c @@ -1,4 +1,4 @@ -/* $Id: http_command.c,v 1.40 2007-04-19 16:07:20 adam Exp $ +/* $Id: http_command.c,v 1.41 2007-04-23 21:05:23 adam Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -20,7 +20,7 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA */ /* - * $Id: http_command.c,v 1.40 2007-04-19 16:07:20 adam Exp $ + * $Id: http_command.c,v 1.41 2007-04-23 21:05:23 adam Exp $ */ #include @@ -45,6 +45,7 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include "http.h" #include "http_command.h" #include "settings.h" +#include "client.h" // Update this when the protocol changes #define PAZPAR2_PROTOCOL_VERSION "1" @@ -385,10 +386,10 @@ static void write_metadata(WRBUF w, struct conf_service *service, static void write_subrecord(struct record *r, WRBUF w, struct conf_service *service, int show_details) { - char *name = session_setting_oneval(r->client->database, PZ_NAME); + char *name = session_setting_oneval(client_get_database(r->client), PZ_NAME); wrbuf_printf(w, "", - r->client->database->database->url, + client_get_database(r->client)->database->url, *name ? name : "Unknown"); if (show_details) write_metadata(w, service, r->metadata, 1); diff --git a/src/logic.c b/src/logic.c index 2a0e961..b3d2a2b 100644 --- a/src/logic.c +++ b/src/logic.c @@ -1,4 +1,4 @@ -/* $Id: logic.c,v 1.19 2007-04-23 08:15:22 marc Exp $ +/* $Id: logic.c,v 1.20 2007-04-23 21:05:23 adam Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -19,8 +19,9 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ -// This file contains the primary business logic. Several parts of it should -// Eventually be factored into separate modules. +/** \file logic.c + \brief high-level logic; mostly user sessions and settings +*/ #include #include @@ -67,32 +68,11 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include "relevance.h" #include "config.h" #include "database.h" +#include "client.h" #include "settings.h" #define MAX_CHUNK 15 -static void client_fatal(struct client *cl); -static void connection_destroy(struct connection *co); -static int client_prep_connection(struct client *cl); -static void ingest_records(struct client *cl, Z_Records *r); -void session_alert_watch(struct session *s, int what); - -static struct connection *connection_freelist = 0; -static struct client *client_freelist = 0; - -static char *client_states[] = { - "Client_Connecting", - "Client_Connected", - "Client_Idle", - "Client_Initializing", - "Client_Searching", - "Client_Presenting", - "Client_Error", - "Client_Failed", - "Client_Disconnected", - "Client_Stopped" -}; - // Note: Some things in this structure will eventually move to configuration struct parameters global_parameters = { @@ -114,101 +94,9 @@ struct parameters global_parameters = 0 }; -static int send_apdu(struct client *c, Z_APDU *a) -{ - struct connection *co = c->connection; - char *buf; - int len, r; - - if (!z_APDU(global_parameters.odr_out, &a, 0, 0)) - { - odr_perror(global_parameters.odr_out, "Encoding APDU"); - abort(); - } - buf = odr_getbuf(global_parameters.odr_out, &len, 0); - r = cs_put(co->link, buf, len); - if (r < 0) - { - yaz_log(YLOG_WARN, "cs_put: %s", cs_errmsg(cs_errno(co->link))); - return -1; - } - else if (r == 1) - { - fprintf(stderr, "cs_put incomplete (ParaZ does not handle that)\n"); - exit(1); - } - odr_reset(global_parameters.odr_out); /* release the APDU structure */ - co->state = Conn_Waiting; - return 0; -} - -// Set authentication token in init if one is set for the client -// TODO: Extend this to handle other schemes than open (should be simple) -static void init_authentication(struct client *cl, Z_InitRequest *req) -{ - struct session_database *sdb = cl->database; - char *auth = session_setting_oneval(sdb, PZ_AUTHENTICATION); - - if (auth) - { - Z_IdAuthentication *idAuth = odr_malloc(global_parameters.odr_out, - sizeof(*idAuth)); - idAuth->which = Z_IdAuthentication_open; - idAuth->u.open = auth; - req->idAuthentication = idAuth; - } -} - -static void send_init(IOCHAN i) -{ - - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_initRequest); - - a->u.initRequest->implementationId = global_parameters.implementationId; - a->u.initRequest->implementationName = global_parameters.implementationName; - a->u.initRequest->implementationVersion = - global_parameters.implementationVersion; - ODR_MASK_SET(a->u.initRequest->options, Z_Options_search); - ODR_MASK_SET(a->u.initRequest->options, Z_Options_present); - ODR_MASK_SET(a->u.initRequest->options, Z_Options_namedResultSets); - - ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_1); - ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_2); - ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_3); - - init_authentication(cl, a->u.initRequest); - - /* add virtual host if tunneling through Z39.50 proxy */ - - if (0 < strlen(global_parameters.zproxy_override) - && 0 < strlen(cl->database->database->url)) - { -#if YAZ_VERSIONL >= 0x020163 - yaz_oi_set_string_oid(&a->u.initRequest->otherInfo, - global_parameters.odr_out, - yaz_oid_userinfo_proxy, - 1, cl->database->database->url); -#else - yaz_oi_set_string_oidval(&a->u.initRequest->otherInfo, - global_parameters.odr_out, VAL_PROXY, - 1, cl->database->database->url); -#endif - } - - if (send_apdu(cl, a) >= 0) - { - iochan_setflags(i, EVENT_INPUT); - cl->state = Client_Initializing; - } - else - cl->state = Client_Error; - odr_reset(global_parameters.odr_out); -} // Recursively traverse query structure to extract terms. -static void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) +void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) { char **words; int numwords; @@ -233,227 +121,6 @@ static void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int * } } -// Extract terms from query into null-terminated termlist -static void extract_terms(NMEM nmem, struct ccl_rpn_node *query, char **termlist) -{ - int num = 0; - - pull_terms(nmem, query, termlist, &num); - termlist[num] = 0; -} - -static void send_search(IOCHAN i) -{ - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - struct session *se = cl->session; - struct session_database *sdb = cl->database; - Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_searchRequest); - int ndb; - char **databaselist; - Z_Query *zquery; - int ssub = 0, lslb = 100000, mspn = 10; - char *recsyn = 0; - char *piggyback = 0; - char *queryenc = 0; - yaz_iconv_t iconv = 0; - - yaz_log(YLOG_DEBUG, "Sending search to %s", cl->database->database->url); - - // constructing RPN query - a->u.searchRequest->query = zquery = odr_malloc(global_parameters.odr_out, - sizeof(Z_Query)); - zquery->which = Z_Query_type_1; - zquery->u.type_1 = p_query_rpn(global_parameters.odr_out, cl->pquery); - - // converting to target encoding - if ((queryenc = session_setting_oneval(sdb, PZ_QUERYENCODING))){ - iconv = yaz_iconv_open(queryenc, "UTF-8"); - if (iconv){ - yaz_query_charset_convert_rpnquery(zquery->u.type_1, - global_parameters.odr_out, - iconv); - yaz_iconv_close(iconv); - } else - yaz_log(YLOG_WARN, "Query encoding failed %s %s", - cl->database->database->url, queryenc); - } - - for (ndb = 0; sdb->database->databases[ndb]; ndb++) - ; - databaselist = odr_malloc(global_parameters.odr_out, sizeof(char*) * ndb); - for (ndb = 0; sdb->database->databases[ndb]; ndb++) - databaselist[ndb] = sdb->database->databases[ndb]; - - if (!(piggyback = session_setting_oneval(sdb, PZ_PIGGYBACK)) || *piggyback == '1') - { - if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX))) - { -#if YAZ_VERSIONL >= 0x020163 - a->u.searchRequest->preferredRecordSyntax = - yaz_string_to_oid_odr(yaz_oid_std(), - CLASS_RECSYN, recsyn, - global_parameters.odr_out); -#else - a->u.searchRequest->preferredRecordSyntax = - yaz_str_to_z3950oid(global_parameters.odr_out, - CLASS_RECSYN, recsyn); -#endif - } - a->u.searchRequest->smallSetUpperBound = &ssub; - a->u.searchRequest->largeSetLowerBound = &lslb; - a->u.searchRequest->mediumSetPresentNumber = &mspn; - } - a->u.searchRequest->resultSetName = "Default"; - a->u.searchRequest->databaseNames = databaselist; - a->u.searchRequest->num_databaseNames = ndb; - - - { //scope for sending and logging queries - WRBUF wbquery = wrbuf_alloc(); - yaz_query_to_wrbuf(wbquery, zquery); - - - if (send_apdu(cl, a) >= 0) - { - iochan_setflags(i, EVENT_INPUT); - cl->state = Client_Searching; - cl->requestid = se->requestid; - yaz_log(YLOG_LOG, "SearchRequest %s %s %s", - cl->database->database->url, - queryenc ? queryenc : "UTF-8", - wrbuf_cstr(wbquery)); - } - else { - cl->state = Client_Error; - yaz_log(YLOG_WARN, "Failed SearchRequest %s %s %s", - cl->database->database->url, - queryenc ? queryenc : "UTF-8", - wrbuf_cstr(wbquery)); - } - - wrbuf_destroy(wbquery); - } - - odr_reset(global_parameters.odr_out); -} - -static void send_present(IOCHAN i) -{ - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - struct session_database *sdb = cl->database; - Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_presentRequest); - int toget; - int start = cl->records + 1; - char *recsyn; - - toget = global_parameters.chunk; - if (toget > global_parameters.toget - cl->records) - toget = global_parameters.toget - cl->records; - if (toget > cl->hits - cl->records) - toget = cl->hits - cl->records; - - yaz_log(YLOG_DEBUG, "Trying to present %d records\n", toget); - - a->u.presentRequest->resultSetStartPoint = &start; - a->u.presentRequest->numberOfRecordsRequested = &toget; - - a->u.presentRequest->resultSetId = "Default"; - - if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX))) - { -#if YAZ_VERSIONL >= 0x020163 - a->u.presentRequest->preferredRecordSyntax = - yaz_string_to_oid_odr(yaz_oid_std(), - CLASS_RECSYN, recsyn, - global_parameters.odr_out); -#else - a->u.presentRequest->preferredRecordSyntax = - yaz_str_to_z3950oid(global_parameters.odr_out, - CLASS_RECSYN, recsyn); -#endif - } - - if (send_apdu(cl, a) >= 0) - { - iochan_setflags(i, EVENT_INPUT); - cl->state = Client_Presenting; - } - else - cl->state = Client_Error; - odr_reset(global_parameters.odr_out); -} - -static void do_initResponse(IOCHAN i, Z_APDU *a) -{ - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - Z_InitResponse *r = a->u.initResponse; - - yaz_log(YLOG_DEBUG, "Init response %s", cl->database->database->url); - - if (*r->result) - { - cl->state = Client_Idle; - } - else - cl->state = Client_Failed; // FIXME need to do something to the connection -} - -static void do_searchResponse(IOCHAN i, Z_APDU *a) -{ - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - struct session *se = cl->session; - Z_SearchResponse *r = a->u.searchResponse; - - yaz_log(YLOG_DEBUG, "Search response %s (status=%d)", - cl->database->database->url, *r->searchStatus); - - if (*r->searchStatus) - { - cl->hits = *r->resultCount; - se->total_hits += cl->hits; - if (r->presentStatus && !*r->presentStatus && r->records) - { - yaz_log(YLOG_DEBUG, "Records in search response %s", - cl->database->database->url); - ingest_records(cl, r->records); - } - cl->state = Client_Idle; - } - else - { /*"FAILED"*/ - cl->hits = 0; - cl->state = Client_Error; - if (r->records) { - Z_Records *recs = r->records; - if (recs->which == Z_Records_NSD) - { - yaz_log(YLOG_WARN, - "Search response: Non-surrogate diagnostic %s", - cl->database->database->url); - cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition; - cl->state = Client_Error; - } - } - } -} - -static void do_closeResponse(IOCHAN i, Z_APDU *a) -{ - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - /* Z_Close *r = a->u.close; */ - - yaz_log(YLOG_WARN, "Close response %s", cl->database->database->url); - - cl->state = Client_Failed; - connection_destroy(co); -} - - char *normalize_mergekey(char *buf, int skiparticle) { char *p = buf, *pout = buf; @@ -495,6 +162,36 @@ char *normalize_mergekey(char *buf, int skiparticle) return buf; } +// Extract what appears to be years from buf, storing highest and +// lowest values. +static int extract_years(const char *buf, int *first, int *last) +{ + *first = -1; + *last = -1; + while (*buf) + { + const char *e; + int len; + + while (*buf && !isdigit(*buf)) + buf++; + len = 0; + for (e = buf; *e && isdigit(*e); e++) + len++; + if (len == 4) + { + int value = atoi(buf); + if (*first < 0 || value < *first) + *first = value; + if (*last < 0 || value > *last) + *last = value; + } + buf = e; + } + return *first; +} + + static void add_facet(struct session *s, const char *type, const char *value) { int i; @@ -519,10 +216,9 @@ static void add_facet(struct session *s, const char *type, const char *value) termlist_insert(s->termlists[i].termlist, value); } -static xmlDoc *normalize_record(struct client *cl, Z_External *rec) +xmlDoc *normalize_record(struct session_database *sdb, Z_External *rec) { struct database_retrievalmap *m; - struct session_database *sdb = cl->database; struct database *db = sdb->database; xmlNode *res; xmlDoc *rdoc; @@ -588,7 +284,7 @@ static xmlDoc *normalize_record(struct client *cl, Z_External *rec) if (!new || !root || !(root->children)) { yaz_log(YLOG_WARN, "XSLT transformation failed from %s", - cl->database->database->url); + sdb->database->url); xmlFreeDoc(new); xmlFreeDoc(rdoc); return 0; @@ -620,7 +316,7 @@ static xmlDoc *normalize_record(struct client *cl, Z_External *rec) if (global_parameters.dump_records) { fprintf(stderr, "Record from %s\n----------------\n", - cl->database->database->url); + sdb->database->url); #if LIBXML_VERSION >= 20600 xmlDocFormatDump(stderr, rdoc, 1); #else @@ -630,733 +326,71 @@ static xmlDoc *normalize_record(struct client *cl, Z_External *rec) return rdoc; } -// Extract what appears to be years from buf, storing highest and -// lowest values. -static int extract_years(const char *buf, int *first, int *last) +// Retrieve first defined value for 'name' for given database. +// Will be extended to take into account user associated with session +char *session_setting_oneval(struct session_database *db, int offset) { - *first = -1; - *last = -1; - while (*buf) + if (!db->settings[offset]) + return ""; + return db->settings[offset]->value; +} + + + +// Initialize YAZ Map structures for MARC-based targets +static int prepare_yazmarc(struct session_database *sdb) +{ + char *s; + + if (!sdb->settings) { - const char *e; - int len; + yaz_log(YLOG_WARN, "No settings for %s", sdb->database->url); + return -1; + } + if ((s = session_setting_oneval(sdb, PZ_NATIVESYNTAX)) && !strncmp(s, "iso2709", 7)) + { + char *encoding = "marc-8s", *e; + yaz_iconv_t cm; - while (*buf && !isdigit(*buf)) - buf++; - len = 0; - for (e = buf; *e && isdigit(*e); e++) - len++; - if (len == 4) + // See if a native encoding is specified + if ((e = strchr(s, ';'))) + encoding = e + 1; + + sdb->yaz_marc = yaz_marc_create(); + yaz_marc_subfield_str(sdb->yaz_marc, "\t"); + + cm = yaz_iconv_open("utf-8", encoding); + if (!cm) { - int value = atoi(buf); - if (*first < 0 || value < *first) - *first = value; - if (*last < 0 || value > *last) - *last = value; + yaz_log(YLOG_FATAL, + "Unable to map from %s to UTF-8 for target %s", + encoding, sdb->database->url); + return -1; } - buf = e; + yaz_marc_iconv(sdb->yaz_marc, cm); } - return *first; + return 0; } -static struct record *ingest_record(struct client *cl, Z_External *rec) +// Prepare XSLT stylesheets for record normalization +// Structures are allocated on the session_wide nmem to avoid having +// to recompute this for every search. This would lead +// to leaking if a single session was to repeatedly change the PZ_XSLT +// setting. However, this is not a realistic use scenario. +static int prepare_map(struct session *se, struct session_database *sdb) { - xmlDoc *xdoc = normalize_record(cl, rec); - xmlNode *root, *n; - struct record *res; - struct record_cluster *cluster; - struct session *se = cl->session; - xmlChar *mergekey, *mergekey_norm; - xmlChar *type = 0; - xmlChar *value = 0; - struct conf_service *service = global_parameters.server->service; - - if (!xdoc) - return 0; + char *s; - root = xmlDocGetRootElement(xdoc); - if (!(mergekey = xmlGetProp(root, (xmlChar *) "mergekey"))) + if (!sdb->settings) { - yaz_log(YLOG_WARN, "No mergekey found in record"); - xmlFreeDoc(xdoc); - return 0; + yaz_log(YLOG_WARN, "No settings on %s", sdb->database->url); + return -1; } - - res = nmem_malloc(se->nmem, sizeof(struct record)); - res->next = 0; - res->client = cl; - res->metadata = nmem_malloc(se->nmem, - sizeof(struct record_metadata*) * service->num_metadata); - memset(res->metadata, 0, sizeof(struct record_metadata*) * service->num_metadata); - - mergekey_norm = (xmlChar *) nmem_strdup(se->nmem, (char*) mergekey); - xmlFree(mergekey); - normalize_mergekey((char *) mergekey_norm, 0); - - cluster = reclist_insert(se->reclist, - global_parameters.server->service, - res, (char *) mergekey_norm, - &se->total_merged); - if (global_parameters.dump_records) - yaz_log(YLOG_LOG, "Cluster id %d from %s (#%d)", cluster->recid, - cl->database->database->url, cl->records); - if (!cluster) + if ((s = session_setting_oneval(sdb, PZ_XSLT))) { - /* no room for record */ - xmlFreeDoc(xdoc); - return 0; - } - relevance_newrec(se->relevance, cluster); - - for (n = root->children; n; n = n->next) - { - if (type) - xmlFree(type); - if (value) - xmlFree(value); - type = value = 0; - - if (n->type != XML_ELEMENT_NODE) - continue; - if (!strcmp((const char *) n->name, "metadata")) - { - struct conf_metadata *md = 0; - struct conf_sortkey *sk = 0; - struct record_metadata **wheretoput, *newm; - int imeta; - int first, last; - - type = xmlGetProp(n, (xmlChar *) "type"); - value = xmlNodeListGetString(xdoc, n->children, 0); - - if (!type || !value) - continue; - - // First, find out what field we're looking at - for (imeta = 0; imeta < service->num_metadata; imeta++) - if (!strcmp((const char *) type, service->metadata[imeta].name)) - { - md = &service->metadata[imeta]; - if (md->sortkey_offset >= 0) - sk = &service->sortkeys[md->sortkey_offset]; - break; - } - if (!md) - { - yaz_log(YLOG_WARN, "Ignoring unknown metadata element: %s", type); - continue; - } - - // Find out where we are putting it - if (md->merge == Metadata_merge_no) - wheretoput = &res->metadata[imeta]; - else - wheretoput = &cluster->metadata[imeta]; - - // Put it there - newm = nmem_malloc(se->nmem, sizeof(struct record_metadata)); - newm->next = 0; - if (md->type == Metadata_type_generic) - { - char *p, *pe; - for (p = (char *) value; *p && isspace(*p); p++) - ; - for (pe = p + strlen(p) - 1; - pe > p && strchr(" ,/.:([", *pe); pe--) - *pe = '\0'; - newm->data.text = nmem_strdup(se->nmem, p); - - } - else if (md->type == Metadata_type_year) - { - if (extract_years((char *) value, &first, &last) < 0) - continue; - } - else - { - yaz_log(YLOG_WARN, "Unknown type in metadata element %s", type); - continue; - } - if (md->type == Metadata_type_year && md->merge != Metadata_merge_range) - { - yaz_log(YLOG_WARN, "Only range merging supported for years"); - continue; - } - if (md->merge == Metadata_merge_unique) - { - struct record_metadata *mnode; - for (mnode = *wheretoput; mnode; mnode = mnode->next) - if (!strcmp((const char *) mnode->data.text, newm->data.text)) - break; - if (!mnode) - { - newm->next = *wheretoput; - *wheretoput = newm; - } - } - else if (md->merge == Metadata_merge_longest) - { - if (!*wheretoput || - strlen(newm->data.text) > strlen((*wheretoput)->data.text)) - { - *wheretoput = newm; - if (sk) - { - char *s = nmem_strdup(se->nmem, newm->data.text); - if (!cluster->sortkeys[md->sortkey_offset]) - cluster->sortkeys[md->sortkey_offset] = - nmem_malloc(se->nmem, sizeof(union data_types)); - normalize_mergekey(s, - (sk->type == Metadata_sortkey_skiparticle)); - cluster->sortkeys[md->sortkey_offset]->text = s; - } - } - } - else if (md->merge == Metadata_merge_all || md->merge == Metadata_merge_no) - { - newm->next = *wheretoput; - *wheretoput = newm; - } - else if (md->merge == Metadata_merge_range) - { - assert(md->type == Metadata_type_year); - if (!*wheretoput) - { - *wheretoput = newm; - (*wheretoput)->data.number.min = first; - (*wheretoput)->data.number.max = last; - if (sk) - cluster->sortkeys[md->sortkey_offset] = &newm->data; - } - else - { - if (first < (*wheretoput)->data.number.min) - (*wheretoput)->data.number.min = first; - if (last > (*wheretoput)->data.number.max) - (*wheretoput)->data.number.max = last; - } -#ifdef GAGA - if (sk) - { - union data_types *sdata = cluster->sortkeys[md->sortkey_offset]; - yaz_log(YLOG_LOG, "SK range: %d-%d", sdata->number.min, sdata->number.max); - } -#endif - } - else - yaz_log(YLOG_WARN, "Don't know how to merge on element name %s", md->name); - - if (md->rank) - relevance_countwords(se->relevance, cluster, - (char *) value, md->rank); - if (md->termlist) - { - if (md->type == Metadata_type_year) - { - char year[64]; - sprintf(year, "%d", last); - add_facet(se, (char *) type, year); - if (first != last) - { - sprintf(year, "%d", first); - add_facet(se, (char *) type, year); - } - } - else - add_facet(se, (char *) type, (char *) value); - } - xmlFree(type); - xmlFree(value); - type = value = 0; - } - else - yaz_log(YLOG_WARN, "Unexpected element %s in internal record", n->name); - } - if (type) - xmlFree(type); - if (value) - xmlFree(value); - - xmlFreeDoc(xdoc); - - relevance_donerecord(se->relevance, cluster); - se->total_records++; - - return res; -} - -// Retrieve first defined value for 'name' for given database. -// Will be extended to take into account user associated with session -char *session_setting_oneval(struct session_database *db, int offset) -{ - if (!db->settings[offset]) - return ""; - return db->settings[offset]->value; -} - -static void ingest_records(struct client *cl, Z_Records *r) -{ -#if USE_TIMING - yaz_timing_t t = yaz_timing_create(); -#endif - struct record *rec; - struct session *s = cl->session; - Z_NamePlusRecordList *rlist; - int i; - - if (r->which != Z_Records_DBOSD) - return; - rlist = r->u.databaseOrSurDiagnostics; - for (i = 0; i < rlist->num_records; i++) - { - Z_NamePlusRecord *npr = rlist->records[i]; - - cl->records++; - if (npr->which != Z_NamePlusRecord_databaseRecord) - { - yaz_log(YLOG_WARN, - "Unexpected record type, probably diagnostic %s", - cl->database->database->url); - continue; - } - - rec = ingest_record(cl, npr->u.databaseRecord); - if (!rec) - continue; - } - if (s->watchlist[SESSION_WATCH_RECORDS].fun && rlist->num_records) - session_alert_watch(s, SESSION_WATCH_RECORDS); - -#if USE_TIMING - yaz_timing_stop(t); - yaz_log(YLOG_LOG, "ingest_records %6.5f %3.2f %3.2f", - yaz_timing_get_real(t), yaz_timing_get_user(t), - yaz_timing_get_sys(t)); - yaz_timing_destroy(&t); -#endif -} - -static void do_presentResponse(IOCHAN i, Z_APDU *a) -{ - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - Z_PresentResponse *r = a->u.presentResponse; - - if (r->records) { - Z_Records *recs = r->records; - if (recs->which == Z_Records_NSD) - { - yaz_log(YLOG_WARN, "Non-surrogate diagnostic %s", - cl->database->database->url); - cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition; - cl->state = Client_Error; - } - } - - if (!*r->presentStatus && cl->state != Client_Error) - { - yaz_log(YLOG_DEBUG, "Good Present response %s", - cl->database->database->url); - ingest_records(cl, r->records); - cl->state = Client_Idle; - } - else if (*r->presentStatus) - { - yaz_log(YLOG_WARN, "Bad Present response %s", - cl->database->database->url); - cl->state = Client_Error; - } -} - -void connection_handler(IOCHAN i, int event) -{ - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - struct session *se = 0; - - if (cl) - se = cl->session; - else - { - yaz_log(YLOG_WARN, "Destroying orphan connection"); - connection_destroy(co); - return; - } - - if (co->state == Conn_Connecting && event & EVENT_OUTPUT) - { - int errcode; - socklen_t errlen = sizeof(errcode); - - if (getsockopt(cs_fileno(co->link), SOL_SOCKET, SO_ERROR, &errcode, - &errlen) < 0 || errcode != 0) - { - client_fatal(cl); - return; - } - else - { - yaz_log(YLOG_DEBUG, "Connect OK"); - co->state = Conn_Open; - if (cl) - cl->state = Client_Connected; - } - } - - else if (event & EVENT_INPUT) - { - int len = cs_get(co->link, &co->ibuf, &co->ibufsize); - - if (len < 0) - { - yaz_log(YLOG_WARN|YLOG_ERRNO, "Error reading from %s", - cl->database->database->url); - connection_destroy(co); - return; - } - else if (len == 0) - { - yaz_log(YLOG_WARN, "EOF reading from %s", cl->database->database->url); - connection_destroy(co); - return; - } - else if (len > 1) // We discard input if we have no connection - { - co->state = Conn_Open; - - if (cl && (cl->requestid == se->requestid || cl->state == Client_Initializing)) - { - Z_APDU *a; - - odr_reset(global_parameters.odr_in); - odr_setbuf(global_parameters.odr_in, co->ibuf, len, 0); - if (!z_APDU(global_parameters.odr_in, &a, 0, 0)) - { - client_fatal(cl); - return; - } - switch (a->which) - { - case Z_APDU_initResponse: - do_initResponse(i, a); - break; - case Z_APDU_searchResponse: - do_searchResponse(i, a); - break; - case Z_APDU_presentResponse: - do_presentResponse(i, a); - break; - case Z_APDU_close: - do_closeResponse(i, a); - break; - default: - yaz_log(YLOG_WARN, - "Unexpected Z39.50 response from %s", - cl->database->database->url); - client_fatal(cl); - return; - } - // We aren't expecting staggered output from target - // if (cs_more(t->link)) - // iochan_setevent(i, EVENT_INPUT); - } - else // we throw away response and go to idle mode - { - yaz_log(YLOG_DEBUG, "Ignoring result of expired operation"); - cl->state = Client_Idle; - } - } - /* if len==1 we do nothing but wait for more input */ - } - - if (cl->state == Client_Connected) { - send_init(i); - } - - if (cl->state == Client_Idle) - { - if (cl->requestid != se->requestid && cl->pquery) { - send_search(i); - } - else if (cl->hits > 0 && cl->records < global_parameters.toget && - cl->records < cl->hits) { - send_present(i); - } - } -} - -// Disassociate connection from client -static void connection_release(struct connection *co) -{ - struct client *cl = co->client; - - yaz_log(YLOG_DEBUG, "Connection release %s", co->host->hostport); - if (!cl) - return; - cl->connection = 0; - co->client = 0; -} - -// Close connection and recycle structure -static void connection_destroy(struct connection *co) -{ - struct host *h = co->host; - - if (co->link) - { - cs_close(co->link); - iochan_destroy(co->iochan); - } - - yaz_log(YLOG_DEBUG, "Connection destroy %s", co->host->hostport); - if (h->connections == co) - h->connections = co->next; - else - { - struct connection *pco; - for (pco = h->connections; pco && pco->next != co; pco = pco->next) - ; - if (pco) - pco->next = co->next; - else - abort(); - } - if (co->client) - { - if (co->client->state != Client_Idle) - co->client->state = Client_Disconnected; - co->client->connection = 0; - } - co->next = connection_freelist; - connection_freelist = co; -} - -static int connection_connect(struct connection *con) -{ - COMSTACK link = 0; - struct client *cl = con->client; - struct host *host = con->host; - void *addr; - int res; - - assert(host->ipport); - assert(cl); - - if (!(link = cs_create(tcpip_type, 0, PROTO_Z3950))) - { - yaz_log(YLOG_FATAL|YLOG_ERRNO, "Failed to create comstack"); - return -1; - } - - if (0 == strlen(global_parameters.zproxy_override)){ - /* no Z39.50 proxy needed - direct connect */ - yaz_log(YLOG_DEBUG, "Connection create %s", cl->database->database->url); - - if (!(addr = cs_straddr(link, host->ipport))) - { - yaz_log(YLOG_WARN|YLOG_ERRNO, - "Lookup of IP address %s failed", host->ipport); - return -1; - } - - } else { - /* Z39.50 proxy connect */ - yaz_log(YLOG_DEBUG, "Connection create %s proxy %s", - cl->database->database->url, global_parameters.zproxy_override); - - if (!(addr = cs_straddr(link, global_parameters.zproxy_override))) - { - yaz_log(YLOG_WARN|YLOG_ERRNO, - "Lookup of IP address %s failed", - global_parameters.zproxy_override); - return -1; - } - } - - res = cs_connect(link, addr); - if (res < 0) - { - yaz_log(YLOG_WARN|YLOG_ERRNO, "cs_connect %s", cl->database->database->url); - return -1; - } - con->link = link; - con->state = Conn_Connecting; - con->iochan = iochan_create(cs_fileno(link), connection_handler, 0); - iochan_setdata(con->iochan, con); - pazpar2_add_channel(con->iochan); - - /* this fragment is bad DRY: from client_prep_connection */ - cl->state = Client_Connecting; - iochan_setflag(con->iochan, EVENT_OUTPUT); - return 0; -} - -void connect_resolver_host(struct host *host) -{ - struct connection *con = host->connections; - - while (con) - { - if (con->state == Conn_Resolving) - { - if (!host->ipport) /* unresolved */ - { - connection_destroy(con); - /* start all over .. at some point it will be NULL */ - con = host->connections; - } - else if (!con->client) - { - yaz_log(YLOG_WARN, "connect_unresolved_host : ophan client"); - connection_destroy(con); - /* start all over .. at some point it will be NULL */ - con = host->connections; - } - else - { - connection_connect(con); - con = con->next; - } - } - } -} - - -// Creates a new connection for client, associated with the host of -// client's database -static struct connection *connection_create(struct client *cl) -{ - struct connection *new; - struct host *host = cl->database->database->host; - - if ((new = connection_freelist)) - connection_freelist = new->next; - else - { - new = xmalloc(sizeof (struct connection)); - new->ibuf = 0; - new->ibufsize = 0; - } - new->host = host; - new->next = new->host->connections; - new->host->connections = new; - new->client = cl; - cl->connection = new; - new->link = 0; - new->state = Conn_Resolving; - if (host->ipport) - connection_connect(new); - return new; -} - -// Close connection and set state to error -static void client_fatal(struct client *cl) -{ - yaz_log(YLOG_WARN, "Fatal error from %s", cl->database->database->url); - connection_destroy(cl->connection); - cl->state = Client_Error; -} - -// Ensure that client has a connection associated -static int client_prep_connection(struct client *cl) -{ - struct connection *co; - struct session *se = cl->session; - struct host *host = cl->database->database->host; - - co = cl->connection; - - yaz_log(YLOG_DEBUG, "Client prep %s", cl->database->database->url); - - if (!co) - { - // See if someone else has an idle connection - // We should look at timestamps here to select the longest-idle connection - for (co = host->connections; co; co = co->next) - if (co->state == Conn_Open && (!co->client || co->client->session != se)) - break; - if (co) - { - connection_release(co); - cl->connection = co; - co->client = cl; - } - else - co = connection_create(cl); - } - if (co) - { - if (co->state == Conn_Connecting) - { - cl->state = Client_Connecting; - iochan_setflag(co->iochan, EVENT_OUTPUT); - } - else if (co->state == Conn_Open) - { - if (cl->state == Client_Error || cl->state == Client_Disconnected) - cl->state = Client_Idle; - iochan_setflag(co->iochan, EVENT_OUTPUT); - } - return 1; - } - else - return 0; -} - -// Initialize YAZ Map structures for MARC-based targets -static int prepare_yazmarc(struct session_database *sdb) -{ - char *s; - - if (!sdb->settings) - { - yaz_log(YLOG_WARN, "No settings for %s", sdb->database->url); - return -1; - } - if ((s = session_setting_oneval(sdb, PZ_NATIVESYNTAX)) && !strncmp(s, "iso2709", 7)) - { - char *encoding = "marc-8s", *e; - yaz_iconv_t cm; - - // See if a native encoding is specified - if ((e = strchr(s, ';'))) - encoding = e + 1; - - sdb->yaz_marc = yaz_marc_create(); - yaz_marc_subfield_str(sdb->yaz_marc, "\t"); - - cm = yaz_iconv_open("utf-8", encoding); - if (!cm) - { - yaz_log(YLOG_FATAL, - "Unable to map from %s to UTF-8 for target %s", - encoding, sdb->database->url); - return -1; - } - yaz_marc_iconv(sdb->yaz_marc, cm); - } - return 0; -} - -// Prepare XSLT stylesheets for record normalization -// Structures are allocated on the session_wide nmem to avoid having -// to recompute this for every search. This would lead -// to leaking if a single session was to repeatedly change the PZ_XSLT -// setting. However, this is not a realistic use scenario. -static int prepare_map(struct session *se, struct session_database *sdb) -{ - char *s; - - if (!sdb->settings) - { - yaz_log(YLOG_WARN, "No settings on %s", sdb->database->url); - return -1; - } - if ((s = session_setting_oneval(sdb, PZ_XSLT))) - { - char **stylesheets; - struct database_retrievalmap **m = &sdb->map; - int num, i; + char **stylesheets; + struct database_retrievalmap **m = &sdb->map; + int num, i; nmem_strsplit(se->session_nmem, ",", s, &stylesheets, &num); for (i = 0; i < num; i++) @@ -1368,144 +402,38 @@ static int prepare_map(struct session *se, struct session_database *sdb) yaz_log(YLOG_FATAL, "Unable to load stylesheet: %s", stylesheets[i]); return -1; - } - m = &(*m)->next; - } - } - if (!sdb->map) - yaz_log(YLOG_WARN, "No Normalization stylesheet for target %s", - sdb->database->url); - return 0; -} - -// This analyzes settings and recomputes any supporting data structures -// if necessary. -static int prepare_session_database(struct session *se, struct session_database *sdb) -{ - if (!sdb->settings) - { - yaz_log(YLOG_WARN, "No settings associated with %s", sdb->database->url); - return -1; - } - if (sdb->settings[PZ_NATIVESYNTAX] && !sdb->yaz_marc) - { - if (prepare_yazmarc(sdb) < 0) - return -1; - } - if (sdb->settings[PZ_XSLT] && !sdb->map) - { - if (prepare_map(se, sdb) < 0) - return -1; - } - return 0; -} - -// Initialize CCL map for a target -static CCL_bibset prepare_cclmap(struct client *cl) -{ - struct session_database *sdb = cl->database; - struct setting *s; - CCL_bibset res; - - if (!sdb->settings) - return 0; - res = ccl_qual_mk(); - for (s = sdb->settings[PZ_CCLMAP]; s; s = s->next) - { - char *p = strchr(s->name + 3, ':'); - if (!p) - { - yaz_log(YLOG_WARN, "Malformed cclmap name: %s", s->name); - ccl_qual_rm(&res); - return 0; + } + m = &(*m)->next; } - p++; - ccl_qual_fitem(res, s->value, p); } - return res; + if (!sdb->map) + yaz_log(YLOG_WARN, "No Normalization stylesheet for target %s", + sdb->database->url); + return 0; } -// Parse the query given the settings specific to this client -static int client_parse_query(struct client *cl, const char *query) +// This analyzes settings and recomputes any supporting data structures +// if necessary. +static int prepare_session_database(struct session *se, struct session_database *sdb) { - struct session *se = cl->session; - struct ccl_rpn_node *cn; - int cerror, cpos; - CCL_bibset ccl_map = prepare_cclmap(cl); - - if (!ccl_map) - return -1; - cn = ccl_find_str(ccl_map, query, &cerror, &cpos); - ccl_qual_rm(&ccl_map); - if (!cn) + if (!sdb->settings) { - cl->state = Client_Error; - yaz_log(YLOG_WARN, "Failed to parse query for %s", - cl->database->database->url); + yaz_log(YLOG_WARN, "No settings associated with %s", sdb->database->url); return -1; } - wrbuf_rewind(se->wrbuf); - ccl_pquery(se->wrbuf, cn); - wrbuf_putc(se->wrbuf, '\0'); - if (cl->pquery) - xfree(cl->pquery); - cl->pquery = xstrdup(wrbuf_buf(se->wrbuf)); - - if (!se->relevance) + if (sdb->settings[PZ_NATIVESYNTAX] && !sdb->yaz_marc) { - // Initialize relevance structure with query terms - char *p[512]; - extract_terms(se->nmem, cn, p); - se->relevance = relevance_create(se->nmem, (const char **) p, - se->expected_maxrecs); + if (prepare_yazmarc(sdb) < 0) + return -1; } - - ccl_rpn_delete(cn); - return 0; -} - -static struct client *client_create(void) -{ - struct client *r; - if (client_freelist) + if (sdb->settings[PZ_XSLT] && !sdb->map) { - r = client_freelist; - client_freelist = client_freelist->next; + if (prepare_map(se, sdb) < 0) + return -1; } - else - r = xmalloc(sizeof(struct client)); - r->pquery = 0; - r->database = 0; - r->connection = 0; - r->session = 0; - r->hits = 0; - r->records = 0; - r->setno = 0; - r->requestid = -1; - r->diagnostic = 0; - r->state = Client_Disconnected; - r->next = 0; - return r; + return 0; } -void client_destroy(struct client *c) -{ - struct session *se = c->session; - if (c == se->clients) - se->clients = c->next; - else - { - struct client *cc; - for (cc = se->clients; cc && cc->next != c; cc = cc->next) - ; - if (cc) - cc->next = c->next; - } - if (c->connection) - connection_release(c->connection); - c->next = client_freelist; - client_freelist = c; -} void session_set_watch(struct session *s, int what, session_watchfun fun, void *data) { @@ -1527,10 +455,8 @@ static void select_targets_callback(void *context, struct session_database *db) { struct session *se = (struct session*) context; struct client *cl = client_create(); - cl->database = db; - cl->session = se; - cl->next = se->clients; - se->clients = cl; + client_set_database(cl, db); + client_set_session(cl, se); } // Associates a set of clients with a session; @@ -1548,11 +474,8 @@ int session_active_clients(struct session *s) struct client *c; int res = 0; - for (c = s->clients; c; c = c->next) - if (c->connection && (c->state == Client_Connecting || - c->state == Client_Initializing || - c->state == Client_Searching || - c->state == Client_Presenting)) + for (c = s->clients; c; c = client_next_in_session(c)) + if (client_is_active(c)) res++; return res; @@ -1624,15 +547,15 @@ char *search(struct session *se, char *query, char *filter) se->relevance = 0; - for (cl = se->clients; cl; cl = cl->next) + for (cl = se->clients; cl; cl = client_next_in_session(cl)) { - if (prepare_session_database(se, cl->database) < 0) + if (prepare_session_database(se, client_get_database(cl)) < 0) return "CONFIG_ERROR"; if (client_parse_query(cl, query) < 0) // Query must parse for all targets return "QUERY"; } - for (cl = se->clients; cl; cl = cl->next) + for (cl = se->clients; cl; cl = client_next_in_session(cl)) { client_prep_connection(cl); } @@ -1792,17 +715,17 @@ struct hitsbytarget *hitsbytarget(struct session *se, int *count) struct client *cl; *count = 0; - for (cl = se->clients; cl; cl = cl->next) + for (cl = se->clients; cl; cl = client_next_in_session(cl)) { - char *name = session_setting_oneval(cl->database, PZ_NAME); + char *name = session_setting_oneval(client_get_database(cl), PZ_NAME); - res[*count].id = cl->database->database->url; + res[*count].id = client_get_database(cl)->database->url; res[*count].name = *name ? name : "Unknown"; - res[*count].hits = cl->hits; - res[*count].records = cl->records; - res[*count].diagnostic = cl->diagnostic; - res[*count].state = client_states[cl->state]; - res[*count].connected = cl->connection ? 1 : 0; + res[*count].hits = client_get_hits(cl); + res[*count].records = client_get_num_records(cl); + res[*count].diagnostic = client_get_diagnostic(cl); + res[*count].state = client_get_state_str(cl); + res[*count].connected = client_get_connection(cl) ? 1 : 0; (*count)++; } @@ -1899,11 +822,11 @@ void statistics(struct session *se, struct statistics *stat) int count = 0; memset(stat, 0, sizeof(*stat)); - for (cl = se->clients; cl; cl = cl->next) + for (cl = se->clients; cl; cl = client_next_in_session(cl)) { - if (!cl->connection) + if (!client_get_connection(cl)) stat->num_no_connection++; - switch (cl->state) + switch (client_get_state(cl)) { case Client_Connecting: stat->num_connecting++; break; case Client_Initializing: stat->num_initializing++; break; @@ -2011,6 +934,235 @@ void pazpar2_event_loop() event_loop(&channel_list); } +struct record *ingest_record(struct client *cl, Z_External *rec, + int record_no) +{ + xmlDoc *xdoc = normalize_record(client_get_database(cl), rec); + xmlNode *root, *n; + struct record *res; + struct record_cluster *cluster; + struct session *se = client_get_session(cl); + xmlChar *mergekey, *mergekey_norm; + xmlChar *type = 0; + xmlChar *value = 0; + struct conf_service *service = global_parameters.server->service; + + if (!xdoc) + return 0; + + root = xmlDocGetRootElement(xdoc); + if (!(mergekey = xmlGetProp(root, (xmlChar *) "mergekey"))) + { + yaz_log(YLOG_WARN, "No mergekey found in record"); + xmlFreeDoc(xdoc); + return 0; + } + + res = nmem_malloc(se->nmem, sizeof(struct record)); + res->next = 0; + res->client = cl; + res->metadata = nmem_malloc(se->nmem, + sizeof(struct record_metadata*) * service->num_metadata); + memset(res->metadata, 0, sizeof(struct record_metadata*) * service->num_metadata); + + mergekey_norm = (xmlChar *) nmem_strdup(se->nmem, (char*) mergekey); + xmlFree(mergekey); + normalize_mergekey((char *) mergekey_norm, 0); + + cluster = reclist_insert(se->reclist, + global_parameters.server->service, + res, (char *) mergekey_norm, + &se->total_merged); + if (global_parameters.dump_records) + yaz_log(YLOG_LOG, "Cluster id %d from %s (#%d)", cluster->recid, + client_get_database(cl)->database->url, record_no); + if (!cluster) + { + /* no room for record */ + xmlFreeDoc(xdoc); + return 0; + } + relevance_newrec(se->relevance, cluster); + + for (n = root->children; n; n = n->next) + { + if (type) + xmlFree(type); + if (value) + xmlFree(value); + type = value = 0; + + if (n->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) n->name, "metadata")) + { + struct conf_metadata *md = 0; + struct conf_sortkey *sk = 0; + struct record_metadata **wheretoput, *newm; + int imeta; + int first, last; + + type = xmlGetProp(n, (xmlChar *) "type"); + value = xmlNodeListGetString(xdoc, n->children, 0); + + if (!type || !value) + continue; + + // First, find out what field we're looking at + for (imeta = 0; imeta < service->num_metadata; imeta++) + if (!strcmp((const char *) type, service->metadata[imeta].name)) + { + md = &service->metadata[imeta]; + if (md->sortkey_offset >= 0) + sk = &service->sortkeys[md->sortkey_offset]; + break; + } + if (!md) + { + yaz_log(YLOG_WARN, "Ignoring unknown metadata element: %s", type); + continue; + } + + // Find out where we are putting it + if (md->merge == Metadata_merge_no) + wheretoput = &res->metadata[imeta]; + else + wheretoput = &cluster->metadata[imeta]; + + // Put it there + newm = nmem_malloc(se->nmem, sizeof(struct record_metadata)); + newm->next = 0; + if (md->type == Metadata_type_generic) + { + char *p, *pe; + for (p = (char *) value; *p && isspace(*p); p++) + ; + for (pe = p + strlen(p) - 1; + pe > p && strchr(" ,/.:([", *pe); pe--) + *pe = '\0'; + newm->data.text = nmem_strdup(se->nmem, p); + + } + else if (md->type == Metadata_type_year) + { + if (extract_years((char *) value, &first, &last) < 0) + continue; + } + else + { + yaz_log(YLOG_WARN, "Unknown type in metadata element %s", type); + continue; + } + if (md->type == Metadata_type_year && md->merge != Metadata_merge_range) + { + yaz_log(YLOG_WARN, "Only range merging supported for years"); + continue; + } + if (md->merge == Metadata_merge_unique) + { + struct record_metadata *mnode; + for (mnode = *wheretoput; mnode; mnode = mnode->next) + if (!strcmp((const char *) mnode->data.text, newm->data.text)) + break; + if (!mnode) + { + newm->next = *wheretoput; + *wheretoput = newm; + } + } + else if (md->merge == Metadata_merge_longest) + { + if (!*wheretoput || + strlen(newm->data.text) > strlen((*wheretoput)->data.text)) + { + *wheretoput = newm; + if (sk) + { + char *s = nmem_strdup(se->nmem, newm->data.text); + if (!cluster->sortkeys[md->sortkey_offset]) + cluster->sortkeys[md->sortkey_offset] = + nmem_malloc(se->nmem, sizeof(union data_types)); + normalize_mergekey(s, + (sk->type == Metadata_sortkey_skiparticle)); + cluster->sortkeys[md->sortkey_offset]->text = s; + } + } + } + else if (md->merge == Metadata_merge_all || md->merge == Metadata_merge_no) + { + newm->next = *wheretoput; + *wheretoput = newm; + } + else if (md->merge == Metadata_merge_range) + { + assert(md->type == Metadata_type_year); + if (!*wheretoput) + { + *wheretoput = newm; + (*wheretoput)->data.number.min = first; + (*wheretoput)->data.number.max = last; + if (sk) + cluster->sortkeys[md->sortkey_offset] = &newm->data; + } + else + { + if (first < (*wheretoput)->data.number.min) + (*wheretoput)->data.number.min = first; + if (last > (*wheretoput)->data.number.max) + (*wheretoput)->data.number.max = last; + } +#ifdef GAGA + if (sk) + { + union data_types *sdata = cluster->sortkeys[md->sortkey_offset]; + yaz_log(YLOG_LOG, "SK range: %d-%d", sdata->number.min, sdata->number.max); + } +#endif + } + else + yaz_log(YLOG_WARN, "Don't know how to merge on element name %s", md->name); + + if (md->rank) + relevance_countwords(se->relevance, cluster, + (char *) value, md->rank); + if (md->termlist) + { + if (md->type == Metadata_type_year) + { + char year[64]; + sprintf(year, "%d", last); + add_facet(se, (char *) type, year); + if (first != last) + { + sprintf(year, "%d", first); + add_facet(se, (char *) type, year); + } + } + else + add_facet(se, (char *) type, (char *) value); + } + xmlFree(type); + xmlFree(value); + type = value = 0; + } + else + yaz_log(YLOG_WARN, "Unexpected element %s in internal record", n->name); + } + if (type) + xmlFree(type); + if (value) + xmlFree(value); + + xmlFreeDoc(xdoc); + + relevance_donerecord(se->relevance, cluster); + se->total_records++; + + return res; +} + + + /* * Local variables: * c-basic-offset: 4 diff --git a/src/parameters.h b/src/parameters.h new file mode 100644 index 0000000..ef4d54f --- /dev/null +++ b/src/parameters.h @@ -0,0 +1,56 @@ +/* $Id: parameters.h,v 1.1 2007-04-23 21:05:23 adam Exp $ + Copyright (c) 2006-2007, Index Data. + +This file is part of Pazpar2. + +Pazpar2 is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Pazpar2; see the file LICENSE. If not, write to the +Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA +02111-1307, USA. + */ + +#ifndef PARAMETERS_H +#define PARAMETERS_H + +#include + +/** \brief global parameters */ +struct parameters { + char proxy_override[128]; + char listener_override[128]; + char zproxy_override[128]; + char settings_path_override[128]; + struct conf_server *server; + int dump_records; + int timeout; /* operations timeout, in seconds */ + char implementationId[128]; + char implementationName[128]; + char implementationVersion[128]; + int target_timeout; // seconds + int session_timeout; + int toget; + int chunk; + ODR odr_out; + ODR odr_in; +}; +extern struct parameters global_parameters; + +#endif + +/* + * Local variables: + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ diff --git a/src/pazpar2.c b/src/pazpar2.c index 6778388..721f501 100644 --- a/src/pazpar2.c +++ b/src/pazpar2.c @@ -1,4 +1,4 @@ -/* $Id: pazpar2.c,v 1.82 2007-04-22 15:07:10 adam Exp $ +/* $Id: pazpar2.c,v 1.83 2007-04-23 21:05:23 adam Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -25,6 +25,7 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include +#include #include "pazpar2.h" #include "database.h" @@ -101,6 +102,7 @@ int main(int argc, char **argv) return 0; } + /* * Local variables: * c-basic-offset: 4 diff --git a/src/pazpar2.h b/src/pazpar2.h index 9d42c37..d5c05ea 100644 --- a/src/pazpar2.h +++ b/src/pazpar2.h @@ -1,4 +1,4 @@ -/* $Id: pazpar2.h,v 1.33 2007-04-22 16:41:42 adam Exp $ +/* $Id: pazpar2.h,v 1.34 2007-04-23 21:05:23 adam Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -38,20 +38,13 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include "reclists.h" #include "eventl.h" #include "config.h" +#include "parameters.h" struct record; struct client; struct connection; -// Represents a host (irrespective of databases) -struct host { - char *hostport; - char *ipport; - struct connection *connections; // All connections to this - struct host *next; -}; - // Represents a (virtual) database on a host struct database { struct host *host; @@ -74,50 +67,6 @@ struct database_criterion { struct database_criterion *next; }; -// Represents a physical, reusable connection to a remote Z39.50 host -struct connection { - IOCHAN iochan; - COMSTACK link; - struct host *host; - struct client *client; - char *ibuf; - int ibufsize; - enum { - Conn_Resolving, - Conn_Connecting, - Conn_Open, - Conn_Waiting, - } state; - struct connection *next; -}; - -// Represents client state for a connection to one search target -struct client { - struct session_database *database; - struct connection *connection; - struct session *session; - char *pquery; // Current search - int hits; - int records; - int setno; - int requestid; // ID of current outstanding request - int diagnostic; - enum client_state - { - Client_Connecting, - Client_Connected, - Client_Idle, - Client_Initializing, - Client_Searching, - Client_Presenting, - Client_Error, - Client_Failed, - Client_Disconnected, - Client_Stopped - } state; - struct client *next; -}; - // Normalization filter. Turns incoming record into internal representation // Simple sequence of stylesheets run in series. struct database_retrievalmap { @@ -191,29 +140,10 @@ struct hitsbytarget { int hits; int diagnostic; int records; - char* state; + const char *state; int connected; }; -struct parameters { - char proxy_override[128]; - char listener_override[128]; - char zproxy_override[128]; - char settings_path_override[128]; - struct conf_server *server; - int dump_records; - int timeout; /* operations timeout, in seconds */ - char implementationId[128]; - char implementationName[128]; - char implementationVersion[128]; - int target_timeout; // seconds - int session_timeout; - int toget; - int chunk; - ODR odr_out; - ODR odr_in; -}; - struct hitsbytarget *hitsbytarget(struct session *s, int *count); int select_targets(struct session *se, struct database_criterion *crit); struct session *new_session(NMEM nmem); @@ -234,12 +164,19 @@ void start_http_listener(void); void start_proxy(void); void start_zproxy(void); -extern struct parameters global_parameters; void pazpar2_add_channel(IOCHAN c); void pazpar2_event_loop(void); int host_getaddrinfo(struct host *host); -void connect_resolver_host(struct host *host); + +xmlDoc *normalize_record(struct session_database *sdb, Z_External *rec); +void connection_destroy(struct connection *co); + +struct record *ingest_record(struct client *cl, Z_External *rec, + int record_no); +void session_alert_watch(struct session *s, int what); +void connection_release(struct connection *co); +void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num); #endif diff --git a/src/relevance.h b/src/relevance.h index 388c6a3..231fbc5 100644 --- a/src/relevance.h +++ b/src/relevance.h @@ -1,4 +1,4 @@ -/* $Id: relevance.h,v 1.4 2007-04-10 08:48:56 adam Exp $ +/* $Id: relevance.h,v 1.5 2007-04-23 21:05:23 adam Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -24,10 +24,9 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include -#include "pazpar2.h" -#include "reclists.h" - struct relevance; +struct record_cluster; +struct reclist; struct relevance *relevance_create(NMEM nmem, const char **terms, int numrecs); void relevance_newrec(struct relevance *r, struct record_cluster *cluster); -- 1.7.10.4