X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Flogic.c;h=8abdcaef7126542b4255fd41c993d9d50841fb49;hb=cc178327044177f177d0474b757f53bcd86b4d42;hp=bce0ad11d610ab81edd03e21196d9b125dab170b;hpb=f030fa5726d942a82ffebb2ee22e35338ccfed78;p=pazpar2-moved-to-github.git diff --git a/src/logic.c b/src/logic.c index bce0ad1..8abdcae 100644 --- a/src/logic.c +++ b/src/logic.c @@ -1,7 +1,5 @@ -/* $Id: logic.c,v 1.18 2007-04-22 16:41:42 adam Exp $ - Copyright (c) 2006-2007, Index Data. - -This file is part of Pazpar2. +/* This file is part of Pazpar2. + Copyright (C) 2006-2008 Index Data Pazpar2 is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -14,21 +12,30 @@ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with Pazpar2; see the file LICENSE. If not, write to the -Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA -02111-1307, USA. - */ +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -// This file contains the primary business logic. Several parts of it should -// Eventually be factored into separate modules. +*/ + +/** \file logic.c + \brief high-level logic; mostly user sessions and settings +*/ #include #include #include +#if HAVE_SYS_TIME_H #include +#endif +#if HAVE_UNISTD_H #include +#endif +#if HAVE_SYS_SOCKET_H #include +#endif +#if HAVE_NETDB_H #include +#endif #include #include #include @@ -44,12 +51,11 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include #include #include -#if YAZ_VERSIONL >= 0x020163 #include -#endif +#include #if HAVE_CONFIG_H -#include "cconfig.h" +#include #endif #define USE_TIMING 0 @@ -57,7 +63,9 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include #endif +#if HAVE_NETINET_IN_H #include +#endif #include "pazpar2.h" #include "eventl.h" @@ -65,150 +73,39 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include "termlists.h" #include "reclists.h" #include "relevance.h" -#include "config.h" #include "database.h" +#include "client.h" #include "settings.h" +#include "normalize7bit.h" -#define MAX_CHUNK 15 +#define TERMLIST_HIGH_SCORE 25 -static void client_fatal(struct client *cl); -static void connection_destroy(struct connection *co); -static int client_prep_connection(struct client *cl); -static void ingest_records(struct client *cl, Z_Records *r); -void session_alert_watch(struct session *s, int what); - -static struct connection *connection_freelist = 0; -static struct client *client_freelist = 0; - -static char *client_states[] = { - "Client_Connecting", - "Client_Connected", - "Client_Idle", - "Client_Initializing", - "Client_Searching", - "Client_Presenting", - "Client_Error", - "Client_Failed", - "Client_Disconnected", - "Client_Stopped" -}; +#define MAX_CHUNK 15 // Note: Some things in this structure will eventually move to configuration struct parameters global_parameters = { "", "", - "", - "", + "", 0, - 0, - 30, + 0, // dump_records + 0, // debug_mode + 30, // operations timeout "81", "Index Data PazPar2", VERSION, - 600, // 10 minutes - 60, + 60, // session timeout 100, MAX_CHUNK, 0, - 0 + 0, + 180, // Z39.50 session timeout + 15 // Connect timeout }; -static int send_apdu(struct client *c, Z_APDU *a) -{ - struct connection *co = c->connection; - char *buf; - int len, r; - - if (!z_APDU(global_parameters.odr_out, &a, 0, 0)) - { - odr_perror(global_parameters.odr_out, "Encoding APDU"); - abort(); - } - buf = odr_getbuf(global_parameters.odr_out, &len, 0); - r = cs_put(co->link, buf, len); - if (r < 0) - { - yaz_log(YLOG_WARN, "cs_put: %s", cs_errmsg(cs_errno(co->link))); - return -1; - } - else if (r == 1) - { - fprintf(stderr, "cs_put incomplete (ParaZ does not handle that)\n"); - exit(1); - } - odr_reset(global_parameters.odr_out); /* release the APDU structure */ - co->state = Conn_Waiting; - return 0; -} - -// Set authentication token in init if one is set for the client -// TODO: Extend this to handle other schemes than open (should be simple) -static void init_authentication(struct client *cl, Z_InitRequest *req) -{ - struct session_database *sdb = cl->database; - char *auth = session_setting_oneval(sdb, PZ_AUTHENTICATION); - - if (auth) - { - Z_IdAuthentication *idAuth = odr_malloc(global_parameters.odr_out, - sizeof(*idAuth)); - idAuth->which = Z_IdAuthentication_open; - idAuth->u.open = auth; - req->idAuthentication = idAuth; - } -} - -static void send_init(IOCHAN i) -{ - - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_initRequest); - - a->u.initRequest->implementationId = global_parameters.implementationId; - a->u.initRequest->implementationName = global_parameters.implementationName; - a->u.initRequest->implementationVersion = - global_parameters.implementationVersion; - ODR_MASK_SET(a->u.initRequest->options, Z_Options_search); - ODR_MASK_SET(a->u.initRequest->options, Z_Options_present); - ODR_MASK_SET(a->u.initRequest->options, Z_Options_namedResultSets); - - ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_1); - ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_2); - ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_3); - - init_authentication(cl, a->u.initRequest); - - /* add virtual host if tunneling through Z39.50 proxy */ - - if (0 < strlen(global_parameters.zproxy_override) - && 0 < strlen(cl->database->database->url)) - { -#if YAZ_VERSIONL >= 0x020163 - yaz_oi_set_string_oid(&a->u.initRequest->otherInfo, - global_parameters.odr_out, - yaz_oid_userinfo_proxy, - 1, cl->database->database->url); -#else - yaz_oi_set_string_oidval(&a->u.initRequest->otherInfo, - global_parameters.odr_out, VAL_PROXY, - 1, cl->database->database->url); -#endif - } - - if (send_apdu(cl, a) >= 0) - { - iochan_setflags(i, EVENT_INPUT); - cl->state = Client_Initializing; - } - else - cl->state = Client_Error; - odr_reset(global_parameters.odr_out); -} - // Recursively traverse query structure to extract terms. -static void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) +void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) { char **words; int numwords; @@ -216,284 +113,24 @@ static void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int * switch (n->kind) { - case CCL_RPN_AND: - case CCL_RPN_OR: - case CCL_RPN_NOT: - case CCL_RPN_PROX: - pull_terms(nmem, n->u.p[0], termlist, num); - pull_terms(nmem, n->u.p[1], termlist, num); - break; - case CCL_RPN_TERM: - nmem_strsplit(nmem, " ", n->u.t.term, &words, &numwords); - for (i = 0; i < numwords; i++) - termlist[(*num)++] = words[i]; - break; - default: // NOOP - break; - } -} - -// Extract terms from query into null-terminated termlist -static void extract_terms(NMEM nmem, struct ccl_rpn_node *query, char **termlist) -{ - int num = 0; - - pull_terms(nmem, query, termlist, &num); - termlist[num] = 0; -} - -static void send_search(IOCHAN i) -{ - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - struct session *se = cl->session; - struct session_database *sdb = cl->database; - Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_searchRequest); - int ndb; - char **databaselist; - Z_Query *zquery; - int ssub = 0, lslb = 100000, mspn = 10; - char *recsyn = 0; - char *piggyback = 0; - char *queryenc = 0; - yaz_iconv_t iconv = 0; - - yaz_log(YLOG_DEBUG, "Sending search to %s", cl->database->database->url); - - // constructing RPN query - a->u.searchRequest->query = zquery = odr_malloc(global_parameters.odr_out, - sizeof(Z_Query)); - zquery->which = Z_Query_type_1; - zquery->u.type_1 = p_query_rpn(global_parameters.odr_out, cl->pquery); - - // converting to target encoding - if ((queryenc = session_setting_oneval(sdb, PZ_QUERYENCODING))){ - iconv = yaz_iconv_open(queryenc, "UTF-8"); - if (iconv){ - yaz_query_charset_convert_rpnquery(zquery->u.type_1, - global_parameters.odr_out, - iconv); - yaz_iconv_close(iconv); - } else - yaz_log(YLOG_WARN, "Query encoding failed %s %s", - cl->database->database->url, queryenc); - } - - for (ndb = 0; sdb->database->databases[ndb]; ndb++) - ; - databaselist = odr_malloc(global_parameters.odr_out, sizeof(char*) * ndb); - for (ndb = 0; sdb->database->databases[ndb]; ndb++) - databaselist[ndb] = sdb->database->databases[ndb]; - - if (!(piggyback = session_setting_oneval(sdb, PZ_PIGGYBACK)) || *piggyback == '1') - { - if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX))) - { -#if YAZ_VERSIONL >= 0x020163 - a->u.searchRequest->preferredRecordSyntax = - yaz_string_to_oid_odr(yaz_oid_std(), - CLASS_RECSYN, recsyn, - global_parameters.odr_out); -#else - a->u.searchRequest->preferredRecordSyntax = - yaz_str_to_z3950oid(global_parameters.odr_out, - CLASS_RECSYN, recsyn); -#endif - } - a->u.searchRequest->smallSetUpperBound = &ssub; - a->u.searchRequest->largeSetLowerBound = &lslb; - a->u.searchRequest->mediumSetPresentNumber = &mspn; - } - a->u.searchRequest->resultSetName = "Default"; - a->u.searchRequest->databaseNames = databaselist; - a->u.searchRequest->num_databaseNames = ndb; - - - { //scope for sending and logging queries - WRBUF wbquery = wrbuf_alloc(); - yaz_query_to_wrbuf(wbquery, zquery); - - - if (send_apdu(cl, a) >= 0) - { - iochan_setflags(i, EVENT_INPUT); - cl->state = Client_Searching; - cl->requestid = se->requestid; - yaz_log(YLOG_LOG, "SearchRequest %s %s %s", - cl->database->database->url, - queryenc ? queryenc : "UTF-8", - wrbuf_cstr(wbquery)); - } - else { - cl->state = Client_Error; - yaz_log(YLOG_WARN, "Failed SearchRequest %s %s %s", - cl->database->database->url, - queryenc ? queryenc : "UTF-8", - wrbuf_cstr(wbquery)); - } - - wrbuf_destroy(wbquery); - } - - odr_reset(global_parameters.odr_out); -} - -static void send_present(IOCHAN i) -{ - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - struct session_database *sdb = cl->database; - Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_presentRequest); - int toget; - int start = cl->records + 1; - char *recsyn; - - toget = global_parameters.chunk; - if (toget > global_parameters.toget - cl->records) - toget = global_parameters.toget - cl->records; - if (toget > cl->hits - cl->records) - toget = cl->hits - cl->records; - - yaz_log(YLOG_DEBUG, "Trying to present %d records\n", toget); - - a->u.presentRequest->resultSetStartPoint = &start; - a->u.presentRequest->numberOfRecordsRequested = &toget; - - a->u.presentRequest->resultSetId = "Default"; - - if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX))) - { -#if YAZ_VERSIONL >= 0x020163 - a->u.presentRequest->preferredRecordSyntax = - yaz_string_to_oid_odr(yaz_oid_std(), - CLASS_RECSYN, recsyn, - global_parameters.odr_out); -#else - a->u.presentRequest->preferredRecordSyntax = - yaz_str_to_z3950oid(global_parameters.odr_out, - CLASS_RECSYN, recsyn); -#endif - } - - if (send_apdu(cl, a) >= 0) - { - iochan_setflags(i, EVENT_INPUT); - cl->state = Client_Presenting; - } - else - cl->state = Client_Error; - odr_reset(global_parameters.odr_out); -} - -static void do_initResponse(IOCHAN i, Z_APDU *a) -{ - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - Z_InitResponse *r = a->u.initResponse; - - yaz_log(YLOG_DEBUG, "Init response %s", cl->database->database->url); - - if (*r->result) - { - cl->state = Client_Idle; - } - else - cl->state = Client_Failed; // FIXME need to do something to the connection -} - -static void do_searchResponse(IOCHAN i, Z_APDU *a) -{ - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - struct session *se = cl->session; - Z_SearchResponse *r = a->u.searchResponse; - - yaz_log(YLOG_DEBUG, "Search response %s (status=%d)", - cl->database->database->url, *r->searchStatus); - - if (*r->searchStatus) - { - cl->hits = *r->resultCount; - se->total_hits += cl->hits; - if (r->presentStatus && !*r->presentStatus && r->records) - { - yaz_log(YLOG_DEBUG, "Records in search response %s", - cl->database->database->url); - ingest_records(cl, r->records); - } - cl->state = Client_Idle; - } - else - { /*"FAILED"*/ - cl->hits = 0; - cl->state = Client_Error; - if (r->records) { - Z_Records *recs = r->records; - if (recs->which == Z_Records_NSD) - { - yaz_log(YLOG_WARN, - "Search response: Non-surrogate diagnostic %s", - cl->database->database->url); - cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition; - cl->state = Client_Error; - } - } + case CCL_RPN_AND: + case CCL_RPN_OR: + case CCL_RPN_NOT: + case CCL_RPN_PROX: + pull_terms(nmem, n->u.p[0], termlist, num); + pull_terms(nmem, n->u.p[1], termlist, num); + break; + case CCL_RPN_TERM: + nmem_strsplit(nmem, " ", n->u.t.term, &words, &numwords); + for (i = 0; i < numwords; i++) + termlist[(*num)++] = words[i]; + break; + default: // NOOP + break; } } -static void do_closeResponse(IOCHAN i, Z_APDU *a) -{ - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - /* Z_Close *r = a->u.close; */ - - yaz_log(YLOG_WARN, "Close response %s", cl->database->database->url); - cl->state = Client_Failed; - connection_destroy(co); -} - - -char *normalize_mergekey(char *buf, int skiparticle) -{ - char *p = buf, *pout = buf; - - if (skiparticle) - { - char firstword[64]; - char articles[] = "the den der die des an a "; // must end in space - - while (*p && !isalnum(*p)) - p++; - pout = firstword; - while (*p && *p != ' ' && pout - firstword < 62) - *(pout++) = tolower(*(p++)); - *(pout++) = ' '; - *(pout++) = '\0'; - if (!strstr(articles, firstword)) - p = buf; - pout = buf; - } - - while (*p) - { - while (*p && !isalnum(*p)) - p++; - while (isalnum(*p)) - *(pout++) = tolower(*(p++)); - if (*p) - *(pout++) = ' '; - while (*p && !isalnum(*p)) - p++; - } - if (buf != pout) - do { - *(pout--) = '\0'; - } - while (pout > buf && *pout == ' '); - - return buf; -} static void add_facet(struct session *s, const char *type, const char *value) { @@ -509,810 +146,280 @@ static void add_facet(struct session *s, const char *type, const char *value) if (i == SESSION_MAX_TERMLISTS) { yaz_log(YLOG_FATAL, "Too many termlists"); - exit(1); + return; } + s->termlists[i].name = nmem_strdup(s->nmem, type); - s->termlists[i].termlist = termlist_create(s->nmem, s->expected_maxrecs, 15); + s->termlists[i].termlist + = termlist_create(s->nmem, s->expected_maxrecs, + TERMLIST_HIGH_SCORE); s->num_termlists = i + 1; } termlist_insert(s->termlists[i].termlist, value); } -static xmlDoc *normalize_record(struct client *cl, Z_External *rec) +xmlDoc *record_to_xml(struct session_database *sdb, Z_External *rec) { - struct database_retrievalmap *m; - struct session_database *sdb = cl->database; struct database *db = sdb->database; - xmlNode *res; - xmlDoc *rdoc; + xmlDoc *rdoc = 0; + const Odr_oid *oid = rec->direct_reference; - // First normalize to XML - if (sdb->yaz_marc) + /* convert response record to XML somehow */ + if (rec->which == Z_External_octet && oid + && !oid_oidcmp(oid, yaz_oid_recsyn_xml)) { - char *buf; - int len; - if (rec->which != Z_External_octet) + /* xml already */ + rdoc = xmlParseMemory((char*) rec->u.octet_aligned->buf, + rec->u.octet_aligned->len); + if (!rdoc) { - yaz_log(YLOG_WARN, "Unexpected external branch, probably BER %s", + yaz_log(YLOG_FATAL, "Non-wellformed XML received from %s", db->url); return 0; } - buf = (char*) rec->u.octet_aligned->buf; - len = rec->u.octet_aligned->len; - if (yaz_marc_read_iso2709(sdb->yaz_marc, buf, len) < 0) - { - yaz_log(YLOG_WARN, "Failed to decode MARC %s", db->url); - return 0; - } - - yaz_marc_write_using_libxml2(sdb->yaz_marc, 1); - if (yaz_marc_write_xml(sdb->yaz_marc, &res, - "http://www.loc.gov/MARC21/slim", 0, 0) < 0) - { - yaz_log(YLOG_WARN, "Failed to encode as XML %s", - db->url); - return 0; - } - rdoc = xmlNewDoc((xmlChar *) "1.0"); - xmlDocSetRootElement(rdoc, res); - } - else + else if (rec->which == Z_External_OPAC) { - yaz_log(YLOG_FATAL, - "Unknown native_syntax in normalize_record from %s", - db->url); - exit(1); - } - - if (global_parameters.dump_records){ - fprintf(stderr, - "Input Record (normalized) from %s\n----------------\n", - db->url); -#if LIBXML_VERSION >= 20600 - xmlDocFormatDump(stderr, rdoc, 1); -#else - xmlDocDump(stderr, rdoc); -#endif - } - - for (m = sdb->map; m; m = m->next){ - xmlDoc *new = 0; - -#if 1 - { - xmlNodePtr root = 0; - new = xsltApplyStylesheet(m->stylesheet, rdoc, 0); - root= xmlDocGetRootElement(new); - if (!new || !root || !(root->children)) + if (!sdb->yaz_marc) { - yaz_log(YLOG_WARN, "XSLT transformation failed from %s", - cl->database->database->url); - xmlFreeDoc(new); - xmlFreeDoc(rdoc); + yaz_log(YLOG_WARN, "MARC decoding not configured"); return 0; } - } -#endif - -#if 0 - // do it another way to detect transformation errors right now - // but does not seem to work either! - { - xsltTransformContextPtr ctxt; - ctxt = xsltNewTransformContext(m->stylesheet, rdoc); - new = xsltApplyStylesheetUser(m->stylesheet, rdoc, 0, 0, 0, ctxt); - if ((ctxt->state == XSLT_STATE_ERROR) || - (ctxt->state == XSLT_STATE_STOPPED)){ - yaz_log(YLOG_WARN, "XSLT transformation failed from %s", - cl->database->database->url); - xmlFreeDoc(new); - xmlFreeDoc(rdoc); - return 0; - } - } -#endif - - xmlFreeDoc(rdoc); - rdoc = new; - } - if (global_parameters.dump_records) - { - fprintf(stderr, "Record from %s\n----------------\n", - cl->database->database->url); -#if LIBXML_VERSION >= 20600 - xmlDocFormatDump(stderr, rdoc, 1); -#else - xmlDocDump(stderr, rdoc); -#endif - } - return rdoc; -} - -// Extract what appears to be years from buf, storing highest and -// lowest values. -static int extract_years(const char *buf, int *first, int *last) -{ - *first = -1; - *last = -1; - while (*buf) - { - const char *e; - int len; - - while (*buf && !isdigit(*buf)) - buf++; - len = 0; - for (e = buf; *e && isdigit(*e); e++) - len++; - if (len == 4) - { - int value = atoi(buf); - if (*first < 0 || value < *first) - *first = value; - if (*last < 0 || value > *last) - *last = value; - } - buf = e; - } - return *first; -} - -static struct record *ingest_record(struct client *cl, Z_External *rec) -{ - xmlDoc *xdoc = normalize_record(cl, rec); - xmlNode *root, *n; - struct record *res; - struct record_cluster *cluster; - struct session *se = cl->session; - xmlChar *mergekey, *mergekey_norm; - xmlChar *type = 0; - xmlChar *value = 0; - struct conf_service *service = global_parameters.server->service; - - if (!xdoc) - return 0; - - root = xmlDocGetRootElement(xdoc); - if (!(mergekey = xmlGetProp(root, (xmlChar *) "mergekey"))) - { - yaz_log(YLOG_WARN, "No mergekey found in record"); - xmlFreeDoc(xdoc); - return 0; - } - - res = nmem_malloc(se->nmem, sizeof(struct record)); - res->next = 0; - res->client = cl; - res->metadata = nmem_malloc(se->nmem, - sizeof(struct record_metadata*) * service->num_metadata); - memset(res->metadata, 0, sizeof(struct record_metadata*) * service->num_metadata); - - mergekey_norm = (xmlChar *) nmem_strdup(se->nmem, (char*) mergekey); - xmlFree(mergekey); - normalize_mergekey((char *) mergekey_norm, 0); - - cluster = reclist_insert(se->reclist, - global_parameters.server->service, - res, (char *) mergekey_norm, - &se->total_merged); - if (global_parameters.dump_records) - yaz_log(YLOG_LOG, "Cluster id %d from %s (#%d)", cluster->recid, - cl->database->database->url, cl->records); - if (!cluster) - { - /* no room for record */ - xmlFreeDoc(xdoc); - return 0; - } - relevance_newrec(se->relevance, cluster); - - for (n = root->children; n; n = n->next) - { - if (type) - xmlFree(type); - if (value) - xmlFree(value); - type = value = 0; - - if (n->type != XML_ELEMENT_NODE) - continue; - if (!strcmp((const char *) n->name, "metadata")) + else { - struct conf_metadata *md = 0; - struct conf_sortkey *sk = 0; - struct record_metadata **wheretoput, *newm; - int imeta; - int first, last; - - type = xmlGetProp(n, (xmlChar *) "type"); - value = xmlNodeListGetString(xdoc, n->children, 0); - - if (!type || !value) - continue; - - // First, find out what field we're looking at - for (imeta = 0; imeta < service->num_metadata; imeta++) - if (!strcmp((const char *) type, service->metadata[imeta].name)) - { - md = &service->metadata[imeta]; - if (md->sortkey_offset >= 0) - sk = &service->sortkeys[md->sortkey_offset]; - break; - } - if (!md) - { - yaz_log(YLOG_WARN, "Ignoring unknown metadata element: %s", type); - continue; - } - - // Find out where we are putting it - if (md->merge == Metadata_merge_no) - wheretoput = &res->metadata[imeta]; - else - wheretoput = &cluster->metadata[imeta]; + /* OPAC gets converted to XML too */ + WRBUF wrbuf_opac = wrbuf_alloc(); + /* MARCXML inside the OPAC XML. Charset is in effect because we + use the yaz_marc handle */ + yaz_marc_xml(sdb->yaz_marc, YAZ_MARC_MARCXML); + yaz_opac_decode_wrbuf(sdb->yaz_marc, rec->u.opac, wrbuf_opac); - // Put it there - newm = nmem_malloc(se->nmem, sizeof(struct record_metadata)); - newm->next = 0; - if (md->type == Metadata_type_generic) - { - char *p, *pe; - for (p = (char *) value; *p && isspace(*p); p++) - ; - for (pe = p + strlen(p) - 1; - pe > p && strchr(" ,/.:([", *pe); pe--) - *pe = '\0'; - newm->data.text = nmem_strdup(se->nmem, p); - - } - else if (md->type == Metadata_type_year) - { - if (extract_years((char *) value, &first, &last) < 0) - continue; - } - else - { - yaz_log(YLOG_WARN, "Unknown type in metadata element %s", type); - continue; - } - if (md->type == Metadata_type_year && md->merge != Metadata_merge_range) - { - yaz_log(YLOG_WARN, "Only range merging supported for years"); - continue; - } - if (md->merge == Metadata_merge_unique) + rdoc = xmlParseMemory((char*) wrbuf_buf(wrbuf_opac), + wrbuf_len(wrbuf_opac)); + if (!rdoc) { - struct record_metadata *mnode; - for (mnode = *wheretoput; mnode; mnode = mnode->next) - if (!strcmp((const char *) mnode->data.text, newm->data.text)) - break; - if (!mnode) - { - newm->next = *wheretoput; - *wheretoput = newm; - } - } - else if (md->merge == Metadata_merge_longest) - { - if (!*wheretoput || - strlen(newm->data.text) > strlen((*wheretoput)->data.text)) - { - *wheretoput = newm; - if (sk) - { - char *s = nmem_strdup(se->nmem, newm->data.text); - if (!cluster->sortkeys[md->sortkey_offset]) - cluster->sortkeys[md->sortkey_offset] = - nmem_malloc(se->nmem, sizeof(union data_types)); - normalize_mergekey(s, - (sk->type == Metadata_sortkey_skiparticle)); - cluster->sortkeys[md->sortkey_offset]->text = s; - } - } - } - else if (md->merge == Metadata_merge_all || md->merge == Metadata_merge_no) - { - newm->next = *wheretoput; - *wheretoput = newm; - } - else if (md->merge == Metadata_merge_range) - { - assert(md->type == Metadata_type_year); - if (!*wheretoput) - { - *wheretoput = newm; - (*wheretoput)->data.number.min = first; - (*wheretoput)->data.number.max = last; - if (sk) - cluster->sortkeys[md->sortkey_offset] = &newm->data; - } - else - { - if (first < (*wheretoput)->data.number.min) - (*wheretoput)->data.number.min = first; - if (last > (*wheretoput)->data.number.max) - (*wheretoput)->data.number.max = last; - } -#ifdef GAGA - if (sk) + yaz_log(YLOG_WARN, "Unable to parse OPAC XML"); + /* Was used to debug bug #1348 */ +#if 0 + FILE *f = fopen("/tmp/opac.xml.txt", "wb"); + if (f) { - union data_types *sdata = cluster->sortkeys[md->sortkey_offset]; - yaz_log(YLOG_LOG, "SK range: %d-%d", sdata->number.min, sdata->number.max); + fwrite(wrbuf_buf(wrbuf_opac), 1, wrbuf_len(wrbuf_opac), f); + fclose(f); } #endif } - else - yaz_log(YLOG_WARN, "Don't know how to merge on element name %s", md->name); - - if (md->rank) - relevance_countwords(se->relevance, cluster, - (char *) value, md->rank); - if (md->termlist) - { - if (md->type == Metadata_type_year) - { - char year[64]; - sprintf(year, "%d", last); - add_facet(se, (char *) type, year); - if (first != last) - { - sprintf(year, "%d", first); - add_facet(se, (char *) type, year); - } - } - else - add_facet(se, (char *) type, (char *) value); - } - xmlFree(type); - xmlFree(value); - type = value = 0; + wrbuf_destroy(wrbuf_opac); } - else - yaz_log(YLOG_WARN, "Unexpected element %s in internal record", n->name); } - if (type) - xmlFree(type); - if (value) - xmlFree(value); - - xmlFreeDoc(xdoc); - - relevance_donerecord(se->relevance, cluster); - se->total_records++; - - return res; -} - -// Retrieve first defined value for 'name' for given database. -// Will be extended to take into account user associated with session -char *session_setting_oneval(struct session_database *db, int offset) -{ - if (!db->settings[offset]) - return ""; - return db->settings[offset]->value; -} - -static void ingest_records(struct client *cl, Z_Records *r) -{ -#if USE_TIMING - yaz_timing_t t = yaz_timing_create(); -#endif - struct record *rec; - struct session *s = cl->session; - Z_NamePlusRecordList *rlist; - int i; - - if (r->which != Z_Records_DBOSD) - return; - rlist = r->u.databaseOrSurDiagnostics; - for (i = 0; i < rlist->num_records; i++) + else if (oid && yaz_oid_is_iso2709(oid)) { - Z_NamePlusRecord *npr = rlist->records[i]; - - cl->records++; - if (npr->which != Z_NamePlusRecord_databaseRecord) + /* ISO2709 gets converted to MARCXML */ + if (!sdb->yaz_marc) { - yaz_log(YLOG_WARN, - "Unexpected record type, probably diagnostic %s", - cl->database->database->url); - continue; + yaz_log(YLOG_WARN, "MARC decoding not configured"); + return 0; } - - rec = ingest_record(cl, npr->u.databaseRecord); - if (!rec) - continue; - } - if (s->watchlist[SESSION_WATCH_RECORDS].fun && rlist->num_records) - session_alert_watch(s, SESSION_WATCH_RECORDS); - -#if USE_TIMING - yaz_timing_stop(t); - yaz_log(YLOG_LOG, "ingest_records %6.5f %3.2f %3.2f", - yaz_timing_get_real(t), yaz_timing_get_user(t), - yaz_timing_get_sys(t)); - yaz_timing_destroy(&t); -#endif -} - -static void do_presentResponse(IOCHAN i, Z_APDU *a) -{ - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - Z_PresentResponse *r = a->u.presentResponse; - - if (r->records) { - Z_Records *recs = r->records; - if (recs->which == Z_Records_NSD) + else { - yaz_log(YLOG_WARN, "Non-surrogate diagnostic %s", - cl->database->database->url); - cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition; - cl->state = Client_Error; - } - } - - if (!*r->presentStatus && cl->state != Client_Error) - { - yaz_log(YLOG_DEBUG, "Good Present response %s", - cl->database->database->url); - ingest_records(cl, r->records); - cl->state = Client_Idle; - } - else if (*r->presentStatus) - { - yaz_log(YLOG_WARN, "Bad Present response %s", - cl->database->database->url); - cl->state = Client_Error; - } -} - -void connection_handler(IOCHAN i, int event) -{ - struct connection *co = iochan_getdata(i); - struct client *cl = co->client; - struct session *se = 0; - - if (cl) - se = cl->session; - else - { - yaz_log(YLOG_WARN, "Destroying orphan connection"); - connection_destroy(co); - return; - } - - if (co->state == Conn_Connecting && event & EVENT_OUTPUT) - { - int errcode; - socklen_t errlen = sizeof(errcode); - - if (getsockopt(cs_fileno(co->link), SOL_SOCKET, SO_ERROR, &errcode, - &errlen) < 0 || errcode != 0) - { - client_fatal(cl); - return; - } - else - { - yaz_log(YLOG_DEBUG, "Connect OK"); - co->state = Conn_Open; - if (cl) - cl->state = Client_Connected; - } - } - - else if (event & EVENT_INPUT) - { - int len = cs_get(co->link, &co->ibuf, &co->ibufsize); - - if (len < 0) - { - yaz_log(YLOG_WARN|YLOG_ERRNO, "Error reading from %s", - cl->database->database->url); - connection_destroy(co); - return; - } - else if (len == 0) - { - yaz_log(YLOG_WARN, "EOF reading from %s", cl->database->database->url); - connection_destroy(co); - return; - } - else if (len > 1) // We discard input if we have no connection - { - co->state = Conn_Open; - - if (cl && (cl->requestid == se->requestid || cl->state == Client_Initializing)) + xmlNode *res; + char *buf; + int len; + + if (rec->which != Z_External_octet) { - Z_APDU *a; - - odr_reset(global_parameters.odr_in); - odr_setbuf(global_parameters.odr_in, co->ibuf, len, 0); - if (!z_APDU(global_parameters.odr_in, &a, 0, 0)) - { - client_fatal(cl); - return; - } - switch (a->which) - { - case Z_APDU_initResponse: - do_initResponse(i, a); - break; - case Z_APDU_searchResponse: - do_searchResponse(i, a); - break; - case Z_APDU_presentResponse: - do_presentResponse(i, a); - break; - case Z_APDU_close: - do_closeResponse(i, a); - break; - default: - yaz_log(YLOG_WARN, - "Unexpected Z39.50 response from %s", - cl->database->database->url); - client_fatal(cl); - return; - } - // We aren't expecting staggered output from target - // if (cs_more(t->link)) - // iochan_setevent(i, EVENT_INPUT); + yaz_log(YLOG_WARN, "Unexpected external branch, probably BER %s", + db->url); + return 0; } - else // we throw away response and go to idle mode + buf = (char*) rec->u.octet_aligned->buf; + len = rec->u.octet_aligned->len; + if (yaz_marc_read_iso2709(sdb->yaz_marc, buf, len) < 0) { - yaz_log(YLOG_DEBUG, "Ignoring result of expired operation"); - cl->state = Client_Idle; + yaz_log(YLOG_WARN, "Failed to decode MARC %s", db->url); + return 0; } - } - /* if len==1 we do nothing but wait for more input */ - } - - if (cl->state == Client_Connected) { - send_init(i); - } - - if (cl->state == Client_Idle) - { - if (cl->requestid != se->requestid && cl->pquery) { - send_search(i); - } - else if (cl->hits > 0 && cl->records < global_parameters.toget && - cl->records < cl->hits) { - send_present(i); + + if (yaz_marc_write_xml(sdb->yaz_marc, &res, + "http://www.loc.gov/MARC21/slim", 0, 0) < 0) + { + yaz_log(YLOG_WARN, "Failed to encode as XML %s", + db->url); + return 0; + } + rdoc = xmlNewDoc((xmlChar *) "1.0"); + xmlDocSetRootElement(rdoc, res); } } -} - -// Disassociate connection from client -static void connection_release(struct connection *co) -{ - struct client *cl = co->client; - - yaz_log(YLOG_DEBUG, "Connection release %s", co->host->hostport); - if (!cl) - return; - cl->connection = 0; - co->client = 0; -} - -// Close connection and recycle structure -static void connection_destroy(struct connection *co) -{ - struct host *h = co->host; - - if (co->link) - { - cs_close(co->link); - iochan_destroy(co->iochan); - } - - yaz_log(YLOG_DEBUG, "Connection destroy %s", co->host->hostport); - if (h->connections == co) - h->connections = co->next; else { - struct connection *pco; - for (pco = h->connections; pco && pco->next != co; pco = pco->next) - ; - if (pco) - pco->next = co->next; - else - abort(); - } - if (co->client) - { - if (co->client->state != Client_Idle) - co->client->state = Client_Disconnected; - co->client->connection = 0; - } - co->next = connection_freelist; - connection_freelist = co; -} - -static int connection_connect(struct connection *con) -{ - COMSTACK link = 0; - struct client *cl = con->client; - struct host *host = con->host; - void *addr; - int res; - - assert(host->ipport); - assert(cl); - - if (!(link = cs_create(tcpip_type, 0, PROTO_Z3950))) - { - yaz_log(YLOG_FATAL|YLOG_ERRNO, "Failed to create comstack"); - exit(1); + char oid_name_buf[OID_STR_MAX]; + const char *oid_name = yaz_oid_to_string_buf(oid, 0, oid_name_buf); + yaz_log(YLOG_FATAL, + "Unable to handle record of type %s from %s", + oid_name, db->url); + return 0; } - if (0 == strlen(global_parameters.zproxy_override)){ - /* no Z39.50 proxy needed - direct connect */ - yaz_log(YLOG_DEBUG, "Connection create %s", cl->database->database->url); - - if (!(addr = cs_straddr(link, host->ipport))) - { - yaz_log(YLOG_WARN|YLOG_ERRNO, - "Lookup of IP address %s failed", host->ipport); - return -1; - } - - } else { - /* Z39.50 proxy connect */ - yaz_log(YLOG_DEBUG, "Connection create %s proxy %s", - cl->database->database->url, global_parameters.zproxy_override); - - if (!(addr = cs_straddr(link, global_parameters.zproxy_override))) + if (global_parameters.dump_records) + { + FILE *lf = yaz_log_file(); + if (lf) { - yaz_log(YLOG_WARN|YLOG_ERRNO, - "Lookup of IP address %s failed", - global_parameters.zproxy_override); - return -1; + yaz_log(YLOG_LOG, "Un-normalized record from %s", db->url); +#if LIBXML_VERSION >= 20600 + xmlDocFormatDump(lf, rdoc, 1); +#else + xmlDocDump(lf, rdoc); +#endif + fprintf(lf, "\n"); } } - - res = cs_connect(link, addr); - if (res < 0) - { - yaz_log(YLOG_WARN|YLOG_ERRNO, "cs_connect %s", cl->database->database->url); - return -1; - } - con->link = link; - con->state = Conn_Connecting; - con->iochan = iochan_create(cs_fileno(link), connection_handler, 0); - iochan_setdata(con->iochan, con); - pazpar2_add_channel(con->iochan); - - /* this fragment is bad DRY: from client_prep_connection */ - cl->state = Client_Connecting; - iochan_setflag(con->iochan, EVENT_OUTPUT); - return 0; + return rdoc; } -void connect_resolver_host(struct host *host) +#define MAX_XSLT_ARGS 16 + +// Add static values from session database settings if applicable +static void insert_settings_parameters(struct session_database *sdb, + struct session *se, char **parms) { - struct connection *con = host->connections; + struct conf_service *service = global_parameters.server->service; + int i; + int nparms = 0; + int offset = 0; - while (con) + for (i = 0; i < service->num_metadata; i++) { - if (con->state == Conn_Resolving) + struct conf_metadata *md = &service->metadata[i]; + int setting; + + if (md->setting == Metadata_setting_parameter && + (setting = settings_offset(md->name)) > 0) { - if (!host->ipport) /* unresolved */ + const char *val = session_setting_oneval(sdb, setting); + if (val && nparms < MAX_XSLT_ARGS) { - connection_destroy(con); - /* start all over .. at some point it will be NULL */ - con = host->connections; - } - else if (!con->client) - { - yaz_log(YLOG_WARN, "connect_unresolved_host : ophan client"); - connection_destroy(con); - /* start all over .. at some point it will be NULL */ - con = host->connections; - } - else - { - connection_connect(con); - con = con->next; + char *buf; + int len = strlen(val); + buf = nmem_malloc(se->nmem, len + 3); + buf[0] = '\''; + strcpy(buf + 1, val); + buf[len+1] = '\''; + buf[len+2] = '\0'; + parms[offset++] = md->name; + parms[offset++] = buf; + nparms++; } } } + parms[offset] = 0; } - -// Creates a new connection for client, associated with the host of -// client's database -static struct connection *connection_create(struct client *cl) +// Add static values from session database settings if applicable +static void insert_settings_values(struct session_database *sdb, xmlDoc *doc) { - struct connection *new; - struct host *host = cl->database->database->host; + struct conf_service *service = global_parameters.server->service; + int i; - if ((new = connection_freelist)) - connection_freelist = new->next; - else + for (i = 0; i < service->num_metadata; i++) { - new = xmalloc(sizeof (struct connection)); - new->ibuf = 0; - new->ibufsize = 0; - } - new->host = host; - new->next = new->host->connections; - new->host->connections = new; - new->client = cl; - cl->connection = new; - new->link = 0; - new->state = Conn_Resolving; - if (host->ipport) - connection_connect(new); - return new; -} - -// Close connection and set state to error -static void client_fatal(struct client *cl) -{ - yaz_log(YLOG_WARN, "Fatal error from %s", cl->database->database->url); - connection_destroy(cl->connection); - cl->state = Client_Error; -} + struct conf_metadata *md = &service->metadata[i]; + int offset; -// Ensure that client has a connection associated -static int client_prep_connection(struct client *cl) -{ - struct connection *co; - struct session *se = cl->session; - struct host *host = cl->database->database->host; - - co = cl->connection; - - yaz_log(YLOG_DEBUG, "Client prep %s", cl->database->database->url); - - if (!co) - { - // See if someone else has an idle connection - // We should look at timestamps here to select the longest-idle connection - for (co = host->connections; co; co = co->next) - if (co->state == Conn_Open && (!co->client || co->client->session != se)) - break; - if (co) + if (md->setting == Metadata_setting_postproc && + (offset = settings_offset(md->name)) > 0) { - connection_release(co); - cl->connection = co; - co->client = cl; + const char *val = session_setting_oneval(sdb, offset); + if (val) + { + xmlNode *r = xmlDocGetRootElement(doc); + xmlNode *n = xmlNewTextChild(r, 0, (xmlChar *) "metadata", + (xmlChar *) val); + xmlSetProp(n, (xmlChar *) "type", (xmlChar *) md->name); + } } - else - co = connection_create(cl); } - if (co) +} + +xmlDoc *normalize_record(struct session_database *sdb, struct session *se, + Z_External *rec) +{ + struct database_retrievalmap *m; + xmlDoc *rdoc = record_to_xml(sdb, rec); + if (rdoc) { - if (co->state == Conn_Connecting) + for (m = sdb->map; m; m = m->next) { - cl->state = Client_Connecting; - iochan_setflag(co->iochan, EVENT_OUTPUT); + xmlDoc *new = 0; + + { + xmlNodePtr root = 0; + char *parms[MAX_XSLT_ARGS*2+1]; + + insert_settings_parameters(sdb, se, parms); + + new = xsltApplyStylesheet(m->stylesheet, rdoc, (const char **) parms); + root= xmlDocGetRootElement(new); + if (!new || !root || !(root->children)) + { + yaz_log(YLOG_WARN, "XSLT transformation failed from %s", + sdb->database->url); + xmlFreeDoc(new); + xmlFreeDoc(rdoc); + return 0; + } + } + + xmlFreeDoc(rdoc); + rdoc = new; } - else if (co->state == Conn_Open) + + insert_settings_values(sdb, rdoc); + + if (global_parameters.dump_records) { - if (cl->state == Client_Error || cl->state == Client_Disconnected) - cl->state = Client_Idle; - iochan_setflag(co->iochan, EVENT_OUTPUT); + FILE *lf = yaz_log_file(); + + if (lf) + { + yaz_log(YLOG_LOG, "Normalized record from %s", + sdb->database->url); +#if LIBXML_VERSION >= 20600 + xmlDocFormatDump(lf, rdoc, 1); +#else + xmlDocDump(lf, rdoc); +#endif + fprintf(lf, "\n"); + } } - return 1; } - else - return 0; + return rdoc; +} + +// Retrieve first defined value for 'name' for given database. +// Will be extended to take into account user associated with session +const char *session_setting_oneval(struct session_database *db, int offset) +{ + if (!db->settings[offset]) + return ""; + return db->settings[offset]->value; } + + // Initialize YAZ Map structures for MARC-based targets static int prepare_yazmarc(struct session_database *sdb) { - char *s; + const char *s; if (!sdb->settings) { yaz_log(YLOG_WARN, "No settings for %s", sdb->database->url); return -1; } - if ((s = session_setting_oneval(sdb, PZ_NATIVESYNTAX)) && !strncmp(s, "iso2709", 7)) + if ((s = session_setting_oneval(sdb, PZ_NATIVESYNTAX)) + && !strncmp(s, "iso2709", 7)) { char *encoding = "marc-8s", *e; yaz_iconv_t cm; @@ -1344,7 +451,7 @@ static int prepare_yazmarc(struct session_database *sdb) // setting. However, this is not a realistic use scenario. static int prepare_map(struct session *se, struct session_database *sdb) { - char *s; + const char *s; if (!sdb->settings) { @@ -1356,7 +463,30 @@ static int prepare_map(struct session *se, struct session_database *sdb) char **stylesheets; struct database_retrievalmap **m = &sdb->map; int num, i; + char auto_stylesheet[256]; + if (!strcmp(s, "auto")) + { + const char *request_syntax = session_setting_oneval( + sdb, PZ_REQUESTSYNTAX); + if (request_syntax) + { + char *cp; + yaz_snprintf(auto_stylesheet, sizeof(auto_stylesheet), + "%s.xsl", request_syntax); + for (cp = auto_stylesheet; *cp; cp++) + { + /* deliberately only consider ASCII */ + if (*cp > 32 && *cp < 127) + *cp = tolower(*cp); + } + s = auto_stylesheet; + } + else + { + yaz_log(YLOG_WARN, "No pz:requestsyntax for auto stylesheet"); + } + } nmem_strsplit(se->session_nmem, ",", s, &stylesheets, &num); for (i = 0; i < num; i++) { @@ -1364,7 +494,7 @@ static int prepare_map(struct session *se, struct session_database *sdb) (*m)->next = 0; if (!((*m)->stylesheet = conf_load_stylesheet(stylesheets[i]))) { - yaz_log(YLOG_FATAL, "Unable to load stylesheet: %s", + yaz_log(YLOG_FATAL|YLOG_ERRNO, "Unable to load stylesheet: %s", stylesheets[i]); return -1; } @@ -1379,11 +509,13 @@ static int prepare_map(struct session *se, struct session_database *sdb) // This analyzes settings and recomputes any supporting data structures // if necessary. -static int prepare_session_database(struct session *se, struct session_database *sdb) +static int prepare_session_database(struct session *se, + struct session_database *sdb) { if (!sdb->settings) { - yaz_log(YLOG_WARN, "No settings associated with %s", sdb->database->url); + yaz_log(YLOG_WARN, + "No settings associated with %s", sdb->database->url); return -1; } if (sdb->settings[PZ_NATIVESYNTAX] && !sdb->yaz_marc) @@ -1399,126 +531,51 @@ static int prepare_session_database(struct session *se, struct session_database return 0; } -// Initialize CCL map for a target -static CCL_bibset prepare_cclmap(struct client *cl) +// called if watch should be removed because http_channel is to be destroyed +static void session_watch_cancel(void *data, struct http_channel *c, + void *data2) { - struct session_database *sdb = cl->database; - struct setting *s; - CCL_bibset res; + struct session_watchentry *ent = data; - if (!sdb->settings) - return 0; - res = ccl_qual_mk(); - for (s = sdb->settings[PZ_CCLMAP]; s; s = s->next) - { - char *p = strchr(s->name + 3, ':'); - if (!p) - { - yaz_log(YLOG_WARN, "Malformed cclmap name: %s", s->name); - ccl_qual_rm(&res); - return 0; - } - p++; - ccl_qual_fitem(res, s->value, p); - } - return res; + ent->fun = 0; + ent->data = 0; + ent->obs = 0; } -// Parse the query given the settings specific to this client -static int client_parse_query(struct client *cl, const char *query) +// set watch. Returns 0=OK, -1 if watch is already set +int session_set_watch(struct session *s, int what, + session_watchfun fun, void *data, + struct http_channel *chan) { - struct session *se = cl->session; - struct ccl_rpn_node *cn; - int cerror, cpos; - CCL_bibset ccl_map = prepare_cclmap(cl); - - if (!ccl_map) - return -1; - cn = ccl_find_str(ccl_map, query, &cerror, &cpos); - ccl_qual_rm(&ccl_map); - if (!cn) - { - cl->state = Client_Error; - yaz_log(YLOG_WARN, "Failed to parse query for %s", - cl->database->database->url); + if (s->watchlist[what].fun) return -1; - } - wrbuf_rewind(se->wrbuf); - ccl_pquery(se->wrbuf, cn); - wrbuf_putc(se->wrbuf, '\0'); - if (cl->pquery) - xfree(cl->pquery); - cl->pquery = xstrdup(wrbuf_buf(se->wrbuf)); - - if (!se->relevance) - { - // Initialize relevance structure with query terms - char *p[512]; - extract_terms(se->nmem, cn, p); - se->relevance = relevance_create(se->nmem, (const char **) p, - se->expected_maxrecs); - } - - ccl_rpn_delete(cn); + s->watchlist[what].fun = fun; + s->watchlist[what].data = data; + s->watchlist[what].obs = http_add_observer(chan, &s->watchlist[what], + session_watch_cancel); return 0; } -static struct client *client_create(void) +void session_alert_watch(struct session *s, int what) { - struct client *r; - if (client_freelist) + if (s->watchlist[what].fun) { - r = client_freelist; - client_freelist = client_freelist->next; - } - else - r = xmalloc(sizeof(struct client)); - r->pquery = 0; - r->database = 0; - r->connection = 0; - r->session = 0; - r->hits = 0; - r->records = 0; - r->setno = 0; - r->requestid = -1; - r->diagnostic = 0; - r->state = Client_Disconnected; - r->next = 0; - return r; -} + /* our watch is no longer associated with http_channel */ + void *data; + session_watchfun fun; -void client_destroy(struct client *c) -{ - struct session *se = c->session; - if (c == se->clients) - se->clients = c->next; - else - { - struct client *cc; - for (cc = se->clients; cc && cc->next != c; cc = cc->next) - ; - if (cc) - cc->next = c->next; - } - if (c->connection) - connection_release(c->connection); - c->next = client_freelist; - client_freelist = c; -} + http_remove_observer(s->watchlist[what].obs); + fun = s->watchlist[what].fun; + data = s->watchlist[what].data; -void session_set_watch(struct session *s, int what, session_watchfun fun, void *data) -{ - s->watchlist[what].fun = fun; - s->watchlist[what].data = data; -} + /* reset watch before fun is invoked - in case fun wants to set + it again */ + s->watchlist[what].fun = 0; + s->watchlist[what].data = 0; + s->watchlist[what].obs = 0; -void session_alert_watch(struct session *s, int what) -{ - if (!s->watchlist[what].fun) - return; - (*s->watchlist[what].fun)(s->watchlist[what].data); - s->watchlist[what].fun = 0; - s->watchlist[what].data = 0; + fun(data); + } } //callback for grep_databases @@ -1526,14 +583,13 @@ static void select_targets_callback(void *context, struct session_database *db) { struct session *se = (struct session*) context; struct client *cl = client_create(); - cl->database = db; - cl->session = se; - cl->next = se->clients; - se->clients = cl; + client_set_database(cl, db); + client_set_session(cl, se); } // Associates a set of clients with a session; -// Note: Session-databases represent databases with per-session setting overrides +// Note: Session-databases represent databases with per-session +// setting overrides int select_targets(struct session *se, struct database_criterion *crit) { while (se->clients) @@ -1547,11 +603,8 @@ int session_active_clients(struct session *s) struct client *c; int res = 0; - for (c = s->clients; c; c = c->next) - if (c->connection && (c->state == Client_Connecting || - c->state == Client_Initializing || - c->state == Client_Searching || - c->state == Client_Presenting)) + for (c = s->clients; c; c = client_next_in_session(c)) + if (client_is_active(c)) res++; return res; @@ -1586,7 +639,8 @@ static struct database_criterion *parse_filter(NMEM m, const char *buf) new->values = 0; for (subi = 0; subi < subnum; subi++) { - struct database_criterion_value *newv = nmem_malloc(m, sizeof(*newv)); + struct database_criterion_value *newv + = nmem_malloc(m, sizeof(*newv)); newv->value = subvalues[subi]; newv->next = new->values; new->values = newv; @@ -1597,46 +651,60 @@ static struct database_criterion *parse_filter(NMEM m, const char *buf) return res; } -char *search(struct session *se, char *query, char *filter) +enum pazpar2_error_code search(struct session *se, + char *query, char *filter, + const char **addinfo) { int live_channels = 0; + int no_working = 0; + int no_failed = 0; struct client *cl; struct database_criterion *criteria; yaz_log(YLOG_DEBUG, "Search"); + *addinfo = 0; nmem_reset(se->nmem); + se->relevance = 0; + se->total_records = se->total_hits = se->total_merged = 0; + se->reclist = 0; + se->num_termlists = 0; criteria = parse_filter(se->nmem, filter); se->requestid++; live_channels = select_targets(se, criteria); if (live_channels) { int maxrecs = live_channels * global_parameters.toget; - se->num_termlists = 0; se->reclist = reclist_create(se->nmem, maxrecs); - // This will be initialized in send_search() - se->total_records = se->total_hits = se->total_merged = 0; se->expected_maxrecs = maxrecs; } else - return "NOTARGETS"; - - se->relevance = 0; + return PAZPAR2_NO_TARGETS; - for (cl = se->clients; cl; cl = cl->next) + for (cl = se->clients; cl; cl = client_next_in_session(cl)) { - if (prepare_session_database(se, cl->database) < 0) - return "CONFIG_ERROR"; - if (client_parse_query(cl, query) < 0) // Query must parse for all targets - return "QUERY"; + if (prepare_session_database(se, client_get_database(cl)) < 0) + { + *addinfo = client_get_database(cl)->database->url; + return PAZPAR2_CONFIG_TARGET; + } + // Parse query for target + if (client_parse_query(cl, query) < 0) + no_failed++; + else + { + no_working++; + client_prep_connection(cl); + } } - for (cl = se->clients; cl; cl = cl->next) + // If no queries could be mapped, we signal an error + if (no_working == 0) { - client_prep_connection(cl); + *addinfo = "query"; + return PAZPAR2_MALFORMED_PARAMETER_VALUE; } - - return 0; + return PAZPAR2_NO_ERROR; } // Creates a new session_database object for a database @@ -1649,9 +717,12 @@ static void session_init_databases_fun(void *context, struct database *db) new->database = db; new->yaz_marc = 0; + new->map = 0; - new->settings = nmem_malloc(se->session_nmem, sizeof(struct settings *) * num); + new->settings + = nmem_malloc(se->session_nmem, sizeof(struct settings *) * num); memset(new->settings, 0, sizeof(struct settings*) * num); + if (db->settings) { for (i = 0; i < num; i++) @@ -1677,12 +748,13 @@ static void session_database_destroy(struct session_database *sdb) void session_init_databases(struct session *se) { se->databases = 0; - grep_databases(se, 0, session_init_databases_fun); + predef_grep_databases(se, 0, session_init_databases_fun); } // Probably session_init_databases_fun should be refactored instead of // called here. -static struct session_database *load_session_database(struct session *se, char *id) +static struct session_database *load_session_database(struct session *se, + char *id) { struct database *db = find_database(id, 0); @@ -1692,7 +764,8 @@ static struct session_database *load_session_database(struct session *se, char * } // Find an existing session database. If not found, load it -static struct session_database *find_session_database(struct session *se, char *id) +static struct session_database *find_session_database(struct session *se, + char *id) { struct session_database *sdb; @@ -1703,17 +776,24 @@ static struct session_database *find_session_database(struct session *se, char * } // Apply a session override to a database -void session_apply_setting(struct session *se, char *dbname, char *setting, char *value) +void session_apply_setting(struct session *se, char *dbname, char *setting, + char *value) { struct session_database *sdb = find_session_database(se, dbname); struct setting *new = nmem_malloc(se->session_nmem, sizeof(*new)); - int offset = settings_offset(setting); + int offset = settings_offset_cprefix(setting); if (offset < 0) { yaz_log(YLOG_WARN, "Unknown setting %s", setting); return; } + // Jakub: This breaks the filter setting. + /*if (offset == PZ_ID) + { + yaz_log(YLOG_WARN, "No need to set pz:id setting. Ignoring"); + return; + }*/ new->precedence = 0; new->target = dbname; new->name = setting; @@ -1725,23 +805,23 @@ void session_apply_setting(struct session *se, char *dbname, char *setting, char // (happens when a search starts and client connections are prepared) switch (offset) { - case PZ_NATIVESYNTAX: - if (sdb->yaz_marc) - { - yaz_marc_destroy(sdb->yaz_marc); - sdb->yaz_marc = 0; - } - break; - case PZ_XSLT: - if (sdb->map) - { - struct database_retrievalmap *m; - // We don't worry about the map structure -- it's in nmem - for (m = sdb->map; m; m = m->next) - xsltFreeStylesheet(m->stylesheet); - sdb->map = 0; - } - break; + case PZ_NATIVESYNTAX: + if (sdb->yaz_marc) + { + yaz_marc_destroy(sdb->yaz_marc); + sdb->yaz_marc = 0; + } + break; + case PZ_XSLT: + if (sdb->map) + { + struct database_retrievalmap *m; + // We don't worry about the map structure -- it's in nmem + for (m = sdb->map; m; m = m->next) + xsltFreeStylesheet(m->stylesheet); + sdb->map = 0; + } + break; } } @@ -1749,7 +829,6 @@ void destroy_session(struct session *s) { struct session_database *sdb; - yaz_log(YLOG_LOG, "Destroying session"); while (s->clients) client_destroy(s->clients); for (sdb = s->databases; sdb; sdb = sdb->next) @@ -1765,6 +844,7 @@ struct session *new_session(NMEM nmem) yaz_log(YLOG_DEBUG, "New Pazpar2 session"); + session->relevance = 0; session->total_hits = 0; session->total_records = 0; session->num_termlists = 0; @@ -1775,36 +855,40 @@ struct session *new_session(NMEM nmem) session->session_nmem = nmem; session->nmem = nmem_create(); session->wrbuf = wrbuf_alloc(); - session_init_databases(session); + session->databases = 0; for (i = 0; i <= SESSION_WATCH_MAX; i++) { session->watchlist[i].data = 0; session->watchlist[i].fun = 0; } - return session; } -struct hitsbytarget *hitsbytarget(struct session *se, int *count) +struct hitsbytarget *hitsbytarget(struct session *se, int *count, NMEM nmem) { - static struct hitsbytarget res[1000]; // FIXME MM + struct hitsbytarget *res = 0; struct client *cl; + size_t sz = 0; + + for (cl = se->clients; cl; cl = client_next_in_session(cl)) + sz++; + res = nmem_malloc(nmem, sizeof(*res) * sz); *count = 0; - for (cl = se->clients; cl; cl = cl->next) + for (cl = se->clients; cl; cl = client_next_in_session(cl)) { - char *name = session_setting_oneval(cl->database, PZ_NAME); + const char *name = session_setting_oneval(client_get_database(cl), + PZ_NAME); - res[*count].id = cl->database->database->url; + res[*count].id = client_get_database(cl)->database->url; res[*count].name = *name ? name : "Unknown"; - res[*count].hits = cl->hits; - res[*count].records = cl->records; - res[*count].diagnostic = cl->diagnostic; - res[*count].state = client_states[cl->state]; - res[*count].connected = cl->connection ? 1 : 0; + res[*count].hits = client_get_hits(cl); + res[*count].records = client_get_num_records(cl); + res[*count].diagnostic = client_get_diagnostic(cl); + res[*count].state = client_get_state_str(cl); + res[*count].connected = client_get_connection(cl) ? 1 : 0; (*count)++; } - return res; } @@ -1831,56 +915,67 @@ void report_nmem_stats(void) } #endif -struct record_cluster *show_single(struct session *s, int id) +struct record_cluster *show_single(struct session *s, const char *id) { struct record_cluster *r; reclist_rewind(s->reclist); while ((r = reclist_read_record(s->reclist))) - if (r->recid == id) + if (!strcmp(r->recid, id)) return r; return 0; } -struct record_cluster **show(struct session *s, struct reclist_sortparms *sp, int start, - int *num, int *total, int *sumhits, NMEM nmem_show) +struct record_cluster **show(struct session *s, struct reclist_sortparms *sp, + int start, int *num, int *total, int *sumhits, + NMEM nmem_show) { struct record_cluster **recs = nmem_malloc(nmem_show, *num - * sizeof(struct record_cluster *)); + * sizeof(struct record_cluster *)); struct reclist_sortparms *spp; int i; #if USE_TIMING yaz_timing_t t = yaz_timing_create(); #endif - for (spp = sp; spp; spp = spp->next) - if (spp->type == Metadata_sortkey_relevance) - { - relevance_prepare_read(s->relevance, s->reclist); - break; - } - reclist_sort(s->reclist, sp); - - *total = s->reclist->num_records; - *sumhits = s->total_hits; - - for (i = 0; i < start; i++) - if (!reclist_read_record(s->reclist)) - { - *num = 0; - recs = 0; - break; - } - - for (i = 0; i < *num; i++) + if (!s->relevance) + { + *num = 0; + *total = 0; + *sumhits = 0; + recs = 0; + } + else { - struct record_cluster *r = reclist_read_record(s->reclist); - if (!r) + for (spp = sp; spp; spp = spp->next) + if (spp->type == Metadata_sortkey_relevance) + { + relevance_prepare_read(s->relevance, s->reclist); + break; + } + reclist_sort(s->reclist, sp); + + *total = s->reclist->num_records; + *sumhits = s->total_hits; + + for (i = 0; i < start; i++) + if (!reclist_read_record(s->reclist)) + { + *num = 0; + recs = 0; + break; + } + + for (i = 0; i < *num; i++) { - *num = i; - break; + struct record_cluster *r = reclist_read_record(s->reclist); + if (!r) + { + *num = i; + break; + } + recs[i] = r; } - recs[i] = r; } #if USE_TIMING yaz_timing_stop(t); @@ -1898,20 +993,20 @@ void statistics(struct session *se, struct statistics *stat) int count = 0; memset(stat, 0, sizeof(*stat)); - for (cl = se->clients; cl; cl = cl->next) + for (cl = se->clients; cl; cl = client_next_in_session(cl)) { - if (!cl->connection) + if (!client_get_connection(cl)) stat->num_no_connection++; - switch (cl->state) + switch (client_get_state(cl)) { - case Client_Connecting: stat->num_connecting++; break; - case Client_Initializing: stat->num_initializing++; break; - case Client_Searching: stat->num_searching++; break; - case Client_Presenting: stat->num_presenting++; break; - case Client_Idle: stat->num_idle++; break; - case Client_Failed: stat->num_failed++; break; - case Client_Error: stat->num_error++; break; - default: break; + case Client_Connecting: stat->num_connecting++; break; + case Client_Initializing: stat->num_initializing++; break; + case Client_Searching: stat->num_searching++; break; + case Client_Presenting: stat->num_presenting++; break; + case Client_Idle: stat->num_idle++; break; + case Client_Failed: stat->num_failed++; break; + case Client_Error: stat->num_error++; break; + default: break; } count++; } @@ -1964,38 +1059,6 @@ void start_proxy(void) http_set_proxyaddr(hp, ser->myurl ? ser->myurl : ""); } -void start_zproxy(void) -{ - struct conf_server *ser = global_parameters.server; - - if (*global_parameters.zproxy_override){ - yaz_log(YLOG_LOG, "Z39.50 proxy %s", - global_parameters.zproxy_override); - return; - } - - else if (ser->zproxy_host || ser->zproxy_port) - { - char hp[128] = ""; - - strcpy(hp, ser->zproxy_host ? ser->zproxy_host : ""); - if (ser->zproxy_port) - { - if (*hp) - strcat(hp, ":"); - else - strcat(hp, "@:"); - - sprintf(hp + strlen(hp), "%d", ser->zproxy_port); - } - strcpy(global_parameters.zproxy_override, hp); - yaz_log(YLOG_LOG, "Z39.50 proxy %s", - global_parameters.zproxy_override); - - } - else - return; -} // Master list of connections we're handling events to static IOCHAN channel_list = 0; @@ -2010,6 +1073,300 @@ void pazpar2_event_loop() event_loop(&channel_list); } +static struct record_metadata *record_metadata_init( + NMEM nmem, char *value, enum conf_metadata_type type) +{ + struct record_metadata *rec_md = record_metadata_create(nmem); + if (type == Metadata_type_generic) + { + char * p = value; + p = normalize7bit_generic(p, " ,/.:(["); + + rec_md->data.text.disp = nmem_strdup(nmem, p); + rec_md->data.text.sort = 0; + } + else if (type == Metadata_type_year || type == Metadata_type_date) + { + int first, last; + int longdate = 0; + + if (type == Metadata_type_date) + longdate = 1; + if (extract7bit_dates((char *) value, &first, &last, longdate) < 0) + return 0; + + rec_md->data.number.min = first; + rec_md->data.number.max = last; + } + else + return 0; + return rec_md; +} + +struct record *ingest_record(struct client *cl, Z_External *rec, + int record_no) +{ + xmlDoc *xdoc = normalize_record(client_get_database(cl), + client_get_session(cl), rec); + xmlNode *root, *n; + struct record *record; + struct record_cluster *cluster; + struct session *se = client_get_session(cl); + xmlChar *mergekey, *mergekey_norm; + xmlChar *type = 0; + xmlChar *value = 0; + struct conf_service *service = global_parameters.server->service; + const char *norm_str = 0; + pp2_relevance_token_t prt = 0; + WRBUF norm_wr = 0; + + if (!xdoc) + return 0; + + root = xmlDocGetRootElement(xdoc); + if (!(mergekey = xmlGetProp(root, (xmlChar *) "mergekey"))) + { + yaz_log(YLOG_WARN, "No mergekey found in record"); + xmlFreeDoc(xdoc); + return 0; + } + + record = record_create(se->nmem, + service->num_metadata, service->num_sortkeys, cl, + record_no); + + prt = pp2_relevance_tokenize( + global_parameters.server->mergekey_pct, (const char *) mergekey); + + + norm_wr = wrbuf_alloc(); + + while ((norm_str = pp2_relevance_token_next(prt))) + { + if (*norm_str) + { + if (wrbuf_len(norm_wr)) + wrbuf_puts(norm_wr, " "); + wrbuf_puts(norm_wr, norm_str); + } + } + + mergekey_norm = (xmlChar *)nmem_strdup(se->nmem, wrbuf_cstr(norm_wr)); + wrbuf_destroy(norm_wr); + + pp2_relevance_token_destroy(prt); + + xmlFree(mergekey); + + cluster = reclist_insert(se->reclist, + global_parameters.server->service, + record, (char *) mergekey_norm, + &se->total_merged); + if (global_parameters.dump_records) + yaz_log(YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid, + client_get_database(cl)->database->url, record_no); + if (!cluster) + { + /* no room for record */ + xmlFreeDoc(xdoc); + return 0; + } + relevance_newrec(se->relevance, cluster); + + + // now parsing XML record and adding data to cluster or record metadata + for (n = root->children; n; n = n->next) + { + if (type) + xmlFree(type); + if (value) + xmlFree(value); + type = value = 0; + + if (n->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) n->name, "metadata")) + { + struct conf_metadata *ser_md = 0; + struct conf_sortkey *ser_sk = 0; + struct record_metadata **wheretoput = 0; + struct record_metadata *rec_md = 0; + int md_field_id = -1; + int sk_field_id = -1; + + type = xmlGetProp(n, (xmlChar *) "type"); + value = xmlNodeListGetString(xdoc, n->children, 1); + + if (!type || !value || !*value) + continue; + + md_field_id + = conf_service_metadata_field_id(service, (const char *) type); + if (md_field_id < 0) + { + yaz_log(YLOG_WARN, + "Ignoring unknown metadata element: %s", type); + continue; + } + + ser_md = &service->metadata[md_field_id]; + + if (ser_md->sortkey_offset >= 0){ + sk_field_id = ser_md->sortkey_offset; + ser_sk = &service->sortkeys[sk_field_id]; + } + + // non-merged metadata + rec_md = record_metadata_init(se->nmem, (char *) value, + ser_md->type); + if (!rec_md) + { + yaz_log(YLOG_WARN, "bad metadata data '%s' for element '%s'", + value, type); + continue; + } + rec_md->next = record->metadata[md_field_id]; + record->metadata[md_field_id] = rec_md; + + // merged metadata + rec_md = record_metadata_init(se->nmem, (char *) value, + ser_md->type); + wheretoput = &cluster->metadata[md_field_id]; + + // and polulate with data: + // assign cluster or record based on merge action + if (ser_md->merge == Metadata_merge_unique) + { + struct record_metadata *mnode; + for (mnode = *wheretoput; mnode; mnode = mnode->next) + if (!strcmp((const char *) mnode->data.text.disp, + rec_md->data.text.disp)) + break; + if (!mnode) + { + rec_md->next = *wheretoput; + *wheretoput = rec_md; + } + } + else if (ser_md->merge == Metadata_merge_longest) + { + if (!*wheretoput + || strlen(rec_md->data.text.disp) + > strlen((*wheretoput)->data.text.disp)) + { + *wheretoput = rec_md; + if (ser_sk) + { + const char *sort_str = 0; + int skip_article = + ser_sk->type == Metadata_sortkey_skiparticle; + + if (!cluster->sortkeys[sk_field_id]) + cluster->sortkeys[sk_field_id] = + nmem_malloc(se->nmem, + sizeof(union data_types)); + + prt = pp2_relevance_tokenize( + global_parameters.server->sort_pct, + rec_md->data.text.disp); + + pp2_relevance_token_next(prt); + + sort_str = pp2_get_sort(prt, skip_article); + + cluster->sortkeys[sk_field_id]->text.disp = + rec_md->data.text.disp; + if (!sort_str) + { + sort_str = rec_md->data.text.disp; + yaz_log(YLOG_WARN, + "Could not make sortkey. Bug #1858"); + } + cluster->sortkeys[sk_field_id]->text.sort = + nmem_strdup(se->nmem, sort_str); +#if 0 + yaz_log(YLOG_LOG, "text disp=%s", + cluster->sortkeys[sk_field_id]->text.disp); + yaz_log(YLOG_LOG, "text sort=%s", + cluster->sortkeys[sk_field_id]->text.sort); +#endif + pp2_relevance_token_destroy(prt); + } + } + } + else if (ser_md->merge == Metadata_merge_all) + { + rec_md->next = *wheretoput; + *wheretoput = rec_md; + } + else if (ser_md->merge == Metadata_merge_range) + { + if (!*wheretoput) + { + *wheretoput = rec_md; + if (ser_sk) + cluster->sortkeys[sk_field_id] + = &rec_md->data; + } + else + { + int this_min = rec_md->data.number.min; + int this_max = rec_md->data.number.max; + if (this_min < (*wheretoput)->data.number.min) + (*wheretoput)->data.number.min = this_min; + if (this_max > (*wheretoput)->data.number.max) + (*wheretoput)->data.number.max = this_max; + } + } + + + // ranking of _all_ fields enabled ... + if (ser_md->rank) + relevance_countwords(se->relevance, cluster, + (char *) value, ser_md->rank); + + // construct facets ... + if (ser_md->termlist) + { + if (ser_md->type == Metadata_type_year) + { + char year[64]; + sprintf(year, "%d", rec_md->data.number.max); + add_facet(se, (char *) type, year); + if (rec_md->data.number.max != rec_md->data.number.min) + { + sprintf(year, "%d", rec_md->data.number.min); + add_facet(se, (char *) type, year); + } + } + else + add_facet(se, (char *) type, (char *) value); + } + + // cleaning up + xmlFree(type); + xmlFree(value); + type = value = 0; + } + else + yaz_log(YLOG_WARN, + "Unexpected element %s in internal record", n->name); + } + if (type) + xmlFree(type); + if (value) + xmlFree(value); + + xmlFreeDoc(xdoc); + + relevance_donerecord(se->relevance, cluster); + se->total_records++; + + return record; +} + + + /* * Local variables: * c-basic-offset: 4