X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Flogic.c;h=f98d2407214a982a436d7b9ab1056b6a199cfc3c;hb=245fa946733907320f50a88095d71771ce4cda6a;hp=c9a0f67c65b9b385c69cf1a281a6ceb2c7f60cfa;hpb=f61b6ca2619d1ec14bdfca81f8a978d4e1db70d3;p=pazpar2-moved-to-github.git diff --git a/src/logic.c b/src/logic.c index c9a0f67..f98d240 100644 --- a/src/logic.c +++ b/src/logic.c @@ -1,4 +1,4 @@ -/* $Id: logic.c,v 1.40 2007-06-11 12:08:23 adam Exp $ +/* $Id: logic.c,v 1.62 2007-08-17 12:39:11 adam Exp $ Copyright (c) 2006-2007, Index Data. This file is part of Pazpar2. @@ -45,9 +45,8 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include #include #include -#if YAZ_VERSIONL >= 0x020163 #include -#endif +#include #if HAVE_CONFIG_H #include "cconfig.h" @@ -94,10 +93,11 @@ struct parameters global_parameters = 100, MAX_CHUNK, 0, - 0 + 0, + 180, + 30 }; - // Recursively traverse query structure to extract terms. void pull_terms(NMEM nmem, struct ccl_rpn_node *n, char **termlist, int *num) { @@ -152,9 +152,8 @@ static void add_facet(struct session *s, const char *type, const char *value) termlist_insert(s->termlists[i].termlist, value); } -xmlDoc *normalize_record(struct session_database *sdb, Z_External *rec) +xmlDoc *record_to_xml(struct session_database *sdb, Z_External *rec) { - struct database_retrievalmap *m; struct database *db = sdb->database; xmlDoc *rdoc = 0; const Odr_oid *oid = rec->direct_reference; @@ -173,12 +172,46 @@ xmlDoc *normalize_record(struct session_database *sdb, Z_External *rec) return 0; } } + else if (rec->which == Z_External_OPAC) + { + if (!sdb->yaz_marc) + { + yaz_log(YLOG_WARN, "MARC decoding not configured"); + return 0; + } + else + { + /* OPAC gets converted to XML too */ + WRBUF wrbuf_opac = wrbuf_alloc(); + /* MARCXML inside the OPAC XML. Charset is in effect because we + use the yaz_marc handle */ + yaz_marc_xml(sdb->yaz_marc, YAZ_MARC_MARCXML); + yaz_opac_decode_wrbuf(sdb->yaz_marc, rec->u.opac, wrbuf_opac); + + rdoc = xmlParseMemory((char*) wrbuf_buf(wrbuf_opac), + wrbuf_len(wrbuf_opac)); + if (!rdoc) + { + yaz_log(YLOG_WARN, "Unable to parse OPAC XML"); + /* Was used to debug bug #1348 */ +#if 0 + FILE *f = fopen("/tmp/opac.xml.txt", "wb"); + if (f) + { + fwrite(wrbuf_buf(wrbuf_opac), 1, wrbuf_len(wrbuf_opac), f); + fclose(f); + } +#endif + } + wrbuf_destroy(wrbuf_opac); + } + } else if (oid && yaz_oid_is_iso2709(oid)) { /* ISO2709 gets converted to MARCXML */ if (!sdb->yaz_marc) { - yaz_log(YLOG_FATAL, "Unable to handle ISO2709 record"); + yaz_log(YLOG_WARN, "MARC decoding not configured"); return 0; } else @@ -201,7 +234,6 @@ xmlDoc *normalize_record(struct session_database *sdb, Z_External *rec) return 0; } - yaz_marc_write_using_libxml2(sdb->yaz_marc, 1); if (yaz_marc_write_xml(sdb->yaz_marc, &res, "http://www.loc.gov/MARC21/slim", 0, 0) < 0) { @@ -223,53 +255,145 @@ xmlDoc *normalize_record(struct session_database *sdb, Z_External *rec) return 0; } - if (global_parameters.dump_records){ - fprintf(stderr, - "Input Record (normalized) from %s\n----------------\n", - db->url); + if (global_parameters.dump_records) + { + FILE *lf = yaz_log_file(); + if (lf) + { + yaz_log(YLOG_LOG, "Un-normalized record from %s", db->url); #if LIBXML_VERSION >= 20600 - xmlDocFormatDump(stderr, rdoc, 1); + xmlDocFormatDump(lf, rdoc, 1); #else - xmlDocDump(stderr, rdoc); + xmlDocDump(lf, rdoc); #endif + fprintf(lf, "\n"); + } } + return rdoc; +} - for (m = sdb->map; m; m = m->next){ - xmlDoc *new = 0; +#define MAX_XSLT_ARGS 16 +// Add static values from session database settings if applicable +static void insert_settings_parameters(struct session_database *sdb, + struct session *se, char **parms) +{ + struct conf_service *service = global_parameters.server->service; + int i; + int nparms = 0; + int offset = 0; + + for (i = 0; i < service->num_metadata; i++) + { + struct conf_metadata *md = &service->metadata[i]; + int setting; + + if (md->setting == Metadata_setting_parameter && + (setting = settings_offset(md->name)) > 0) { - xmlNodePtr root = 0; - new = xsltApplyStylesheet(m->stylesheet, rdoc, 0); - root= xmlDocGetRootElement(new); - if (!new || !root || !(root->children)) - { - yaz_log(YLOG_WARN, "XSLT transformation failed from %s", - sdb->database->url); - xmlFreeDoc(new); - xmlFreeDoc(rdoc); - return 0; + const char *val = session_setting_oneval(sdb, setting); + if (val && nparms < MAX_XSLT_ARGS) + { + char *buf; + int len = strlen(val); + buf = nmem_malloc(se->nmem, len + 3); + buf[0] = '\''; + strcpy(buf + 1, val); + buf[len+1] = '\''; + buf[len+2] = '\0'; + parms[offset++] = md->name; + parms[offset++] = buf; + nparms++; + } } + } + parms[offset] = 0; +} + +// Add static values from session database settings if applicable +static void insert_settings_values(struct session_database *sdb, xmlDoc *doc) +{ + struct conf_service *service = global_parameters.server->service; + int i; + + for (i = 0; i < service->num_metadata; i++) + { + struct conf_metadata *md = &service->metadata[i]; + int offset; + + if (md->setting == Metadata_setting_postproc && + (offset = settings_offset(md->name)) > 0) + { + const char *val = session_setting_oneval(sdb, offset); + if (val) + { + xmlNode *r = xmlDocGetRootElement(doc); + xmlNode *n = xmlNewTextChild(r, 0, (xmlChar *) "metadata", + (xmlChar *) val); + xmlSetProp(n, (xmlChar *) "type", (xmlChar *) md->name); + } } - - xmlFreeDoc(rdoc); - rdoc = new; } - if (global_parameters.dump_records) +} + +xmlDoc *normalize_record(struct session_database *sdb, struct session *se, + Z_External *rec) +{ + struct database_retrievalmap *m; + xmlDoc *rdoc = record_to_xml(sdb, rec); + if (rdoc) { - fprintf(stderr, "Record from %s\n----------------\n", - sdb->database->url); + for (m = sdb->map; m; m = m->next) + { + xmlDoc *new = 0; + + { + xmlNodePtr root = 0; + char *parms[MAX_XSLT_ARGS*2+1]; + + insert_settings_parameters(sdb, se, parms); + + new = xsltApplyStylesheet(m->stylesheet, rdoc, (const char **) parms); + root= xmlDocGetRootElement(new); + if (!new || !root || !(root->children)) + { + yaz_log(YLOG_WARN, "XSLT transformation failed from %s", + sdb->database->url); + xmlFreeDoc(new); + xmlFreeDoc(rdoc); + return 0; + } + } + + xmlFreeDoc(rdoc); + rdoc = new; + } + + insert_settings_values(sdb, rdoc); + + if (global_parameters.dump_records) + { + FILE *lf = yaz_log_file(); + + if (lf) + { + yaz_log(YLOG_LOG, "Normalized record from %s", + sdb->database->url); #if LIBXML_VERSION >= 20600 - xmlDocFormatDump(stderr, rdoc, 1); + xmlDocFormatDump(lf, rdoc, 1); #else - xmlDocDump(stderr, rdoc); + xmlDocDump(lf, rdoc); #endif + fprintf(lf, "\n"); + } + } } return rdoc; } // Retrieve first defined value for 'name' for given database. // Will be extended to take into account user associated with session -char *session_setting_oneval(struct session_database *db, int offset) +const char *session_setting_oneval(struct session_database *db, int offset) { if (!db->settings[offset]) return ""; @@ -281,7 +405,7 @@ char *session_setting_oneval(struct session_database *db, int offset) // Initialize YAZ Map structures for MARC-based targets static int prepare_yazmarc(struct session_database *sdb) { - char *s; + const char *s; if (!sdb->settings) { @@ -321,7 +445,7 @@ static int prepare_yazmarc(struct session_database *sdb) // setting. However, this is not a realistic use scenario. static int prepare_map(struct session *se, struct session_database *sdb) { - char *s; + const char *s; if (!sdb->settings) { @@ -333,7 +457,30 @@ static int prepare_map(struct session *se, struct session_database *sdb) char **stylesheets; struct database_retrievalmap **m = &sdb->map; int num, i; + char auto_stylesheet[256]; + if (!strcmp(s, "auto")) + { + const char *request_syntax = session_setting_oneval( + sdb, PZ_REQUESTSYNTAX); + if (request_syntax) + { + char *cp; + yaz_snprintf(auto_stylesheet, sizeof(auto_stylesheet), + "%s.xsl", request_syntax); + for (cp = auto_stylesheet; *cp; cp++) + { + /* deliberately only consider ASCII */ + if (*cp > 32 && *cp < 127) + *cp = tolower(*cp); + } + s = auto_stylesheet; + } + else + { + yaz_log(YLOG_WARN, "No pz:requestsyntax for auto stylesheet"); + } + } nmem_strsplit(se->session_nmem, ",", s, &stylesheets, &num); for (i = 0; i < num; i++) { @@ -341,7 +488,7 @@ static int prepare_map(struct session *se, struct session_database *sdb) (*m)->next = 0; if (!((*m)->stylesheet = conf_load_stylesheet(stylesheets[i]))) { - yaz_log(YLOG_FATAL, "Unable to load stylesheet: %s", + yaz_log(YLOG_FATAL|YLOG_ERRNO, "Unable to load stylesheet: %s", stylesheets[i]); return -1; } @@ -378,21 +525,39 @@ static int prepare_session_database(struct session *se, return 0; } +// called if watch should be removed because http_channel is to be destroyed +static void session_watch_cancel(void *data, struct http_channel *c) +{ + struct session_watchentry *ent = data; + + ent->fun = 0; + ent->data = 0; + ent->obs = 0; +} -void session_set_watch(struct session *s, int what, - session_watchfun fun, void *data) +// set watch. Returns 0=OK, -1 if watch is already set +int session_set_watch(struct session *s, int what, + session_watchfun fun, void *data, + struct http_channel *chan) { + if (s->watchlist[what].fun) + return -1; s->watchlist[what].fun = fun; s->watchlist[what].data = data; + s->watchlist[what].obs = http_add_observer(chan, &s->watchlist[what], + session_watch_cancel); + return 0; } void session_alert_watch(struct session *s, int what) { if (!s->watchlist[what].fun) return; + http_remove_observer(s->watchlist[what].obs); (*s->watchlist[what].fun)(s->watchlist[what].data); s->watchlist[what].fun = 0; s->watchlist[what].data = 0; + s->watchlist[what].obs = 0; } //callback for grep_databases @@ -468,45 +633,60 @@ static struct database_criterion *parse_filter(NMEM m, const char *buf) return res; } -char *search(struct session *se, char *query, char *filter) +enum pazpar2_error_code search(struct session *se, + char *query, char *filter, + const char **addinfo) { int live_channels = 0; + int no_working = 0; + int no_failed = 0; struct client *cl; struct database_criterion *criteria; yaz_log(YLOG_DEBUG, "Search"); + *addinfo = 0; nmem_reset(se->nmem); + se->relevance = 0; + se->total_records = se->total_hits = se->total_merged = 0; + se->reclist = 0; + se->num_termlists = 0; criteria = parse_filter(se->nmem, filter); se->requestid++; live_channels = select_targets(se, criteria); if (live_channels) { int maxrecs = live_channels * global_parameters.toget; - se->num_termlists = 0; se->reclist = reclist_create(se->nmem, maxrecs); - // This will be initialized in send_search() - se->total_records = se->total_hits = se->total_merged = 0; se->expected_maxrecs = maxrecs; } else - return "NOTARGETS"; - - se->relevance = 0; + return PAZPAR2_NO_TARGETS; for (cl = se->clients; cl; cl = client_next_in_session(cl)) { if (prepare_session_database(se, client_get_database(cl)) < 0) - return "CONFIG_ERROR"; - // Query must parse for all targets - if (client_parse_query(cl, query) < 0) - return "QUERY"; + { + *addinfo = client_get_database(cl)->database->url; + return PAZPAR2_CONFIG_TARGET; + } + // Parse query for target + if (client_parse_query(cl, query) < 0) + no_failed++; + else + { + no_working++; + client_prep_connection(cl); + } } - for (cl = se->clients; cl; cl = client_next_in_session(cl)) - client_prep_connection(cl); - - return 0; + // If no queries could be mapped, we signal an error + if (no_working == 0) + { + *addinfo = "query"; + return PAZPAR2_MALFORMED_PARAMETER_VALUE; + } + return PAZPAR2_NO_ERROR; } // Creates a new session_database object for a database @@ -561,7 +741,7 @@ static void session_database_destroy(struct session_database *sdb) void session_init_databases(struct session *se) { se->databases = 0; - grep_databases(se, 0, session_init_databases_fun); + predef_grep_databases(se, 0, session_init_databases_fun); } // Probably session_init_databases_fun should be refactored instead of @@ -642,7 +822,6 @@ void destroy_session(struct session *s) { struct session_database *sdb; - yaz_log(YLOG_LOG, "Destroying session"); while (s->clients) client_destroy(s->clients); for (sdb = s->databases; sdb; sdb = sdb->next) @@ -658,6 +837,7 @@ struct session *new_session(NMEM nmem) yaz_log(YLOG_DEBUG, "New Pazpar2 session"); + session->relevance = 0; session->total_hits = 0; session->total_records = 0; session->num_termlists = 0; @@ -668,7 +848,7 @@ struct session *new_session(NMEM nmem) session->session_nmem = nmem; session->nmem = nmem_create(); session->wrbuf = wrbuf_alloc(); - session_init_databases(session); + session->databases = 0; for (i = 0; i <= SESSION_WATCH_MAX; i++) { session->watchlist[i].data = 0; @@ -686,7 +866,8 @@ struct hitsbytarget *hitsbytarget(struct session *se, int *count) *count = 0; for (cl = se->clients; cl; cl = client_next_in_session(cl)) { - char *name = session_setting_oneval(client_get_database(cl), PZ_NAME); + const char *name = session_setting_oneval(client_get_database(cl), + PZ_NAME); res[*count].id = client_get_database(cl)->database->url; res[*count].name = *name ? name : "Unknown"; @@ -724,13 +905,13 @@ void report_nmem_stats(void) } #endif -struct record_cluster *show_single(struct session *s, int id) +struct record_cluster *show_single(struct session *s, const char *id) { struct record_cluster *r; reclist_rewind(s->reclist); while ((r = reclist_read_record(s->reclist))) - if (r->recid == id) + if (!strcmp(r->recid, id)) return r; return 0; } @@ -747,34 +928,44 @@ struct record_cluster **show(struct session *s, struct reclist_sortparms *sp, yaz_timing_t t = yaz_timing_create(); #endif - for (spp = sp; spp; spp = spp->next) - if (spp->type == Metadata_sortkey_relevance) - { - relevance_prepare_read(s->relevance, s->reclist); - break; - } - reclist_sort(s->reclist, sp); - - *total = s->reclist->num_records; - *sumhits = s->total_hits; - - for (i = 0; i < start; i++) - if (!reclist_read_record(s->reclist)) - { - *num = 0; - recs = 0; - break; - } - - for (i = 0; i < *num; i++) + if (!s->relevance) + { + *num = 0; + *total = 0; + *sumhits = 0; + recs = 0; + } + else { - struct record_cluster *r = reclist_read_record(s->reclist); - if (!r) + for (spp = sp; spp; spp = spp->next) + if (spp->type == Metadata_sortkey_relevance) + { + relevance_prepare_read(s->relevance, s->reclist); + break; + } + reclist_sort(s->reclist, sp); + + *total = s->reclist->num_records; + *sumhits = s->total_hits; + + for (i = 0; i < start; i++) + if (!reclist_read_record(s->reclist)) + { + *num = 0; + recs = 0; + break; + } + + for (i = 0; i < *num; i++) { - *num = i; - break; + struct record_cluster *r = reclist_read_record(s->reclist); + if (!r) + { + *num = i; + break; + } + recs[i] = r; } - recs[i] = r; } #if USE_TIMING yaz_timing_stop(t); @@ -872,10 +1063,35 @@ void pazpar2_event_loop() event_loop(&channel_list); } +static struct record_metadata *record_metadata_init( + NMEM nmem, char *value, enum conf_metadata_type type) +{ + struct record_metadata *rec_md = record_metadata_create(nmem); + if (type == Metadata_type_generic) + { + char * p = value; + p = normalize7bit_generic(p, " ,/.:(["); + + rec_md->data.text = nmem_strdup(nmem, p); + } + else if (type == Metadata_type_year) + { + int first, last; + if (extract7bit_years((char *) value, &first, &last) < 0) + return 0; + rec_md->data.number.min = first; + rec_md->data.number.max = last; + } + else + return 0; + return rec_md; +} + struct record *ingest_record(struct client *cl, Z_External *rec, int record_no) { - xmlDoc *xdoc = normalize_record(client_get_database(cl), rec); + xmlDoc *xdoc = normalize_record(client_get_database(cl), + client_get_session(cl), rec); xmlNode *root, *n; struct record *record; struct record_cluster *cluster; @@ -897,7 +1113,8 @@ struct record *ingest_record(struct client *cl, Z_External *rec, } record = record_create(se->nmem, - service->num_metadata, service->num_sortkeys, cl); + service->num_metadata, service->num_sortkeys, cl, + record_no); mergekey_norm = (xmlChar *) nmem_strdup(se->nmem, (char*) mergekey); xmlFree(mergekey); @@ -908,7 +1125,7 @@ struct record *ingest_record(struct client *cl, Z_External *rec, record, (char *) mergekey_norm, &se->total_merged); if (global_parameters.dump_records) - yaz_log(YLOG_LOG, "Cluster id %d from %s (#%d)", cluster->recid, + yaz_log(YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid, client_get_database(cl)->database->url, record_no); if (!cluster) { @@ -938,12 +1155,11 @@ struct record *ingest_record(struct client *cl, Z_External *rec, struct record_metadata *rec_md = 0; int md_field_id = -1; int sk_field_id = -1; - int first, last; type = xmlGetProp(n, (xmlChar *) "type"); value = xmlNodeListGetString(xdoc, n->children, 1); - if (!type || !value) + if (!type || !value || !*value) continue; md_field_id @@ -962,37 +1178,22 @@ struct record *ingest_record(struct client *cl, Z_External *rec, ser_sk = &service->sortkeys[sk_field_id]; } - // Find out where we are putting it - based on merge or not - if (ser_md->merge == Metadata_merge_no) - wheretoput = &record->metadata[md_field_id]; - else - wheretoput = &cluster->metadata[md_field_id]; - - // create new record_metadata - rec_md = record_metadata_create(se->nmem); - - // and polulate with data: - // type based charmapping decisions follow here - if (ser_md->type == Metadata_type_generic) - { - - char * p = (char *) value; - p = normalize7bit_generic(p, " ,/.:(["); - - rec_md->data.text = nmem_strdup(se->nmem, p); - - } - else if (ser_md->type == Metadata_type_year) + // non-merged metadata + rec_md = record_metadata_init(se->nmem, (char *) value, + ser_md->type); + if (!rec_md) { - if (extract7bit_years((char *) value, &first, &last) < 0) - continue; - } - else - { - yaz_log(YLOG_WARN, - "Unknown type in metadata element %s", type); + yaz_log(YLOG_WARN, "bad metadata data '%s' for element '%s'", + value, type); continue; } + rec_md->next = record->metadata[md_field_id]; + record->metadata[md_field_id] = rec_md; + + // merged metadata + rec_md = record_metadata_init(se->nmem, (char *) value, + ser_md->type); + wheretoput = &cluster->metadata[md_field_id]; // and polulate with data: // assign cluster or record based on merge action @@ -1029,8 +1230,7 @@ struct record *ingest_record(struct client *cl, Z_External *rec, } } } - else if (ser_md->merge == Metadata_merge_all - || ser_md->merge == Metadata_merge_no) + else if (ser_md->merge == Metadata_merge_all) { rec_md->next = *wheretoput; *wheretoput = rec_md; @@ -1040,18 +1240,18 @@ struct record *ingest_record(struct client *cl, Z_External *rec, if (!*wheretoput) { *wheretoput = rec_md; - (*wheretoput)->data.number.min = first; - (*wheretoput)->data.number.max = last; if (ser_sk) cluster->sortkeys[sk_field_id] = &rec_md->data; } else { - if (first < (*wheretoput)->data.number.min) - (*wheretoput)->data.number.min = first; - if (last > (*wheretoput)->data.number.max) - (*wheretoput)->data.number.max = last; + int this_min = rec_md->data.number.min; + int this_max = rec_md->data.number.max; + if (this_min < (*wheretoput)->data.number.min) + (*wheretoput)->data.number.min = this_min; + if (this_max > (*wheretoput)->data.number.max) + (*wheretoput)->data.number.max = this_max; } #ifdef GAGA if (ser_sk) @@ -1076,11 +1276,11 @@ struct record *ingest_record(struct client *cl, Z_External *rec, if (ser_md->type == Metadata_type_year) { char year[64]; - sprintf(year, "%d", last); + sprintf(year, "%d", rec_md->data.number.max); add_facet(se, (char *) type, year); - if (first != last) + if (rec_md->data.number.max != rec_md->data.number.min) { - sprintf(year, "%d", first); + sprintf(year, "%d", rec_md->data.number.min); add_facet(se, (char *) type, year); } }