X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Fpazpar2.c;h=3a696c60f02cdb45e3989581629971c41d7e1d5d;hb=aabc654ef7b6c5a6740e95b9580223726a86e43a;hp=3482d839a9db8b618258448a888d46886131fd9e;hpb=1ef3f77533020ff7c2373482a5b35caf10387c7a;p=pazpar2-moved-to-github.git

diff --git a/src/pazpar2.c b/src/pazpar2.c
index 3482d83..3a696c6 100644
--- a/src/pazpar2.c
+++ b/src/pazpar2.c
@@ -1,4 +1,4 @@
-/* $Id: pazpar2.c,v 1.11 2007-01-04 07:27:29 adam Exp $ */;
+/* $Id: pazpar2.c,v 1.44 2007-01-18 18:11:19 quinn Exp $ */
 
 #include <stdlib.h>
 #include <stdio.h>
@@ -11,6 +11,7 @@
 #include <ctype.h>
 #include <assert.h>
 
+#include <yaz/marcdisp.h>
 #include <yaz/comstack.h>
 #include <yaz/tcpip.h>
 #include <yaz/proto.h>
@@ -19,21 +20,25 @@
 #include <yaz/yaz-util.h>
 #include <yaz/nmem.h>
 
+#if HAVE_CONFIG_H
+#include "cconfig.h"
+#endif
+
 #define USE_TIMING 0
 #if USE_TIMING
 #include <yaz/timing.h>
 #endif
 
+#include <netinet/in.h>
+
 #include "pazpar2.h"
 #include "eventl.h"
-#include "command.h"
 #include "http.h"
 #include "termlists.h"
 #include "reclists.h"
 #include "relevance.h"
 #include "config.h"
 
-#define PAZPAR2_VERSION "0.1"
 #define MAX_CHUNK 15
 
 static void client_fatal(struct client *cl);
@@ -64,13 +69,17 @@ static char *client_states[] = {
     "Client_Stopped"
 };
 
+// Note: Some things in this structure will eventually move to configuration
 struct parameters global_parameters = 
 {
+    "",
+    "",
+    0,
     0,
     30,
     "81",
     "Index Data PazPar2 (MasterKey)",
-    PAZPAR2_VERSION,
+    VERSION,
     600, // 10 minutes
     60,
     100,
@@ -198,6 +207,8 @@ static void send_present(IOCHAN i)
     int start = cl->records + 1;
 
     toget = global_parameters.chunk;
+    if (toget > global_parameters.toget - cl->records)
+        toget = global_parameters.toget - cl->records;
     if (toget > cl->hits - cl->records)
 	toget = cl->hits - cl->records;
 
@@ -254,7 +265,6 @@ static void do_searchResponse(IOCHAN i, Z_APDU *a)
         if (r->presentStatus && !*r->presentStatus && r->records)
         {
             yaz_log(YLOG_DEBUG, "Records in search response");
-            cl->records += *r->numberOfRecordsReturned;
             ingest_records(cl, r->records);
         }
         cl->state = Client_Idle;
@@ -275,10 +285,27 @@ static void do_searchResponse(IOCHAN i, Z_APDU *a)
     }
 }
 
-char *normalize_mergekey(char *buf)
+char *normalize_mergekey(char *buf, int skiparticle)
 {
     char *p = buf, *pout = buf;
 
+    if (skiparticle)
+    {
+        char firstword[64];
+        char articles[] = "the den der die des an a "; // must end in space
+
+        while (*p && !isalnum(*p))
+            p++;
+        pout = firstword;
+        while (*p && *p != ' ' && pout - firstword < 62)
+            *(pout++) = tolower(*(p++));
+        *(pout++) = ' ';
+        *(pout++) = '\0';
+        if (!strstr(articles, firstword))
+            p = buf;
+        pout = buf;
+    }
+
     while (*p)
     {
         while (*p && !isalnum(*p))
@@ -291,7 +318,10 @@ char *normalize_mergekey(char *buf)
             p++;
     }
     if (buf != pout)
-        *pout = '\0';
+        do {
+            *(pout--) = '\0';
+        }
+        while (pout > buf && *pout == ' ');
 
     return buf;
 }
@@ -336,6 +366,8 @@ static void add_facet(struct session *s, const char *type, const char *value)
 {
     int i;
 
+    if (!*value)
+        return;
     for (i = 0; i < s->num_termlists; i++)
         if (!strcmp(s->termlists[i].name, type))
             break;
@@ -353,6 +385,8 @@ static void add_facet(struct session *s, const char *type, const char *value)
     termlist_insert(s->termlists[i].termlist, value);
 }
 
+int yaz_marc_write_xml();
+
 static xmlDoc *normalize_record(struct client *cl, Z_External *rec)
 {
     struct conf_retrievalprofile *rprofile = cl->database->rprofile;
@@ -391,6 +425,17 @@ static xmlDoc *normalize_record(struct client *cl, Z_External *rec)
         yaz_log(YLOG_FATAL, "Unknown native_syntax in normalize_record");
         exit(1);
     }
+
+    if (global_parameters.dump_records)
+    {
+        fprintf(stderr, "Input Record (normalized):\n----------------\n");
+#if LIBXML_VERSION >= 20600
+        xmlDocFormatDump(stderr, rdoc, 1);
+#else
+        xmlDocDump(stderr, rdoc);
+#endif
+    }
+
     for (m = rprofile->maplist; m; m = m->next)
     {
         xmlDoc *new;
@@ -410,18 +455,55 @@ static xmlDoc *normalize_record(struct client *cl, Z_External *rec)
     if (global_parameters.dump_records)
     {
         fprintf(stderr, "Record:\n----------------\n");
+#if LIBXML_VERSION >= 20600
         xmlDocFormatDump(stderr, rdoc, 1);
+#else
+        xmlDocDump(stderr, rdoc);
+#endif
     }
     return rdoc;
 }
 
+// Extract what appears to be years from buf, storing highest and
+// lowest values.
+static int extract_years(const char *buf, int *first, int *last)
+{
+    *first = -1;
+    *last = -1;
+    while (*buf)
+    {
+        const char *e;
+        int len;
+
+        while (*buf && !isdigit(*buf))
+            buf++;
+        len = 0;
+        for (e = buf; *e && isdigit(*e); e++)
+            len++;
+        if (len == 4)
+        {
+            int value = atoi(buf);
+            if (*first < 0 || value < *first)
+                *first = value;
+            if (*last < 0 || value > *last)
+                *last = value;
+        }
+        buf = e;
+    }
+    return *first;
+}
+
 static struct record *ingest_record(struct client *cl, Z_External *rec)
 {
     xmlDoc *xdoc = normalize_record(cl, rec);
     xmlNode *root, *n;
-    struct record *res, *head;
+    struct record *res;
+    struct record_cluster *cluster;
     struct session *se = cl->session;
     xmlChar *mergekey, *mergekey_norm;
+    xmlChar *type = 0;
+    xmlChar *value = 0;
+    struct conf_service *service = global_parameters.server->service;
 
     if (!xdoc)
         return 0;
@@ -430,54 +512,204 @@ static struct record *ingest_record(struct client *cl, Z_External *rec)
     if (!(mergekey = xmlGetProp(root, "mergekey")))
     {
         yaz_log(YLOG_WARN, "No mergekey found in record");
+        xmlFreeDoc(xdoc);
         return 0;
     }
 
     res = nmem_malloc(se->nmem, sizeof(struct record));
-    res->next_cluster = 0;
-    res->target_offset = -1;
-    res->term_frequency_vec = 0;
-    res->title = "Unknown";
-    res->relevance = 0;
+    res->next = 0;
+    res->client = cl;
+    res->metadata = nmem_malloc(se->nmem,
+            sizeof(struct record_metadata*) * service->num_metadata);
+    memset(res->metadata, 0, sizeof(struct record_metadata*) * service->num_metadata);
 
     mergekey_norm = nmem_strdup(se->nmem, (char*) mergekey);
     xmlFree(mergekey);
-    res->merge_key = normalize_mergekey(mergekey_norm);
+    normalize_mergekey(mergekey_norm, 0);
 
-    head = reclist_insert(se->reclist, res);
-    relevance_newrec(se->relevance, head);
+    cluster = reclist_insert(se->reclist, res, mergekey_norm, &se->total_merged);
+    if (global_parameters.dump_records)
+        yaz_log(YLOG_LOG, "Cluster id %d from %s (#%d)", cluster->recid,
+                cl->database->url, cl->records);
+    if (!cluster)
+    {
+        /* no room for record */
+        xmlFreeDoc(xdoc);
+        return 0;
+    }
+    relevance_newrec(se->relevance, cluster);
 
     for (n = root->children; n; n = n->next)
     {
-        if (n->type != XML_ELEMENT_NODE)
-            continue;
-        if (!strcmp(n->name, "facet"))
-        {
-            xmlChar *type = xmlGetProp(n, "type");
-            xmlChar *value = xmlNodeListGetString(xdoc, n->children, 0);
-            add_facet(se, type, value);
-            relevance_countwords(se->relevance, head, value, 1);
+        if (type)
             xmlFree(type);
+        if (value)
             xmlFree(value);
-        }
-        else if (!strcmp(n->name, "metadata"))
+        type = value = 0;
+
+        if (n->type != XML_ELEMENT_NODE)
+            continue;
+        if (!strcmp(n->name, "metadata"))
         {
-            xmlChar *type = xmlGetProp(n, "type"), *value;
-            if (!strcmp(type, "title"))
-                res->title = nmem_strdup(se->nmem,
-                        value = xmlNodeListGetString(xdoc, n->children, 0));
+            struct conf_metadata *md = 0;
+            struct conf_sortkey *sk = 0;
+            struct record_metadata **wheretoput, *newm;
+            int imeta;
+            int first, last;
+
+            type = xmlGetProp(n, "type");
+            value = xmlNodeListGetString(xdoc, n->children, 0);
+
+            if (!type || !value)
+                continue;
+
+            // First, find out what field we're looking at
+            for (imeta = 0; imeta < service->num_metadata; imeta++)
+                if (!strcmp(type, service->metadata[imeta].name))
+                {
+                    md = &service->metadata[imeta];
+                    if (md->sortkey_offset >= 0)
+                        sk = &service->sortkeys[md->sortkey_offset];
+                    break;
+                }
+            if (!md)
+            {
+                yaz_log(YLOG_WARN, "Ignoring unknown metadata element: %s", type);
+                continue;
+            }
+
+            // Find out where we are putting it
+            if (md->merge == Metadata_merge_no)
+                wheretoput = &res->metadata[imeta];
+            else
+                wheretoput = &cluster->metadata[imeta];
+            
+            // Put it there
+            newm = nmem_malloc(se->nmem, sizeof(struct record_metadata));
+            newm->next = 0;
+            if (md->type == Metadata_type_generic)
+            {
+                char *p, *pe;
+                for (p = value; *p && isspace(*p); p++)
+                    ;
+                for (pe = p + strlen(p) - 1;
+                        pe > p && strchr(" ,/.:([", *pe); pe--)
+                    *pe = '\0';
+                newm->data.text = nmem_strdup(se->nmem, p);
+
+            }
+            else if (md->type == Metadata_type_year)
+            {
+                if (extract_years(value, &first, &last) < 0)
+                    continue;
+            }
+            else
+            {
+                yaz_log(YLOG_WARN, "Unknown type in metadata element %s", type);
+                continue;
+            }
+            if (md->type == Metadata_type_year && md->merge != Metadata_merge_range)
+            {
+                yaz_log(YLOG_WARN, "Only range merging supported for years");
+                continue;
+            }
+            if (md->merge == Metadata_merge_unique)
+            {
+                struct record_metadata *mnode;
+                for (mnode = *wheretoput; mnode; mnode = mnode->next)
+                    if (!strcmp(mnode->data.text, newm->data.text))
+                        break;
+                if (!mnode)
+                {
+                    newm->next = *wheretoput;
+                    *wheretoput = newm;
+                }
+            }
+            else if (md->merge == Metadata_merge_longest)
+            {
+                if (!*wheretoput ||
+                        strlen(newm->data.text) > strlen((*wheretoput)->data.text))
+                {
+                    *wheretoput = newm;
+                    if (sk)
+                    {
+                        char *s = nmem_strdup(se->nmem, newm->data.text);
+                        if (!cluster->sortkeys[md->sortkey_offset])
+                            cluster->sortkeys[md->sortkey_offset] = 
+                                nmem_malloc(se->nmem, sizeof(union data_types));
+                        normalize_mergekey(s,
+                                (sk->type == Metadata_sortkey_skiparticle));
+                        cluster->sortkeys[md->sortkey_offset]->text = s;
+                    }
+                }
+            }
+            else if (md->merge == Metadata_merge_all || md->merge == Metadata_merge_no)
+            {
+                newm->next = *wheretoput;
+                *wheretoput = newm;
+            }
+            else if (md->merge == Metadata_merge_range)
+            {
+                assert(md->type == Metadata_type_year);
+                if (!*wheretoput)
+                {
+                    *wheretoput = newm;
+                    (*wheretoput)->data.number.min = first;
+                    (*wheretoput)->data.number.max = last;
+                    if (sk)
+                        cluster->sortkeys[md->sortkey_offset] = &newm->data;
+                }
+                else
+                {
+                    if (first < (*wheretoput)->data.number.min)
+                        (*wheretoput)->data.number.min = first;
+                    if (last > (*wheretoput)->data.number.max)
+                        (*wheretoput)->data.number.max = last;
+                }
+#ifdef GAGA
+                if (sk)
+                {
+                    union data_types *sdata = cluster->sortkeys[md->sortkey_offset];
+                    yaz_log(YLOG_LOG, "SK range: %d-%d", sdata->number.min, sdata->number.max);
+                }
+#endif
+            }
+            else
+                yaz_log(YLOG_WARN, "Don't know how to merge on element name %s", md->name);
 
-            relevance_countwords(se->relevance, head, value, 4);
+            if (md->rank)
+                relevance_countwords(se->relevance, cluster, value, md->rank);
+            if (md->termlist)
+            {
+                if (md->type == Metadata_type_year)
+                {
+                    char year[64];
+                    sprintf(year, "%d", last);
+                    add_facet(se, type, year);
+                    if (first != last)
+                    {
+                        sprintf(year, "%d", first);
+                        add_facet(se, type, year);
+                    }
+                }
+                else
+                    add_facet(se, type, value);
+            }
             xmlFree(type);
             xmlFree(value);
+            type = value = 0;
         }
         else
             yaz_log(YLOG_WARN, "Unexpected element %s in internal record", n->name);
     }
+    if (type)
+        xmlFree(type);
+    if (value)
+        xmlFree(value);
 
     xmlFreeDoc(xdoc);
 
-    relevance_donerecord(se->relevance, head);
+    relevance_donerecord(se->relevance, cluster);
     se->total_records++;
 
     return res;
@@ -500,6 +732,7 @@ static void ingest_records(struct client *cl, Z_Records *r)
     {
         Z_NamePlusRecord *npr = rlist->records[i];
 
+        cl->records++;
         if (npr->which != Z_NamePlusRecord_databaseRecord)
         {
             yaz_log(YLOG_WARN, "Unexpected record type, probably diagnostic");
@@ -541,7 +774,6 @@ static void do_presentResponse(IOCHAN i, Z_APDU *a)
     if (!*r->presentStatus && cl->state != Client_Error)
     {
         yaz_log(YLOG_DEBUG, "Good Present response");
-        cl->records += *r->numberOfRecordsReturned;
         ingest_records(cl, r->records);
         cl->state = Client_Idle;
     }
@@ -723,7 +955,8 @@ static struct connection *connection_create(struct client *cl)
 
     if (!(addr = cs_straddr(link, cl->database->host->ipport)))
     {
-        yaz_log(YLOG_WARN|YLOG_ERRNO, "Lookup of IP address failed?");
+        yaz_log(YLOG_WARN|YLOG_ERRNO, "Lookup of IP address %s failed?", 
+	    cl->database->host->ipport);
         return 0;
     }
 
@@ -827,19 +1060,24 @@ void load_simpletargets(const char *fn)
     while (fgets(line, 255, f))
     {
         char *url, *db;
+        char *name;
         struct host *host;
         struct database *database;
 
         if (strncmp(line, "target ", 7))
             continue;
+        line[strlen(line) - 1] = '\0';
+
+        if ((name = strchr(line, ';')))
+            *(name++) = '\0';
+
         url = line + 7;
-        url[strlen(url) - 1] = '\0';
-        yaz_log(YLOG_DEBUG, "Target: %s", url);
         if ((db = strchr(url, '/')))
             *(db++) = '\0';
         else
             db = "Default";
 
+        yaz_log(YLOG_LOG, "Target: %s, '%s'", url, db);
         for (host = hosts; host; host = host->next)
             if (!strcmp(url, host->hostport))
                 break;
@@ -879,7 +1117,7 @@ void load_simpletargets(const char *fn)
             }
             assert(addrinfo->ai_family == PF_INET);
             memcpy(addrbuf, &((struct sockaddr_in*)addrinfo->ai_addr)->sin_addr.s_addr, 4);
-            sprintf(ipport, "%hhd.%hhd.%hhd.%hhd:%s",
+            sprintf(ipport, "%u.%u.%u.%u:%s",
                     addrbuf[0], addrbuf[1], addrbuf[2], addrbuf[3], port);
             host->ipport = xstrdup(ipport);
             freeaddrinfo(addrinfo);
@@ -892,6 +1130,10 @@ void load_simpletargets(const char *fn)
         strcpy(database->url, url);
         strcat(database->url, "/");
         strcat(database->url, db);
+        if (name)
+            database->name = xstrdup(name);
+        else
+            database->name = 0;
         
         database->databases = xmalloc(2 * sizeof(char *));
         database->databases[0] = xstrdup(db);
@@ -1076,7 +1318,7 @@ char *search(struct session *se, char *query)
         se->reclist = reclist_create(se->nmem, maxrecs);
         extract_terms(se->nmem, query, p);
         se->relevance = relevance_create(se->nmem, (const char **) p, maxrecs);
-        se->total_records = se->total_hits = 0;
+        se->total_records = se->total_hits = se->total_merged = 0;
         se->expected_maxrecs = maxrecs;
     }
     else
@@ -1130,7 +1372,8 @@ struct hitsbytarget *hitsbytarget(struct session *se, int *count)
     *count = 0;
     for (cl = se->clients; cl; cl = cl->next)
     {
-        strcpy(res[*count].id, cl->database->host->hostport);
+        res[*count].id = cl->database->url;
+        res[*count].name = cl->database->name;
         res[*count].hits = cl->hits;
         res[*count].records = cl->records;
         res[*count].diagnostic = cl->diagnostic;
@@ -1152,9 +1395,7 @@ struct termlist_score **termlist(struct session *s, const char *name, int *num)
     return 0;
 }
 
-#ifdef REPORT_NMEM
-// conditional compilation by SH: This lead to a warning with currently installed
-// YAZ header files on us1
+#ifdef MISSING_HEADERS
 void report_nmem_stats(void)
 {
     size_t in_use, is_free;
@@ -1167,16 +1408,35 @@ void report_nmem_stats(void)
 }
 #endif
 
-struct record **show(struct session *s, int start, int *num, int *total,
-                     int *sumhits, NMEM nmem_show)
+struct record_cluster *show_single(struct session *s, int id)
+{
+    struct record_cluster *r;
+
+    reclist_rewind(s->reclist);
+    while ((r = reclist_read_record(s->reclist)))
+        if (r->recid == id)
+            return r;
+    return 0;
+}
+
+struct record_cluster **show(struct session *s, struct reclist_sortparms *sp, int start,
+        int *num, int *total, int *sumhits, NMEM nmem_show)
 {
-    struct record **recs = nmem_malloc(nmem_show, *num 
-                                       * sizeof(struct record *));
+    struct record_cluster **recs = nmem_malloc(nmem_show, *num 
+                                       * sizeof(struct record_cluster *));
+    struct reclist_sortparms *spp;
     int i;
 #if USE_TIMING    
     yaz_timing_t t = yaz_timing_create();
 #endif
-    relevance_prepare_read(s->relevance, s->reclist);
+
+    for (spp = sp; spp; spp = spp->next)
+        if (spp->type == Metadata_sortkey_relevance)
+        {
+            relevance_prepare_read(s->relevance, s->reclist);
+            break;
+        }
+    reclist_sort(s->reclist, sp);
 
     *total = s->reclist->num_records;
     *sumhits = s->total_hits;
@@ -1191,7 +1451,7 @@ struct record **show(struct session *s, int start, int *num, int *total,
 
     for (i = 0; i < *num; i++)
     {
-        struct record *r = reclist_read_record(s->reclist);
+        struct record_cluster *r = reclist_read_record(s->reclist);
         if (!r)
         {
             *num = i;
@@ -1214,7 +1474,7 @@ void statistics(struct session *se, struct statistics *stat)
     struct client *cl;
     int count = 0;
 
-    bzero(stat, sizeof(*stat));
+    memset(stat, 0, sizeof(*stat));
     for (cl = se->clients; cl; cl = cl->next)
     {
         if (!cl->connection)
@@ -1249,37 +1509,74 @@ static CCL_bibset load_cclfile(const char *fn)
     return res;
 }
 
+static void start_http_listener(void)
+{
+    char hp[128] = "";
+    struct conf_server *ser = global_parameters.server;
+
+    if (*global_parameters.listener_override)
+        strcpy(hp, global_parameters.listener_override);
+    else
+    {
+        strcpy(hp, ser->host ? ser->host : "");
+        if (ser->port)
+        {
+            if (*hp)
+                strcat(hp, ":");
+            sprintf(hp + strlen(hp), "%d", ser->port);
+        }
+    }
+    http_init(hp);
+}
+
+static void start_proxy(void)
+{
+    char hp[128] = "";
+    struct conf_server *ser = global_parameters.server;
+
+    if (*global_parameters.proxy_override)
+        strcpy(hp, global_parameters.proxy_override);
+    else if (ser->proxy_host || ser->proxy_port)
+    {
+        strcpy(hp, ser->proxy_host ? ser->proxy_host : "");
+        if (ser->proxy_port)
+        {
+            if (*hp)
+                strcat(hp, ":");
+            sprintf(hp + strlen(hp), "%d", ser->proxy_port);
+        }
+    }
+    else
+        return;
+
+    http_set_proxyaddr(hp);
+}
+
 int main(int argc, char **argv)
 {
     int ret;
     char *arg;
-    int setport = 0;
 
-    if (signal(SIGPIPE, SIG_IGN) < 0)
+    if (signal(SIGPIPE, SIG_IGN) == SIG_ERR)
         yaz_log(YLOG_WARN|YLOG_ERRNO, "signal");
 
     yaz_log_init(YLOG_DEFAULT_LEVEL, "pazpar2", 0);
 
-    while ((ret = options("f:x:c:h:p:C:s:d", argv, argc, &arg)) != -2)
+    while ((ret = options("f:x:h:p:C:s:d", argv, argc, &arg)) != -2)
     {
 	switch (ret) {
             case 'f':
                 if (!read_config(arg))
                     exit(1);
                 break;
-	    case 'c':
-		command_init(atoi(arg));
-                setport++;
-		break;
             case 'h':
-                http_init(arg);
-                setport++;
+                strcpy(global_parameters.listener_override, arg);
                 break;
             case 'C':
                 global_parameters.ccl_filter = load_cclfile(arg);
                 break;
             case 'p':
-                http_set_proxyaddr(arg);
+                strcpy(global_parameters.proxy_override, arg);
                 break;
             case 's':
                 load_simpletargets(arg);
@@ -1291,20 +1588,23 @@ int main(int argc, char **argv)
 		fprintf(stderr, "Usage: pazpar2\n"
                         "    -f configfile\n"
                         "    -h [host:]port          (REST protocol listener)\n"
-                        "    -c cmdport              (telnet-style)\n"
                         "    -C cclconfig\n"
                         "    -s simpletargetfile\n"
-                        "    -p hostname[:portno]    (HTTP proxy)\n");
+                        "    -p hostname[:portno]    (HTTP proxy)\n"
+                        "    -d                      (show internal records)\n");
 		exit(1);
 	}
     }
 
-    if (!setport)
+    if (!config)
     {
-        fprintf(stderr, "Set command port with -h or -c\n");
+        yaz_log(YLOG_FATAL, "Load config with -f");
         exit(1);
     }
+    global_parameters.server = config->servers;
 
+    start_http_listener();
+    start_proxy();
     global_parameters.ccl_filter = load_cclfile("../etc/default.bib");
     global_parameters.yaz_marc = yaz_marc_create();
     yaz_marc_subfield_str(global_parameters.yaz_marc, "\t");