Pass /manifest option always YAZ-804
[yaz-moved-to-github.git] / ztest / ztest.c
index a860296..cc7736d 100644 (file)
@@ -1,24 +1,39 @@
 /* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2010 Index Data
+ * Copyright (C) Index Data
  * See the file LICENSE for details.
  */
-
 /** \file
  * \brief yaz-ztest Generic Frontend Server
  */
+#if HAVE_CONFIG_H
+#include <config.h>
+#endif
+
 
 #include <stdio.h>
 #include <math.h>
 #include <stdlib.h>
-#include <ctype.h>
+
+#if HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
 #if HAVE_UNISTD_H
 #include <unistd.h>
 #endif
+#if HAVE_SYS_SELECT_H
+#include <sys/select.h>
+#endif
+#ifdef WIN32
+#include <windows.h>
+#endif
 
 #include <yaz/log.h>
 #include <yaz/backend.h>
 #include <yaz/ill.h>
 #include <yaz/diagbib1.h>
+#include <yaz/otherinfo.h>
+#include <yaz/facet.h>
+#include <yaz/backtrace.h>
 
 #include "ztest.h"
 
@@ -73,15 +88,16 @@ static void remove_sets(struct session_handle *sh)
     sh->result_sets = 0;
 }
 
-/** \brief use term value as hit count 
+/** \brief use term value as hit count
     \param s RPN structure
+    \param hash value for compuation
     \return >= 0: search term number or -1: not found
-   
+
     Traverse RPN tree 'in order' and use term value as hit count.
     Only terms  that looks a numeric is used.. Returns -1 if
     no sub tree has a hit count term
 */
-static Odr_int get_term_hit(Z_RPNStructure *s)
+static Odr_int get_term_hit(Z_RPNStructure *s, unsigned *hash)
 {
     Odr_int h = -1;
     switch(s->which)
@@ -100,13 +116,19 @@ static Odr_int get_term_hit(Z_RPNStructure *s)
                     h = odr_atoi(wrbuf_cstr(hits_str));
                     wrbuf_destroy(hits_str);
                 }
+                else
+                {
+                    int i;
+                    for (i = 0; i < oct->len; i++)
+                        *hash = *hash * 65509 + oct->buf[i];
+                }
             }
         }
         break;
     case Z_RPNStructure_complex:
-        h = get_term_hit(s->u.complex->s1);
+        h = get_term_hit(s->u.complex->s1, hash);
         if (h == -1)
-            h = get_term_hit(s->u.complex->s2);
+            h = get_term_hit(s->u.complex->s2, hash);
         break;
     }
     return h;
@@ -114,8 +136,8 @@ static Odr_int get_term_hit(Z_RPNStructure *s)
 
 /** \brief gets hit count for numeric terms in RPN queries
     \param q RPN Query
-    \return number of hits (random or number for term)
-    
+    \return number of hits
+
     This is just for testing.. A real database of course uses
     the content of a database to establish a value.. In our case, we
     have a way to trigger a certain hit count. Good for testing of
@@ -123,12 +145,27 @@ static Odr_int get_term_hit(Z_RPNStructure *s)
 */
 static Odr_int get_hit_count(Z_Query *q)
 {
-    Odr_int h = -1;
     if (q->which == Z_Query_type_1 || q->which == Z_Query_type_101)
-        h = get_term_hit(q->u.type_1->RPNStructure);
-    if (h == -1)
-        h = rand() % 24;
-    return h;
+    {
+        unsigned hash = 0;
+        Odr_int h = -1;
+        h = get_term_hit(q->u.type_1->RPNStructure, &hash);
+        if (h == -1)
+            h = hash % 24;
+        return h;
+    }
+    else if (q->which == Z_Query_type_104 &&
+             q->u.type_104->which == Z_External_CQL)
+    {
+        unsigned hash = 0;
+        const char *cql = q->u.type_104->u.cql;
+        int i;
+        for (i = 0; cql[i]; i++)
+            hash = hash * 65509 + cql[i];
+        return hash % 24;
+    }
+    else
+        return 24;
 }
 
 /** \brief checks if it's a dummy Slow database
@@ -179,18 +216,131 @@ static void init_delay(struct delay *delayp)
 
 static int parse_delay(struct delay *delayp, const char *value)
 {
-    delayp->d1 = atof(value);
-    delayp->d2 = 0.0;
+    if (sscanf(value, "%lf:%lf", &delayp->d1, &delayp->d2) == 2)
+        ;
+    else if (sscanf(value, "%lf", &delayp->d1) == 1)
+        delayp->d2 = 0.0;
+    else
+        return -1;
     return 0;
 }
 
-static void do_delay(const struct delay *delayp)
+static void ztest_sleep(double d)
 {
+#ifdef WIN32
+    Sleep( (DWORD) (d * 1000));
+#else
     struct timeval tv;
-
-    tv.tv_sec = floor(delayp->d1);
-    tv.tv_usec = (delayp->d1 - floor(delayp->d1)) * 1000000;
+    tv.tv_sec = d;
+    tv.tv_usec = (d - (long) d) * 1000000;
     select(0, 0, 0, 0, &tv);
+#endif
+}
+
+static void do_delay(const struct delay *delayp)
+{
+    double d = delayp->d1;
+
+    if (d > 0.0)
+    {
+        if (delayp->d2 > d)
+            d += (rand()) * (delayp->d2 - d) / RAND_MAX;
+        ztest_sleep(d);
+    }
+}
+
+static void addterms(ODR odr, Z_FacetField *facet_field, const char *facet_name)
+{
+    int index;
+    int freq = 100;
+    int length = strlen(facet_name) + 10;
+    char *key = odr_malloc(odr, length);
+    key[0] = '\0';
+    for (index = 0; index < facet_field->num_terms; index++)
+    {
+        Z_FacetTerm *facet_term;
+        sprintf(key, "%s%d", facet_name, index);
+        yaz_log(YLOG_DEBUG, "facet add term %s %d %s", facet_name, index, key);
+
+        facet_term = facet_term_create_cstr(odr, key, freq);
+        freq = freq - 10 ;
+        facet_field_term_set(odr, facet_field, facet_term, index);
+    }
+}
+
+Z_OtherInformation *build_facet_response(ODR odr, Z_FacetList *facet_list) {
+    int index, new_index = 0;
+    Z_FacetList *new_list = facet_list_create(odr, facet_list->num);
+
+    for (index = 0; index < facet_list->num; index++) {
+        struct yaz_facet_attr attrvalues;
+        yaz_facet_attr_init(&attrvalues);
+        attrvalues.limit = 10;
+        yaz_facet_attr_get_z_attributes(facet_list->elements[index]->attributes,
+                                        &attrvalues);
+        yaz_log(YLOG_LOG, "Attributes: %s limit=%d start=%d sort=%d",
+                attrvalues.useattr ? attrvalues.useattr : "NONE",
+                attrvalues.limit,
+                attrvalues.start,
+                attrvalues.sortorder);
+        if (attrvalues.errstring)
+            yaz_log(YLOG_LOG, "Error parsing attributes: %s", attrvalues.errstring);
+        if (attrvalues.limit > 0 && attrvalues.useattr) {
+            new_list->elements[new_index] = facet_field_create(odr, facet_list->elements[index]->attributes, attrvalues.limit);
+            addterms(odr, new_list->elements[new_index], attrvalues.useattr);
+            new_index++;
+        }
+        else {
+            yaz_log(YLOG_DEBUG, "Facet: skipping %s due to 0 limit.", attrvalues.useattr);
+        }
+
+    }
+    new_list->num = new_index;
+    if (new_index > 0) {
+        Z_OtherInformation *oi = odr_malloc(odr, sizeof(*oi));
+        Z_OtherInformationUnit *oiu = odr_malloc(odr, sizeof(*oiu));
+        oi->num_elements = 1;
+        oi->list = odr_malloc(odr, oi->num_elements * sizeof(*oi->list));
+        oiu->category = 0;
+        oiu->which = Z_OtherInfo_externallyDefinedInfo;
+        oiu->information.externallyDefinedInfo = odr_malloc(odr, sizeof(*oiu->information.externallyDefinedInfo));
+        oiu->information.externallyDefinedInfo->direct_reference = odr_oiddup(odr, yaz_oid_userinfo_facet_1);
+        oiu->information.externallyDefinedInfo->descriptor = 0;
+        oiu->information.externallyDefinedInfo->indirect_reference = 0;
+        oiu->information.externallyDefinedInfo->which = Z_External_userFacets;
+        oiu->information.externallyDefinedInfo->u.facetList = new_list;
+        oi->list[0] = oiu;
+        return oi;
+    }
+    return 0;
+}
+
+static void echo_extra_args(ODR stream,
+                            Z_SRW_extra_arg *extra_args, char **extra_response)
+{
+    if (extra_args)
+    {
+        Z_SRW_extra_arg *a;
+        WRBUF response_xml = wrbuf_alloc();
+        wrbuf_puts(response_xml, "<extra>");
+        for (a = extra_args; a; a = a->next)
+        {
+            wrbuf_puts(response_xml, "<extra name=\"");
+            wrbuf_xmlputs(response_xml, a->name);
+            wrbuf_puts(response_xml, "\"");
+            if (a->value)
+            {
+                wrbuf_puts(response_xml, " value=\"");
+                wrbuf_xmlputs(response_xml, a->value);
+                wrbuf_puts(response_xml, "\"");
+            }
+            wrbuf_puts(response_xml, "/>");
+        }
+        wrbuf_puts(response_xml, "</extra>");
+        *extra_response = odr_strdup(stream, wrbuf_cstr(response_xml));
+        wrbuf_destroy(response_xml);
+    }
+
 }
 
 int ztest_search(void *handle, bend_search_rr *rr)
@@ -204,7 +354,7 @@ int ztest_search(void *handle, bend_search_rr *rr)
         rr->errcode = YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP;
         return 0;
     }
-    
+
     db = rr->basenames[0];
 
     /* Allow Default, db.* and Slow */
@@ -223,7 +373,7 @@ int ztest_search(void *handle, bend_search_rr *rr)
         return 0;
     }
 
-    new_set = get_set(sh, rr->setname);    
+    new_set = get_set(sh, rr->setname);
     if (new_set)
     {
         if (!rr->replace_set)
@@ -244,6 +394,7 @@ int ztest_search(void *handle, bend_search_rr *rr)
     new_set->db = xstrdup(db);
     init_delay(&new_set->search_delay);
     init_delay(&new_set->present_delay);
+    init_delay(&new_set->fetch_delay);
 
     db_sep = strchr(db, '?');
     if (db_sep)
@@ -257,7 +408,7 @@ int ztest_search(void *handle, bend_search_rr *rr)
             const char *name = names[i];
             const char *value = values[i];
             if (!strcmp(name, "seed"))
-                srandom(atoi(value));
+                srand(atoi(value));
             else if (!strcmp(name, "search-delay"))
                 parse_delay(&new_set->search_delay, value);
             else if (!strcmp(name, "present-delay"))
@@ -272,34 +423,23 @@ int ztest_search(void *handle, bend_search_rr *rr)
         }
     }
 
-    if (rr->extra_args)
+    echo_extra_args(rr->stream, rr->extra_args, &rr->extra_response_data);
+    rr->hits = get_hit_count(rr->query);
+
+    if (1)
     {
-        Z_SRW_extra_arg *a;
-        WRBUF response_xml = wrbuf_alloc();
-        wrbuf_puts(response_xml, "<extra>");
-        for (a = rr->extra_args; a; a = a->next)
-        {
-            wrbuf_puts(response_xml, "<extra name=\"");
-            wrbuf_xmlputs(response_xml, a->name);
-            wrbuf_puts(response_xml, "\"");
-            if (a->value)
-            {
-                wrbuf_puts(response_xml, " value=\"");
-                wrbuf_xmlputs(response_xml, a->value);
-                wrbuf_puts(response_xml, "\"");
-            }
-            wrbuf_puts(response_xml, "/>");
+        Z_FacetList *facet_list = yaz_oi_get_facetlist(&rr->search_input);
+        if (facet_list) {
+            yaz_log(YLOG_LOG, "%d Facets in search request.", facet_list->num);
+            rr->search_info = build_facet_response(rr->stream, facet_list);
         }
-        wrbuf_puts(response_xml, "</extra>");
-        rr->extra_response_data =
-            odr_strdup(rr->stream, wrbuf_cstr(response_xml));
-        wrbuf_destroy(response_xml);
-    }
-    rr->hits = get_hit_count(rr->query);
+        else
+            yaz_log(YLOG_DEBUG, "No facets parsed search request.");
 
+    }
     do_delay(&new_set->search_delay);
     new_set->hits = rr->hits;
-    
+
     return 0;
 }
 
@@ -307,13 +447,6 @@ int ztest_search(void *handle, bend_search_rr *rr)
 /* this huge function handles extended services */
 int ztest_esrequest(void *handle, bend_esrequest_rr *rr)
 {
-    /* user-defined handle - created in bend_init */
-    int *counter = (int*) handle;  
-
-    yaz_log(log_level, "ESRequest no %d", *counter);
-
-    (*counter)++;
-
     if (rr->esr->packageName)
         yaz_log(log_level, "packagename: %s", rr->esr->packageName);
     yaz_log(log_level, "Waitaction: " ODR_INT_PRINTF, *rr->esr->waitAction);
@@ -335,7 +468,7 @@ int ztest_esrequest(void *handle, bend_esrequest_rr *rr)
             Z_IOOriginPartToKeep *k = ir->toKeep;
             Z_IOOriginPartNotToKeep *n = ir->notToKeep;
             const char *xml_in_response = 0;
-            
+
             if (k && k->contact)
             {
                 if (k->contact->name)
@@ -349,7 +482,7 @@ int ztest_esrequest(void *handle, bend_esrequest_rr *rr)
             {
                 yaz_log(log_level, "Billing info (not shown)");
             }
-            
+
             if (n->resultSetItem)
             {
                 yaz_log(log_level, "resultsetItem");
@@ -365,7 +498,7 @@ int ztest_esrequest(void *handle, bend_esrequest_rr *rr)
                 {
                     char oid_name_str[OID_STR_MAX];
                     oid_class oclass;
-                    const char *oid_name = 
+                    const char *oid_name =
                         yaz_oid_to_string_buf(r->direct_reference,
                                               &oclass, oid_name_str);
                     if (oid_name)
@@ -376,10 +509,10 @@ int ztest_esrequest(void *handle, bend_esrequest_rr *rr)
                         if (r->which == Z_External_octet)
                             yaz_log(log_level, "%.*s",
                                     r->u.octet_aligned->len,
-                                    r->u.octet_aligned->buf); 
+                                    r->u.octet_aligned->buf);
                         xml_in_response = "<dummy>x</dummy>";
                     }
-                    if (!oid_oidcmp(r->direct_reference, 
+                    if (!oid_oidcmp(r->direct_reference,
                                     yaz_oid_general_isoill_1))
                     {
                         yaz_log(log_level, "Decode ItemRequest begin");
@@ -388,7 +521,7 @@ int ztest_esrequest(void *handle, bend_esrequest_rr *rr)
                             odr_setbuf(rr->decode,
                                        (char *) r->u.single_ASN1_type->buf,
                                        r->u.single_ASN1_type->len, 0);
-                            
+
                             if (!ill_ItemRequest(rr->decode, &item_req, 0, 0))
                             {
                                 yaz_log(log_level,
@@ -411,7 +544,7 @@ int ztest_esrequest(void *handle, bend_esrequest_rr *rr)
                             odr_setbuf(rr->decode,
                                        (char*) r->u.single_ASN1_type->buf,
                                        r->u.single_ASN1_type->len, 0);
-                            
+
                             if (!ill_APDU(rr->decode, &ill_apdu, 0, 0))
                             {
                                 yaz_log(log_level,
@@ -460,13 +593,8 @@ int ztest_esrequest(void *handle, bend_esrequest_rr *rr)
                 rr->taskPackage->retentionTime = 0;
                 rr->taskPackage->permissions = 0;
                 rr->taskPackage->description = 0;
-                rr->taskPackage->targetReference = (Odr_oct *)
-                    odr_malloc(rr->stream, sizeof(Odr_oct));
-                rr->taskPackage->targetReference->buf =
-                    (unsigned char *) odr_strdup(rr->stream, "911");
-                rr->taskPackage->targetReference->len =
-                    rr->taskPackage->targetReference->size =
-                    strlen((char *) (rr->taskPackage->targetReference->buf));
+                rr->taskPackage->targetReference =
+                    odr_create_Odr_oct(rr->stream, "911", 3);
                 rr->taskPackage->creationDateTime = 0;
                 rr->taskPackage->taskStatus = odr_intdup(rr->stream, 0);
                 rr->taskPackage->packageDiagnostics = 0;
@@ -491,7 +619,7 @@ int ztest_esrequest(void *handle, bend_esrequest_rr *rr)
                                          strlen(xml_in_response));
                 else
                     targetPart->itemRequest = 0;
-                    
+
                 targetPart->statusOrErrorReport = 0;
                 targetPart->auxiliaryStatus = 0;
             }
@@ -506,7 +634,7 @@ int ztest_esrequest(void *handle, bend_esrequest_rr *rr)
             Z_IUUpdateEsRequest *esRequest = up->u.esRequest;
             Z_IUOriginPartToKeep *toKeep = esRequest->toKeep;
             Z_IUSuppliedRecords *notToKeep = esRequest->notToKeep;
-            
+
             yaz_log(log_level, "action");
             if (toKeep->action)
             {
@@ -561,13 +689,8 @@ int ztest_esrequest(void *handle, bend_esrequest_rr *rr)
                 rr->taskPackage->retentionTime = 0;
                 rr->taskPackage->permissions = 0;
                 rr->taskPackage->description = 0;
-                rr->taskPackage->targetReference = (Odr_oct *)
-                    odr_malloc(rr->stream, sizeof(Odr_oct));
-                rr->taskPackage->targetReference->buf =
-                    (unsigned char *) odr_strdup(rr->stream, "123");
-                rr->taskPackage->targetReference->len =
-                    rr->taskPackage->targetReference->size =
-                    strlen((char *) (rr->taskPackage->targetReference->buf));
+                rr->taskPackage->targetReference =
+                    odr_create_Odr_oct(rr->stream, "123", 3);
                 rr->taskPackage->creationDateTime = 0;
                 rr->taskPackage->taskStatus = odr_intdup(rr->stream, 0);
                 rr->taskPackage->packageDiagnostics = 0;
@@ -597,7 +720,7 @@ int ztest_esrequest(void *handle, bend_esrequest_rr *rr)
                 targetPart->num_globalDiagnostics = 0;
                 targetPart->globalDiagnostics = (Z_DiagRec **) odr_nullval();
                 targetPart->num_taskPackageRecords = 1;
-                targetPart->taskPackageRecords = 
+                targetPart->taskPackageRecords =
                     (Z_IUTaskPackageRecordStructure **)
                     odr_malloc(rr->stream,
                                sizeof(Z_IUTaskPackageRecordStructure *));
@@ -605,15 +728,15 @@ int ztest_esrequest(void *handle, bend_esrequest_rr *rr)
                     (Z_IUTaskPackageRecordStructure *)
                     odr_malloc(rr->stream,
                                sizeof(Z_IUTaskPackageRecordStructure));
-                
+
                 targetPart->taskPackageRecords[0]->which =
                     Z_IUTaskPackageRecordStructure_record;
-                targetPart->taskPackageRecords[0]->u.record = 
+                targetPart->taskPackageRecords[0]->u.record =
                     z_ext_record_sutrs(rr->stream, "test", 4);
-                targetPart->taskPackageRecords[0]->correlationInfo = 0; 
+                targetPart->taskPackageRecords[0]->correlationInfo = 0;
                 targetPart->taskPackageRecords[0]->recordStatus =
                     odr_intdup(rr->stream,
-                               Z_IUTaskPackageRecordStructure_success);  
+                               Z_IUTaskPackageRecordStructure_success);
                 targetPart->taskPackageRecords[0]->num_supplementalDiagnostics
                     = 0;
 
@@ -629,7 +752,7 @@ int ztest_esrequest(void *handle, bend_esrequest_rr *rr)
                     if (rec->direct_reference)
                     {
                         char oid_name_str[OID_STR_MAX];
-                        const char *oid_name 
+                        const char *oid_name
                             = oid_name = yaz_oid_to_string_buf(
                                 rec->direct_reference, 0,
                                 oid_name_str);
@@ -688,6 +811,16 @@ int ztest_sort(void *handle, bend_sort_rr *rr)
 /* present request handler */
 int ztest_present(void *handle, bend_present_rr *rr)
 {
+    struct session_handle *sh = (struct session_handle*) handle;
+    struct result_set *set = get_set(sh, rr->setname);
+
+    if (!set)
+    {
+        rr->errcode = YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST;
+        rr->errstring = odr_strdup(rr->stream, rr->setname);
+        return 0;
+    }
+    do_delay(&set->present_delay);
     return 0;
 }
 
@@ -697,7 +830,8 @@ int ztest_fetch(void *handle, bend_fetch_rr *r)
     struct session_handle *sh = (struct session_handle*) handle;
     char *cp;
     const Odr_oid *oid = r->request_format;
-    struct result_set *set = get_set(sh, r->setname);    
+    struct result_set *set = get_set(sh, r->setname);
+    const char *esn = yaz_get_esn(r->comp);
 
     if (!set)
     {
@@ -705,6 +839,7 @@ int ztest_fetch(void *handle, bend_fetch_rr *r)
         r->errstring = odr_strdup(r->stream, r->setname);
         return 0;
     }
+    do_delay(&set->fetch_delay);
     r->last_in_set = 0;
     r->basename = set->db;
     r->output_format = r->request_format;
@@ -746,7 +881,7 @@ int ztest_fetch(void *handle, bend_fetch_rr *r)
     {
         /* this section returns a small record */
         char buf[100];
-        
+
         sprintf(buf, "This is dummy SUTRS record number %d\n", r->number);
 
         r->len = strlen(buf);
@@ -797,12 +932,28 @@ int ztest_fetch(void *handle, bend_fetch_rr *r)
     }
     else if (!oid_oidcmp(oid, yaz_oid_recsyn_xml))
     {
-        if ((cp = dummy_xml_record(r->number, r->stream)))
+        if ((cp = dummy_xml_record(r->number, r->stream, esn)))
         {
             r->len = strlen(cp);
             r->record = cp;
+            r->schema = "info:srw/schema/1/marcxml-1.1";
         }
-        else 
+        else
+        {
+            r->errcode = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
+            r->surrogate_flag = 1;
+            return 0;
+        }
+    }
+    else if (!oid_oidcmp(oid, yaz_oid_recsyn_json))
+    {
+        if ((cp = dummy_json_record(r->number, r->stream, esn)))
+        {
+            r->len = strlen(cp);
+            r->record = cp;
+            r->schema = "info:srw/schema/1/marcxml-1.1";
+        }
+        else
         {
             r->errcode = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
             r->surrogate_flag = 1;
@@ -888,8 +1039,8 @@ int ztest_scan(void *handle, bend_scan_rr *q)
         strcpy(term, "0");
 
     for (p = term; *p; p++)
-        if (islower(*(unsigned char *) p))
-            *p = toupper(*p);
+        if (yaz_islower(*p))
+            *p = yaz_toupper(*p);
 
     fseek(f, 0, SEEK_SET);
     q->num_entries = 0;
@@ -936,6 +1087,7 @@ int ztest_scan(void *handle, bend_scan_rr *q)
         if (q->num_entries >= num_entries_req)
             break;
     }
+    echo_extra_args(q->stream, q->extra_args, &q->extra_response_data);
     if (feof(f))
         q->status = BEND_SCAN_PARTIAL;
     return 0;
@@ -1007,6 +1159,8 @@ void bend_close(void *handle)
 
 int main(int argc, char **argv)
 {
+    yaz_enable_panic_backtrace(argv[0]);
+
     return statserv_main(argc, argv, bend_init, bend_close);
 }
 /*