I think this now supports CCL in scan, but I've not tested it.
[yaz-moved-to-github.git] / src / zoom-c.c
index 3e5c65e..05371be 100644 (file)
@@ -2,7 +2,7 @@
  * Copyright (C) 1995-2005, Index Data ApS
  * See the file LICENSE for details.
  *
- * $Id: zoom-c.c,v 1.48 2005-11-01 15:08:02 adam Exp $
+ * $Id: zoom-c.c,v 1.76 2006-06-13 16:27:23 mike Exp $
  */
 /**
  * \file zoom-c.c
@@ -11,6 +11,7 @@
 
 #include <assert.h>
 #include <string.h>
+#include <errno.h>
 #include "zoom-p.h"
 
 #include <yaz/yaz-util.h>
@@ -23,6 +24,8 @@
 #include <yaz/charneg.h>
 #include <yaz/ill.h>
 #include <yaz/srw.h>
+#include <yaz/cql.h>
+#include <yaz/ccl.h>
 
 #if HAVE_SYS_TYPES_H
 #include <sys/types.h>
@@ -50,6 +53,8 @@ typedef enum {
 
 static zoom_ret ZOOM_connection_send_init (ZOOM_connection c);
 static zoom_ret do_write_ex (ZOOM_connection c, char *buf_out, int len_out);
+static char *cql2pqf(ZOOM_connection c, const char *cql);
+static char *ccl2pqf(ZOOM_connection c, const char *ccl);
 
 static void initlog()
 {
@@ -163,7 +168,13 @@ static void set_ZOOM_error (ZOOM_connection c, int error,
 
 static void clear_error (ZOOM_connection c)
 {
-
+    /*
+     * If an error is tied to an operation then it's ok to clear: for
+     * example, a diagnostic returned from a search is cleared by a
+     * subsequent search.  However, problems such as Connection Lost
+     * or Init Refused are not cleared, because they are not
+     * recoverable: doing another search doesn't help.
+     */
     switch (c->error)
     {
     case ZOOM_ERROR_CONNECT:
@@ -236,6 +247,12 @@ void ZOOM_connection_remove_task (ZOOM_connection c)
             assert (0);
         }
         xfree (task);
+
+        if (!c->tasks)
+        {
+            ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
+            ZOOM_connection_put_event(c, event);
+        }
     }
 }
 
@@ -305,7 +322,7 @@ static char **set_DatabaseNames (ZOOM_connection con, ZOOM_options options,
     char **databaseNames;
     const char *cp = ZOOM_options_get (options, "databaseName");
     
-    if (!cp || !*cp)
+    if ((!cp || !*cp) && con->host_port)
     {
         if (strncmp (con->host_port, "unix:", 5) == 0)
             cp = strchr(con->host_port+5, ':');
@@ -329,6 +346,19 @@ ZOOM_connection_new (const char *host, int portnum)
     return c;
 }
 
+static zoom_sru_mode get_sru_mode_from_string(const char *s)
+{
+    if (!s || !*s)
+        return zoom_sru_soap;
+    if (!yaz_matchstr(s, "soap"))
+        return zoom_sru_soap;
+    else if (!yaz_matchstr(s, "get"))
+        return zoom_sru_get;
+    else if (!yaz_matchstr(s, "post"))
+        return zoom_sru_post;
+    return zoom_sru_error;
+}
+
 ZOOM_API(void)
 ZOOM_connection_connect(ZOOM_connection c,
                         const char *host, int portnum)
@@ -378,6 +408,9 @@ ZOOM_connection_connect(ZOOM_connection c,
     else
         c->lang = 0;
 
+    val = ZOOM_options_get (c->options, "sru");
+    c->sru_mode = get_sru_mode_from_string(val);
+
     xfree (c->host_port);
     if (portnum)
     {
@@ -388,6 +421,37 @@ ZOOM_connection_connect(ZOOM_connection c,
     else
         c->host_port = xstrdup(host);
 
+    {
+        /*
+         * If the "<scheme>:" part of the host string is preceded by one
+         * or more comma-separated <name>=<value> pairs, these are taken
+         * to be options to be set on the connection object.  Among other
+         * applications, this facility can be used to embed authentication
+         * in a host string:
+         *          user=admin,password=secret,tcp:localhost:9999
+         */
+        char *remainder = c->host_port;
+        char *pcolon = strchr(remainder, ':');
+        char *pcomma;
+        char *pequals;
+        while ((pcomma = strchr(remainder, ',')) != 0 &&
+               (pcolon == 0 || pcomma < pcolon)) {
+            *pcomma = '\0';
+            if ((pequals = strchr(remainder, '=')) != 0) {
+                *pequals = '\0';
+                /*printf("# setting '%s'='%s'\n", remainder, pequals+1);*/
+                ZOOM_connection_option_set(c, remainder, pequals+1);
+            }
+            remainder = pcomma+1;
+        }
+
+        if (remainder != c->host_port) {
+            xfree(c->host_port);
+            c->host_port = xstrdup(remainder);
+            /*printf("# reset hp='%s'\n", remainder);*/
+        }
+    }
+
     ZOOM_options_set(c->options, "host", c->host_port);
 
     val = ZOOM_options_get (c->options, "cookie");
@@ -489,6 +553,54 @@ ZOOM_query_cql(ZOOM_query s, const char *str)
     return 0;
 }
 
+/*
+ * Translate the CQL string client-side into RPN which is passed to
+ * the server.  This is useful for server's that don't themselves
+ * support CQL, for which ZOOM_query_cql() is useless.  `conn' is used
+ * only as a place to stash diagnostics if compilation fails; if this
+ * information is not needed, a null pointer may be used.
+ */
+ZOOM_API(int)
+ZOOM_query_cql2rpn(ZOOM_query s, const char *str, ZOOM_connection conn)
+{
+    char *rpn;
+    int ret;
+
+    yaz_log(log_details, "%p ZOOM_query_cql2rpn str=%s conn=%p", s, str, conn);
+    if (conn == 0)
+        conn = ZOOM_connection_create(0);
+
+    if ((rpn = cql2pqf(conn, str)) == 0)
+        return -1;
+
+    ret = ZOOM_query_prefix(s, rpn);
+    xfree(rpn);
+    return ret;
+}
+
+/*
+ * Analogous in every way to ZOOM_query_cql2rpn(), except that there
+ * is no analogous ZOOM_query_ccl() that just sends uninterpreted CCL
+ * to the server, as the YAZ GFS doesn't know how to handle this.
+ */
+ZOOM_API(int)
+ZOOM_query_ccl2rpn(ZOOM_query s, const char *str, ZOOM_connection conn)
+{
+    char *rpn;
+    int ret;
+
+    yaz_log(log_details, "%p ZOOM_query_ccl2rpn str=%s conn=%p", s, str, conn);
+    if (conn == 0)
+        conn = ZOOM_connection_create(0);
+
+    if ((rpn = ccl2pqf(conn, str)) == 0)
+        return -1;
+
+    ret = ZOOM_query_prefix(s, rpn);
+    xfree(rpn);
+    return ret;
+}
+
 ZOOM_API(int)
 ZOOM_query_sortby(ZOOM_query s, const char *criteria)
 {
@@ -547,6 +659,7 @@ void ZOOM_resultset_addref (ZOOM_resultset r)
 
 ZOOM_resultset ZOOM_resultset_create ()
 {
+    int i;
     ZOOM_resultset r = (ZOOM_resultset) xmalloc (sizeof(*r));
 
     initlog();
@@ -561,7 +674,8 @@ ZOOM_resultset ZOOM_resultset_create ()
     r->schema = 0;
     r->count = 0;
     r->step = 0;
-    r->record_cache = 0;
+    for (i = 0; i<RECORD_HASH_SIZE; i++)
+        r->record_hash[i] = 0;
     r->r_sort_spec = 0;
     r->query = 0;
     r->connection = 0;
@@ -644,17 +758,34 @@ ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
     return r;
 }
 
+/*
+ * This is the old result-set sorting API, which is maintained only
+ * for the sake of binary compatibility.  There is no reason ever to
+ * use this rather than ZOOM_resultset_sort1().
+ */
 ZOOM_API(void)
-    ZOOM_resultset_sort(ZOOM_resultset r,
-                        const char *sort_type, const char *sort_spec)
+ZOOM_resultset_sort(ZOOM_resultset r,
+                    const char *sort_type, const char *sort_spec)
+{
+    (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
+}
+
+ZOOM_API(int)
+ZOOM_resultset_sort1(ZOOM_resultset r,
+                     const char *sort_type, const char *sort_spec)
 {
     ZOOM_connection c = r->connection;
     ZOOM_task task;
+    ZOOM_query newq;
+
+    newq = ZOOM_query_create();
+    if (ZOOM_query_sortby(newq, sort_spec) < 0)
+        return -1;
 
     yaz_log(log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
             r, r, sort_type, sort_spec);
     if (!c)
-        return;
+        return 0;
 
     if (c->host_port && c->proto == PROTO_HTTP)
     {
@@ -674,8 +805,7 @@ ZOOM_API(void)
     ZOOM_resultset_cache_reset(r);
     task = ZOOM_connection_add_task (c, ZOOM_TASK_SORT);
     task->u.sort.resultset = r;
-    task->u.sort.q = ZOOM_query_create();
-    ZOOM_query_sortby(task->u.sort.q, sort_spec);
+    task->u.sort.q = newq;
 
     ZOOM_resultset_addref (r);  
 
@@ -684,23 +814,28 @@ ZOOM_API(void)
         while (ZOOM_event (1, &c))
             ;
     }
+
+    return 0;
 }
 
 ZOOM_API(void)
     ZOOM_resultset_cache_reset(ZOOM_resultset r)
 {
-    ZOOM_record_cache rc;
-    
-    for (rc = r->record_cache; rc; rc = rc->next)
+    int i;
+    for (i = 0; i<RECORD_HASH_SIZE; i++)
     {
-        if (rc->rec.wrbuf_marc)
-            wrbuf_free (rc->rec.wrbuf_marc, 1);
-        if (rc->rec.wrbuf_iconv)
-            wrbuf_free (rc->rec.wrbuf_iconv, 1);
-        if (rc->rec.wrbuf_opac)
-            wrbuf_free (rc->rec.wrbuf_opac, 1);
+        ZOOM_record_cache rc;
+        for (rc = r->record_hash[i]; rc; rc = rc->next)
+        {
+            if (rc->rec.wrbuf_marc)
+                wrbuf_free (rc->rec.wrbuf_marc, 1);
+            if (rc->rec.wrbuf_iconv)
+                wrbuf_free (rc->rec.wrbuf_iconv, 1);
+            if (rc->rec.wrbuf_opac)
+                wrbuf_free (rc->rec.wrbuf_opac, 1);
+        }
+        r->record_hash[i] = 0;
     }
-    r->record_cache = 0;
 }
 
 ZOOM_API(void)
@@ -813,8 +948,8 @@ ZOOM_resultset_records (ZOOM_resultset r, ZOOM_record *recs,
 
     if (!r)
         return ;
-    yaz_log(log_api, "%p ZOOM_resultset_records r=%p start=%d count=%d",
-            r, r, start, count);
+    yaz_log(log_api, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
+            r, r, (long) start, (long) count);
     if (count && recs)
         force_present = 1;
     ZOOM_resultset_retrieve (r, force_present, start, count);
@@ -905,7 +1040,7 @@ static zoom_ret do_connect (ZOOM_connection c)
         }
     }
     c->state = STATE_IDLE;
-    set_ZOOM_error(c, ZOOM_ERROR_CONNECT, effective_host);
+    set_ZOOM_error(c, ZOOM_ERROR_CONNECT, c->host_port);
     return zoom_complete;
 }
 
@@ -1045,7 +1180,7 @@ static zoom_ret ZOOM_connection_send_init (ZOOM_connection c)
         ZOOM_options_get(c->options, "implementationName"),
         odr_prepend(c->odr_out, "ZOOM-C", ireq->implementationName));
 
-    version = odr_strdup(c->odr_out, "$Revision: 1.48 $");
+    version = odr_strdup(c->odr_out, "$Revision: 1.76 $");
     if (strlen(version) > 10)   /* check for unexpanded CVS strings */
         version[strlen(version)-2] = '\0';
     ireq->implementationVersion = odr_prepend(c->odr_out,
@@ -1112,26 +1247,11 @@ static zoom_ret ZOOM_connection_send_init (ZOOM_connection c)
         
         if ((oi_unit = yaz_oi_update(oi, c->odr_out, NULL, 0, 0)))
         {
-            char **charsets_addresses = 0;
-            char **langs_addresses = 0;
-            int charsets_count = 0;
-            int langs_count = 0;
-           
-            if (c->charset)
-                nmem_strsplit_blank(c->odr_out->mem, c->charset,
-                                    &charsets_addresses, &charsets_count);
-            if (c->lang)
-                nmem_strsplit_blank(c->odr_out->mem, c->lang,
-                                    &langs_addresses, &langs_count);
             ODR_MASK_SET(ireq->options, Z_Options_negotiationModel);
             oi_unit->which = Z_OtherInfo_externallyDefinedInfo;
             oi_unit->information.externallyDefinedInfo =
-                yaz_set_proposal_charneg(c->odr_out,
-                                         (const char **) charsets_addresses,
-                                         charsets_count,
-                                         (const char **) langs_addresses,
-                                         langs_count, 
-                                         1);
+                yaz_set_proposal_charneg_list(c->odr_out, " ",
+                                              c->charset, c->lang, 1);
         }
     }
     assert (apdu);
@@ -1141,71 +1261,27 @@ static zoom_ret ZOOM_connection_send_init (ZOOM_connection c)
 #if HAVE_XML2
 static zoom_ret send_srw (ZOOM_connection c, Z_SRW_PDU *sr)
 {
-    char ctype[50];
-    Z_SOAP_Handler h[2] = {
-        {"http://www.loc.gov/zing/srw/", 0, (Z_SOAP_fun) yaz_srw_codec},
-        {0, 0, 0}
-    };
-    ODR o = odr_createmem(ODR_ENCODE);
-    int ret;
-    Z_SOAP *p = (Z_SOAP*) odr_malloc(o, sizeof(*p));
     Z_GDU *gdu;
     ZOOM_Event event;
 
-    gdu = z_get_HTTP_Request(c->odr_out);
-    gdu->u.HTTP_Request->path = c->path;
+    gdu = z_get_HTTP_Request_host_path(c->odr_out, c->host_port, c->path);
 
-    if (c->host_port)
+    if (c->sru_mode == zoom_sru_get)
     {
-        const char *cp0 = strstr(c->host_port, "://");
-        const char *cp1 = 0;
-        if (cp0)
-            cp0 = cp0+3;
-        else
-            cp0 = c->host_port;
-
-        cp1 = strchr(cp0, '/');
-        if (!cp1)
-            cp1 = cp0+strlen(cp0);
-
-        if (cp0 && cp1)
-        {
-            char *h = (char*) odr_malloc(c->odr_out, cp1 - cp0 + 1);
-            memcpy (h, cp0, cp1 - cp0);
-            h[cp1-cp0] = '\0';
-            z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
-                              "Host", h);
-        }
+        yaz_sru_get_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
     }
-
-    strcpy(ctype, "text/xml");
-    if (c->charset && strlen(c->charset) < 20)
+    else if (c->sru_mode == zoom_sru_post)
     {
-        strcat(ctype, "; charset=");
-        strcat(ctype, c->charset);
+        yaz_sru_post_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
+    }
+    else if (c->sru_mode == zoom_sru_soap)
+    {
+        yaz_sru_post_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
     }
-    z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
-                      "Content-Type", ctype);
-    z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
-                      "SOAPAction", "\"\"");
-    p->which = Z_SOAP_generic;
-    p->u.generic = (Z_SOAP_Generic *) odr_malloc(o, sizeof(*p->u.generic));
-    p->u.generic->no = 0;
-    p->u.generic->ns = 0;
-    p->u.generic->p = sr;
-    p->ns = "http://schemas.xmlsoap.org/soap/envelope/";
-
-    ret = z_soap_codec_enc(o, &p,
-                           &gdu->u.HTTP_Request->content_buf,
-                           &gdu->u.HTTP_Request->content_len, h,
-                           c->charset);
-
     if (!z_GDU(c->odr_out, &gdu, 0, 0))
         return zoom_complete;
     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
-
-    odr_destroy(o);
-
+        
     event = ZOOM_Event_create (ZOOM_EVENT_SEND_APDU);
     ZOOM_connection_put_event (c, event);
     odr_reset(c->odr_out);
@@ -1489,7 +1565,15 @@ ZOOM_resultset_record (ZOOM_resultset r, size_t pos)
 
     if (!rec)
     {
-        ZOOM_resultset_retrieve (r, 1, pos, 1);
+        /*
+         * MIKE: I think force_sync should always be zero, but I don't
+         * want to make this change until I get the go-ahead from
+         * Adam, in case something depends on the old synchronous
+         * behaviour.
+         */
+        int force_sync = 1;
+        if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
+        ZOOM_resultset_retrieve (r, force_sync, pos, 1);
         rec = ZOOM_resultset_record_immediate (r, pos);
     }
     return rec;
@@ -1770,7 +1854,7 @@ ZOOM_record_get (ZOOM_record rec, const char *type_spec, int *len)
         }
         return 0;
     }
-    else if (!strcmp (type, "xml") || !strcmp(type, "oai"))
+    else if (!strcmp (type, "xml"))
     {
         Z_External *r = (Z_External *) npr->u.databaseRecord;
         oident *ent = oid_getentbyoid(r->direct_reference);
@@ -1794,8 +1878,6 @@ ZOOM_record_get (ZOOM_record rec, const char *type_spec, int *len)
             const char *ret_buf;
             int marc_decode_type = YAZ_MARC_MARCXML;
 
-            if (!strcmp(type, "oai"))
-                marc_decode_type = YAZ_MARC_OAIMARC;
             switch (ent->value)
             {
             case VAL_SOIF:
@@ -1880,6 +1962,13 @@ static int strcmp_null(const char *v1, const char *v2)
     return strcmp(v1, v2);
 }
 
+static size_t record_hash(int pos)
+{
+    if (pos < 0)
+        pos = 0;
+    return pos % RECORD_HASH_SIZE;
+}
+
 static void record_cache_add (ZOOM_resultset r, Z_NamePlusRecord *npr, 
                               int pos)
 {
@@ -1892,7 +1981,7 @@ static void record_cache_add (ZOOM_resultset r, Z_NamePlusRecord *npr,
     ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
     ZOOM_connection_put_event(r->connection, event);
 
-    for (rc = r->record_cache; rc; rc = rc->next)
+    for (rc = r->record_hash[record_hash(pos)]; rc; rc = rc->next)
     {
         if (pos == rc->pos)
         {
@@ -1930,8 +2019,8 @@ static void record_cache_add (ZOOM_resultset r, Z_NamePlusRecord *npr,
         rc->schema = 0;
 
     rc->pos = pos;
-    rc->next = r->record_cache;
-    r->record_cache = rc;
+    rc->next = r->record_hash[record_hash(pos)];
+    r->record_hash[record_hash(pos)] = rc;
 }
 
 static ZOOM_record record_cache_lookup (ZOOM_resultset r, int pos)
@@ -1942,7 +2031,7 @@ static ZOOM_record record_cache_lookup (ZOOM_resultset r, int pos)
     const char *syntax = 
         ZOOM_resultset_option_get (r, "preferredRecordSyntax");
     
-    for (rc = r->record_cache; rc; rc = rc->next)
+    for (rc = r->record_hash[record_hash(pos)]; rc; rc = rc->next)
     {
         if (pos == rc->pos)
         {
@@ -2035,7 +2124,8 @@ static void handle_queryExpressionTerm(ZOOM_options opt, const char *name,
     {
     case Z_Term_general:
         ZOOM_options_setl(opt, name,
-                          term->u.general->buf, term->u.general->len);
+                          (const char *)(term->u.general->buf), 
+                          term->u.general->len);
         break;
     case Z_Term_characterString:
         ZOOM_options_set(opt, name, term->u.characterString);
@@ -2082,7 +2172,7 @@ static void handle_searchResult(ZOOM_connection c, ZOOM_resultset resultset,
                 
                 if (sr->num)
                     ZOOM_options_set_int(
-                        resultset->options, "SearchResult.size", sr->num);
+                        resultset->options, "searchresult.size", sr->num);
 
                 for (j = 0; j < sr->num; j++)
                 {
@@ -2090,12 +2180,12 @@ static void handle_searchResult(ZOOM_connection c, ZOOM_resultset resultset,
                         ext->u.searchResult1->elements[j];
                     char pref[80];
                     
-                    sprintf(pref, "SearchResult.%d", j);
+                    sprintf(pref, "searchresult.%d", j);
 
                     if (ent->subqueryId)
                     {
                         char opt_name[80];
-                        sprintf(opt_name, "%s.subqueryId", pref);
+                        sprintf(opt_name, "%s.id", pref);
                         ZOOM_options_set(resultset->options, opt_name,
                                          ent->subqueryId);
                     }
@@ -2123,7 +2213,7 @@ static void handle_searchResult(ZOOM_connection c, ZOOM_resultset resultset,
                     if (ent->subqueryCount)
                     {
                         char opt_name[80];
-                        sprintf(opt_name, "%s.subqueryCount", pref);
+                        sprintf(opt_name, "%s.count", pref);
                         ZOOM_options_set_int(resultset->options, opt_name,
                                              *ent->subqueryCount);
                     }                                             
@@ -2278,9 +2368,10 @@ static zoom_ret send_present(ZOOM_connection c)
     *req->resultSetStartPoint = resultset->start + 1;
     *req->numberOfRecordsRequested = resultset->step>0 ?
         resultset->step : resultset->count;
+    if (*req->numberOfRecordsRequested + resultset->start > resultset->size)
+        *req->numberOfRecordsRequested = resultset->size -  resultset->start;
     assert (*req->numberOfRecordsRequested > 0);
 
-
     if (syntax && *syntax)
         req->preferredRecordSyntax =
             yaz_str_to_z3950oid (c->odr_out, CLASS_RECSYN, syntax);
@@ -2348,7 +2439,23 @@ static zoom_ret send_present(ZOOM_connection c)
 ZOOM_API(ZOOM_scanset)
 ZOOM_connection_scan (ZOOM_connection c, const char *start)
 {
+    ZOOM_scanset s;
+    ZOOM_query q = ZOOM_query_create();
+
+    ZOOM_query_prefix (q, start);
+
+    s = ZOOM_connection_scan1(c, q);
+    ZOOM_query_destroy (q);
+    return s;
+
+}
+
+ZOOM_API(ZOOM_scanset)
+ZOOM_connection_scan1 (ZOOM_connection c, ZOOM_query q)
+{
     ZOOM_scanset scan = (ZOOM_scanset) xmalloc (sizeof(*scan));
+    char *start;
+    char *freeme = 0;
 
     scan->connection = c;
     scan->odr = odr_createmem (ODR_DECODE);
@@ -2356,9 +2463,39 @@ ZOOM_connection_scan (ZOOM_connection c, const char *start)
     scan->refcount = 1;
     scan->scan_response = 0;
 
-    if ((scan->termListAndStartPoint =
-         p_query_scan(scan->odr, PROTO_Z3950, &scan->attributeSet,
-                      start)))
+    /*
+     * We need to check the query-type, so we can recognise CQL and
+     * CCL and compile them into a form that we can use here.  The
+     * ZOOM_query structure has no explicit `type' member, but
+     * inspection of the ZOOM_query_prefix() and ZOOM_query_cql()
+     * functions shows how the structure is set up in each case.
+     */
+    if (q->z_query->which == Z_Query_type_1) {
+        yaz_log(log_api, "%p ZOOM_connection_scan1 q=%p PQF '%s'",
+                c, q, q->query_string);
+        start = q->query_string;
+    } else if (q->z_query->which == Z_Query_type_104) {
+        yaz_log(log_api, "%p ZOOM_connection_scan1 q=%p CQL '%s'",
+                c, q, q->query_string);
+        start = freeme = cql2pqf(c, q->query_string);
+        if (start == 0)
+            return 0;
+    } else if (q->z_query->which == Z_Query_type_2) {
+        yaz_log(log_api, "%p ZOOM_connection_scan1 q=%p CCL '%s'",
+                c, q, q->query_string);
+        start = freeme = ccl2pqf(c, q->query_string);
+        if (start == 0)
+            return 0;
+    } else {
+        yaz_log(YLOG_FATAL, "%p ZOOM_connection_scan1 q=%p unknown type '%s'",
+                c, q, q->query_string);
+        abort();
+    }
+
+    scan->termListAndStartPoint =
+        p_query_scan(scan->odr, PROTO_Z3950, &scan->attributeSet, start);
+    xfree (freeme);
+    if (scan->termListAndStartPoint != 0)
     {
         ZOOM_task task = ZOOM_connection_add_task (c, ZOOM_TASK_SCAN);
         task->u.scan.scan = scan;
@@ -2400,8 +2537,10 @@ static zoom_ret send_package (ZOOM_connection c)
     event = ZOOM_Event_create (ZOOM_EVENT_SEND_APDU);
     ZOOM_connection_put_event (c, event);
     
-    return do_write_ex (c, c->tasks->u.package->buf_out,
-                        c->tasks->u.package->len_out);
+    c->buf_out = c->tasks->u.package->buf_out;
+    c->len_out = c->tasks->u.package->len_out;
+
+    return do_write(c);
 }
 
 static zoom_ret send_scan (ZOOM_connection c)
@@ -2589,13 +2728,10 @@ static Z_External *encode_ill_request (ZOOM_package p)
         r->descriptor = 0;
         r->which = Z_External_single;
                 
-        r->u.single_ASN1_type = (Odr_oct *)
-            odr_malloc (out, sizeof(*r->u.single_ASN1_type));
-        r->u.single_ASN1_type->buf = (unsigned char*)
-            odr_malloc (out, illRequest_size);
-        r->u.single_ASN1_type->len = illRequest_size;
-        r->u.single_ASN1_type->size = illRequest_size;
-        memcpy (r->u.single_ASN1_type->buf, illRequest_buf, illRequest_size);
+        r->u.single_ASN1_type =
+            odr_create_Odr_oct(out,
+                               (unsigned char *)illRequest_buf,
+                               illRequest_size);
     }
     return r;
 }
@@ -2717,20 +2853,18 @@ static Z_APDU *create_xmlupdate_package(ZOOM_package p)
     Z_External *ext = (Z_External *) odr_malloc(p->odr_out, sizeof(*ext));
     const char *doc = ZOOM_options_get(p->options, "doc");
 
+    if (!doc)
+        doc = "";
+
     req->taskSpecificParameters = ext;
     ext->direct_reference = req->packageType;
     ext->descriptor = 0;
     ext->indirect_reference = 0;
     
     ext->which = Z_External_octet;
-    ext->u.single_ASN1_type = (Odr_oct *)
-        odr_malloc (p->odr_out, sizeof(Odr_oct));
-
-    if (!doc)
-        doc = "";
-    ext->u.single_ASN1_type->buf = (unsigned char*)
-        odr_strdup(p->odr_out, doc);
-    ext->u.single_ASN1_type->size = ext->u.single_ASN1_type->len = strlen(doc);
+    ext->u.single_ASN1_type =
+        odr_create_Odr_oct(p->odr_out, (const unsigned char *) doc,
+                           strlen(doc));
     return apdu;
 }
 
@@ -2820,12 +2954,10 @@ static Z_APDU *create_update_package(ZOOM_package p)
         notToKeep->elements[0]->which = Z_IUSuppliedRecords_elem_opaque;
         if (recordIdOpaque)
         {
-            notToKeep->elements[0]->u.opaque = (Odr_oct *)
-                odr_malloc (p->odr_out, sizeof(Odr_oct));
-            notToKeep->elements[0]->u.opaque->size =
-                notToKeep->elements[0]->u.opaque->len = strlen(recordIdOpaque);
-            notToKeep->elements[0]->u.opaque->buf = (unsigned char*)
-                odr_strdup(p->odr_out, recordIdOpaque);
+            notToKeep->elements[0]->u.opaque = 
+                odr_create_Odr_oct(p->odr_out,
+                                   (const unsigned char *) recordIdOpaque,
+                                   strlen(recordIdOpaque));
         }
         else if (recordIdNumber)
         {
@@ -3237,7 +3369,7 @@ static void recv_apdu (ZOOM_connection c, Z_APDU *apdu)
         }
         else
         {
-            set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, 0);
+            set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
             do_close(c);
         }
         break;
@@ -3298,6 +3430,7 @@ static void handle_srw_response(ZOOM_connection c,
         npr->u.databaseRecord->direct_reference =
             yaz_oidval_to_z3950oid(c->odr_in, CLASS_RECSYN, VAL_TEXT_XML);
         npr->u.databaseRecord->which = Z_External_octet;
+
         npr->u.databaseRecord->u.octet_aligned = (Odr_oct *)
             odr_malloc(c->odr_in, sizeof(Odr_oct));
         npr->u.databaseRecord->u.octet_aligned->buf = (unsigned char*)
@@ -3417,7 +3550,7 @@ static int do_read (ZOOM_connection c)
         }
         else
         {
-            set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, 0);
+            set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
             do_close (c);
         }
     }
@@ -3480,9 +3613,9 @@ static zoom_ret do_write_ex (ZOOM_connection c, char *buf_out, int len_out)
             return zoom_pending;
         }
         if (c->state == STATE_CONNECTING)
-            set_ZOOM_error(c, ZOOM_ERROR_CONNECT, 0);
+            set_ZOOM_error(c, ZOOM_ERROR_CONNECT, c->host_port);
         else
-            set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, 0);
+            set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
         do_close (c);
         return zoom_complete;
     }
@@ -3574,6 +3707,14 @@ ZOOM_connection_addinfo (ZOOM_connection c)
 }
 
 ZOOM_API(const char *)
+ZOOM_connection_diagset (ZOOM_connection c)
+{
+    const char *diagset;
+    ZOOM_connection_error_x (c, 0, 0, &diagset);
+    return diagset;
+}
+
+ZOOM_API(const char *)
 ZOOM_diag_str (int error)
 {
     switch (error)
@@ -3602,6 +3743,14 @@ ZOOM_diag_str (int error)
         return "Unsupported query type";
     case ZOOM_ERROR_INVALID_QUERY:
         return "Invalid query";
+    case ZOOM_ERROR_CQL_PARSE:
+        return "CQL parsing error";
+    case ZOOM_ERROR_CQL_TRANSFORM:
+        return "CQL transformation error";
+    case ZOOM_ERROR_CCL_CONFIG:
+        return "CCL configuration error";
+    case ZOOM_ERROR_CCL_PARSE:
+        return "CCL parsing error";
     default:
         return diagbib1_str (error);
     }
@@ -3649,7 +3798,7 @@ static int ZOOM_connection_do_io(ZOOM_connection c, int mask)
     if (r == CS_NONE)
     {
         event = ZOOM_Event_create (ZOOM_EVENT_CONNECT);
-        set_ZOOM_error(c, ZOOM_ERROR_CONNECT, 0);
+        set_ZOOM_error(c, ZOOM_ERROR_CONNECT, c->host_port);
         do_close (c);
         ZOOM_connection_put_event (c, event);
     }
@@ -3688,7 +3837,7 @@ static int ZOOM_connection_do_io(ZOOM_connection c, int mask)
         }
         else
         {
-            set_ZOOM_error(c, ZOOM_ERROR_CONNECT, 0);
+            set_ZOOM_error(c, ZOOM_ERROR_CONNECT, c->host_port);
             do_close (c);
             ZOOM_connection_put_event (c, event);
         }
@@ -3895,6 +4044,113 @@ ZOOM_event (int no, ZOOM_connection *cs)
     }
     return 0;
 }
+
+
+/*
+ * Returns an xmalloc()d string containing RPN that corresponds to the
+ * CQL passed in.  On error, sets the Connection object's error state
+ * and returns a null pointer.
+ * ### We could cache CQL parser and/or transformer in Connection.
+ */
+static char *cql2pqf(ZOOM_connection c, const char *cql)
+{
+    CQL_parser parser;
+    int error;
+    struct cql_node *node;
+    const char *cqlfile;
+    static cql_transform_t trans;
+    char pqfbuf[512];
+
+    parser = cql_parser_create();
+    if ((error = cql_parser_string(parser, cql)) != 0) {
+        cql_parser_destroy(parser);
+        set_ZOOM_error(c, ZOOM_ERROR_CQL_PARSE, cql);
+        return 0;
+    }
+
+    node = cql_parser_result(parser);
+    /* ### Do not call cql_parser_destroy() yet: it destroys `node'! */
+
+    cqlfile = ZOOM_connection_option_get(c, "cqlfile");
+    if (cqlfile == 0) {
+        cql_parser_destroy(parser);
+        cql_node_destroy(node);
+        set_ZOOM_error(c, ZOOM_ERROR_CQL_TRANSFORM, "no CQL transform file");
+        return 0;
+    }
+
+    if ((trans = cql_transform_open_fname(cqlfile)) == 0) {
+        char buf[512];        
+        cql_parser_destroy(parser);
+        cql_node_destroy(node);
+        sprintf(buf, "can't open CQL transform file '%.200s': %.200s",
+                cqlfile, strerror(errno));
+        set_ZOOM_error(c, ZOOM_ERROR_CQL_TRANSFORM, buf);
+        return 0;
+    }
+
+    error = cql_transform_buf(trans, node, pqfbuf, sizeof pqfbuf);
+    cql_parser_destroy(parser);
+    cql_node_destroy(node);
+    if (error != 0) {
+        char buf[512];
+        const char *addinfo;
+        error = cql_transform_error(trans, &addinfo);
+        cql_transform_close(trans);
+        sprintf(buf, "%.200s (addinfo=%.200s)", cql_strerror(error), addinfo);
+        set_ZOOM_error(c, ZOOM_ERROR_CQL_TRANSFORM, buf);
+        return 0;
+    }
+
+    cql_transform_close(trans);
+    return xstrdup(pqfbuf);
+}
+
+
+/* ### Could cache `bibset' */
+static char *ccl2pqf(ZOOM_connection c, const char *ccl)
+{
+    const char *cclfile;
+    CCL_bibset bibset;
+    struct ccl_rpn_node *node;
+    int errorcode;
+    int errorpos;
+    WRBUF w;
+    char *pqf;
+
+    if ((cclfile = ZOOM_connection_option_get(c, "cclfile")) == 0) {
+        set_ZOOM_error(c, ZOOM_ERROR_CCL_CONFIG, "no CCL qualifier file");
+        return 0;
+    }
+
+    bibset = ccl_qual_mk();
+    if (ccl_qual_fname(bibset, cclfile) < 0) {
+        char buf[512];
+        ccl_qual_rm(&bibset);
+        sprintf(buf, "can't open CCL qualifier file '%.200s': %.200s",
+                cclfile, strerror(errno));
+        set_ZOOM_error(c, ZOOM_ERROR_CCL_CONFIG, buf);
+        return 0;
+    }
+
+    node = ccl_find_str (bibset, ccl, &errorcode, &errorpos);
+    ccl_qual_rm(&bibset);
+    if (node == 0) {
+        /* code is one of the CCL_ERR_* constants; pos is unused here */
+        set_ZOOM_error(c, ZOOM_ERROR_CCL_PARSE, ccl_err_msg(errorcode));
+        return 0;
+    }
+
+    w = wrbuf_alloc();
+    ccl_pquery(w, node);
+    ccl_rpn_delete(node);
+    pqf = xstrdup(wrbuf_buf(w));
+    wrbuf_free(w, 1);
+
+    return pqf;
+}
+
+
 /*
  * Local variables:
  * c-basic-offset: 4