Remove ZOOM lsg msg YAZ-846
[yaz-moved-to-github.git] / src / zoom-c.c
index 923e500..7fd780d 100644 (file)
@@ -1,11 +1,14 @@
 /* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2010 Index Data
+ * Copyright (C) Index Data
  * See the file LICENSE for details.
  */
 /**
  * \file zoom-c.c
  * \brief Implements ZOOM C interface.
  */
+#if HAVE_CONFIG_H
+#include <config.h>
+#endif
 
 #include <assert.h>
 #include <string.h>
 #include <yaz/xmalloc.h>
 #include <yaz/otherinfo.h>
 #include <yaz/log.h>
-#include <yaz/pquery.h>
-#include <yaz/marcdisp.h>
 #include <yaz/diagbib1.h>
 #include <yaz/charneg.h>
-#include <yaz/ill.h>
-#include <yaz/srw.h>
-#include <yaz/cql.h>
-#include <yaz/ccl.h>
 #include <yaz/query-charset.h>
-#include <yaz/copy_types.h>
 #include <yaz/snprintf.h>
 #include <yaz/facet.h>
 
@@ -40,29 +36,11 @@ static int log_details0 = 0;
 
 static void resultset_destroy(ZOOM_resultset r);
 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
-static char *cql2pqf(ZOOM_connection c, const char *cql);
-
-ZOOM_API(const char *) ZOOM_get_event_str(int event)
-{
-    static const char *ar[] = {
-        "NONE",
-        "CONNECT",
-        "SEND_DATA",
-        "RECV_DATA",
-        "TIMEOUT",
-        "UNKNOWN",
-        "SEND_APDU",
-        "RECV_APDU",
-        "RECV_RECORD",
-        "RECV_SEARCH",
-        "END"
-    };
-    return ar[event];
-}
 
 static void initlog(void)
 {
     static int log_level_initialized = 0;
+
     if (!log_level_initialized)
     {
         log_api0 = yaz_log_module_level("zoom");
@@ -71,73 +49,7 @@ static void initlog(void)
     }
 }
 
-ZOOM_Event ZOOM_Event_create(int kind)
-{
-    ZOOM_Event event = (ZOOM_Event) xmalloc(sizeof(*event));
-    event->kind = kind;
-    event->next = 0;
-    event->prev = 0;
-    return event;
-}
-
-static void ZOOM_Event_destroy(ZOOM_Event event)
-{
-    xfree(event);
-}
-
-void ZOOM_connection_put_event(ZOOM_connection c, ZOOM_Event event)
-{
-    if (c->m_queue_back)
-    {
-        c->m_queue_back->prev = event;
-        assert(c->m_queue_front);
-    }
-    else
-    {
-        assert(!c->m_queue_front);
-        c->m_queue_front = event;
-    }
-    event->next = c->m_queue_back;
-    event->prev = 0;
-    c->m_queue_back = event;
-}
-
-static ZOOM_Event ZOOM_connection_get_event(ZOOM_connection c)
-{
-    ZOOM_Event event = c->m_queue_front;
-    if (!event)
-    {
-        c->last_event = ZOOM_EVENT_NONE;
-        return 0;
-    }
-    assert(c->m_queue_back);
-    c->m_queue_front = event->prev;
-    if (c->m_queue_front)
-    {
-        assert(c->m_queue_back);
-        c->m_queue_front->next = 0;
-    }
-    else
-        c->m_queue_back = 0;
-    c->last_event = event->kind;
-    return event;
-}
-
-static void ZOOM_connection_remove_events(ZOOM_connection c)
-{
-    ZOOM_Event event;
-    while ((event = ZOOM_connection_get_event(c)))
-        ZOOM_Event_destroy(event);
-}
-
-ZOOM_API(int) ZOOM_connection_peek_event(ZOOM_connection c)
-{
-    ZOOM_Event event = c->m_queue_front;
-
-    return event ? event->kind : ZOOM_EVENT_NONE;
-}
-
-void ZOOM_connection_remove_tasks(ZOOM_connection c);
+static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri);
 
 void ZOOM_set_dset_error(ZOOM_connection c, int error,
                          const char *dset,
@@ -158,8 +70,9 @@ void ZOOM_set_dset_error(ZOOM_connection c, int error,
     }
     if (addinfo && addinfo2)
     {
-        c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 2);
+        c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 3);
         strcpy(c->addinfo, addinfo);
+        strcat(c->addinfo, ": ");
         strcat(c->addinfo, addinfo2);
     }
     else if (addinfo)
@@ -170,36 +83,18 @@ void ZOOM_set_dset_error(ZOOM_connection c, int error,
                 c, c->host_port ? c->host_port : "<>", dset, error,
                 addinfo ? addinfo : "",
                 addinfo2 ? addinfo2 : "");
-        ZOOM_connection_remove_tasks(c);
     }
 }
 
-static int uri_to_code(const char *uri)
+int ZOOM_uri_to_code(const char *uri)
 {
-    int code = 0;       
+    int code = 0;
     const char *cp;
     if ((cp = strrchr(uri, '/')))
         code = atoi(cp+1);
     return code;
 }
 
-#if YAZ_HAVE_XML2
-static void set_HTTP_error(ZOOM_connection c, int error,
-                           const char *addinfo, const char *addinfo2)
-{
-    ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
-}
-
-static void set_SRU_error(ZOOM_connection c, Z_SRW_diagnostic *d)
-{
-    const char *uri = d->uri;
-    if (uri)
-        ZOOM_set_dset_error(c, uri_to_code(uri), uri, d->details, 0);
-}
-
-#endif
-
-
 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
 {
     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
@@ -238,9 +133,6 @@ void ZOOM_connection_show_task(ZOOM_task task)
     case ZOOM_TASK_SEARCH:
         yaz_log(YLOG_LOG, "search p=%p", task);
         break;
-    case ZOOM_TASK_RETRIEVE:
-        yaz_log(YLOG_LOG, "retrieve p=%p", task);
-        break;
     case ZOOM_TASK_CONNECT:
         yaz_log(YLOG_LOG, "connect p=%p", task);
         break;
@@ -285,7 +177,6 @@ ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
 
     task->running = 0;
     task->which = which;
-    clear_error(c);
     return task;
 }
 
@@ -302,11 +193,7 @@ void ZOOM_connection_remove_task(ZOOM_connection c)
             resultset_destroy(task->u.search.resultset);
             xfree(task->u.search.syntax);
             xfree(task->u.search.elementSetName);
-            break;
-        case ZOOM_TASK_RETRIEVE:
-            resultset_destroy(task->u.retrieve.resultset);
-            xfree(task->u.retrieve.syntax);
-            xfree(task->u.retrieve.elementSetName);
+            xfree(task->u.search.schema);
             break;
         case ZOOM_TASK_CONNECT:
             break;
@@ -339,6 +226,12 @@ void ZOOM_connection_remove_tasks(ZOOM_connection c)
         ZOOM_connection_remove_task(c);
 }
 
+static void odr_wrbuf_write(ODR o, void *handle, int type,
+                            const char *buf, int len)
+{
+    WRBUF w = (WRBUF) handle;
+    wrbuf_write(w, buf, len);
+}
 
 ZOOM_API(ZOOM_connection)
     ZOOM_connection_create(ZOOM_options options)
@@ -369,9 +262,10 @@ ZOOM_API(ZOOM_connection)
     c->options = ZOOM_options_create_with_parent(options);
 
     c->host_port = 0;
-    c->path = 0;
     c->proxy = 0;
-    
+    c->tproxy = 0;
+    c->proxy_mode = 0;
+
     c->charset = c->lang = 0;
 
     c->cookie_out = 0;
@@ -382,6 +276,7 @@ ZOOM_API(ZOOM_connection)
     c->user = 0;
     c->group = 0;
     c->password = 0;
+    c->url_authentication = 0;
 
     c->maximum_record_size = 0;
     c->preferred_message_size = 0;
@@ -389,6 +284,7 @@ ZOOM_API(ZOOM_connection)
     c->odr_in = odr_createmem(ODR_DECODE);
     c->odr_out = odr_createmem(ODR_ENCODE);
     c->odr_print = 0;
+    c->odr_save = 0;
 
     c->async = 0;
     c->support_named_resultsets = 0;
@@ -399,9 +295,27 @@ ZOOM_API(ZOOM_connection)
 
     c->sru_version = 0;
     c->no_redirects = 0;
+    c->cookies = 0;
+    c->location = 0;
+    c->saveAPDU_wrbuf = 0;
+
+    ZOOM_memcached_init(c);
     return c;
 }
 
+ZOOM_API(void) ZOOM_connection_save_apdu_wrbuf(ZOOM_connection c, WRBUF w)
+{
+    if (c->odr_save)
+    {
+        odr_destroy(c->odr_save);
+        c->odr_save = 0;
+    }
+    if (w)
+    {
+        c->odr_save = odr_createmem(ODR_PRINT);
+        odr_set_stream(c->odr_save, w, odr_wrbuf_write, 0);
+    }
+}
 
 /* set database names. Take local databases (if set); otherwise
    take databases given in ZURL (if set); otherwise use Default */
@@ -410,17 +324,10 @@ char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
 {
     char **databaseNames;
     const char *cp = ZOOM_options_get(options, "databaseName");
-    
+
     if ((!cp || !*cp) && con->host_port)
-    {
-        if (strncmp(con->host_port, "unix:", 5) == 0)
-            cp = strchr(con->host_port+5, ':');
-        else
-            cp = strchr(con->host_port, '/');
-        if (cp)
-            cp++;
-    }
-    if (!cp)
+        cs_get_host_args(con->host_port, &cp);
+    if (!cp || !*cp)
         cp = "Default";
     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
     return databaseNames;
@@ -445,6 +352,8 @@ static zoom_sru_mode get_sru_mode_from_string(const char *s)
         return zoom_sru_get;
     else if (!yaz_matchstr(s, "post"))
         return zoom_sru_post;
+    else if (!yaz_matchstr(s, "solr"))
+        return zoom_sru_solr;
     return zoom_sru_error;
 }
 
@@ -453,7 +362,6 @@ ZOOM_API(void)
                             const char *host, int portnum)
 {
     const char *val;
-    ZOOM_task task;
 
     initlog();
 
@@ -463,19 +371,6 @@ ZOOM_API(void)
     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
     ZOOM_connection_remove_tasks(c);
 
-    if (c->odr_print)
-    {
-        odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
-        odr_destroy(c->odr_print);
-    }
-    if (ZOOM_options_get_bool(c->options, "apdulog", 0))
-    {
-        c->odr_print = odr_createmem(ODR_PRINT);
-        odr_setprint(c->odr_print, yaz_log_file());
-    }
-    else
-        c->odr_print = 0;
-
     if (c->cs)
     {
         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
@@ -492,6 +387,15 @@ ZOOM_API(void)
         c->proxy = xstrdup(val);
     }
 
+    xfree(c->tproxy);
+    c->tproxy = 0;
+    val = ZOOM_options_get(c->options, "tproxy");
+    if (val && *val)
+    {
+        yaz_log(c->log_details, "%p ZOOM_connection_connect tproxy=%s", c, val);
+        c->tproxy = xstrdup(val);
+    }
+
     xfree(c->charset);
     c->charset = 0;
     val = ZOOM_options_get(c->options, "charset");
@@ -513,16 +417,26 @@ ZOOM_API(void)
 
     if (host)
     {
+        char hostn[128];
+        const char *http_lead;
+
+        val = ZOOM_options_get(c->options, "sru");
+        if (val && *val && !strstr(host, "://"))
+            http_lead = "http://";
+        else
+            http_lead = "";
+        c->sru_mode = get_sru_mode_from_string(val);
+
         xfree(c->host_port);
         if (portnum)
         {
-            char hostn[128];
             sprintf(hostn, "%.80s:%d", host, portnum);
-            c->host_port = xstrdup(hostn);
+            host = hostn;
         }
-        else
-            c->host_port = xstrdup(host);
-    }        
+        c->host_port = xmalloc(strlen(host) + strlen(http_lead) + 1);
+        strcpy(c->host_port, http_lead);
+        strcat(c->host_port, host);
+    }
 
     {
         /*
@@ -538,26 +452,25 @@ ZOOM_API(void)
         char *pcomma;
         char *pequals;
         while ((pcomma = strchr(remainder, ',')) != 0 &&
-               (pcolon == 0 || pcomma < pcolon)) {
+               (pcolon == 0 || pcomma < pcolon))
+        {
             *pcomma = '\0';
-            if ((pequals = strchr(remainder, '=')) != 0) {
+            if ((pequals = strchr(remainder, '=')) != 0)
+            {
                 *pequals = '\0';
-                /*printf("# setting '%s'='%s'\n", remainder, pequals+1);*/
                 ZOOM_connection_option_set(c, remainder, pequals+1);
             }
             remainder = pcomma+1;
         }
 
-        if (remainder != c->host_port) {
+        if (remainder != c->host_port)
+        {
+            remainder = xstrdup(remainder);
             xfree(c->host_port);
-            c->host_port = xstrdup(remainder);
-            /*printf("# reset hp='%s'\n", remainder);*/
+            c->host_port = remainder;
         }
     }
 
-    val = ZOOM_options_get(c->options, "sru");
-    c->sru_mode = get_sru_mode_from_string(val);
-
     xfree(c->sru_version);
     val = ZOOM_options_get(c->options, "sru_version");
     c->sru_version = xstrdup(val ? val : "1.2");
@@ -568,7 +481,7 @@ ZOOM_API(void)
     c->cookie_out = 0;
     val = ZOOM_options_get(c->options, "cookie");
     if (val && *val)
-    { 
+    {
         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
         c->cookie_out = xstrdup(val);
     }
@@ -600,167 +513,76 @@ ZOOM_API(void)
     val = ZOOM_options_get(c->options, "password");
     if (!val)
         val = ZOOM_options_get(c->options, "pass");
-
     if (val && *val)
         c->password = xstrdup(val);
-    
+
+    val = ZOOM_options_get(c->options, "authenticationMode");
+    if (val && !strcmp(val, "url"))
+        c->url_authentication = 1;
+    else
+        c->url_authentication = 0;
+
     c->maximum_record_size =
-        ZOOM_options_get_int(c->options, "maximumRecordSize", 1024*1024);
+        ZOOM_options_get_int(c->options, "maximumRecordSize", 64*1024*1024);
     c->preferred_message_size =
-        ZOOM_options_get_int(c->options, "preferredMessageSize", 1024*1024);
+        ZOOM_options_get_int(c->options, "preferredMessageSize", 64*1024*1024);
 
     c->async = ZOOM_options_get_bool(c->options, "async", 0);
-    yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
-    task = ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 
-    if (!c->async)
+    yaz_cookies_destroy(c->cookies);
+    c->cookies = yaz_cookies_create();
+
+    if (ZOOM_memcached_configure(c))
     {
-        while (ZOOM_event(1, &c))
-            ;
+        ZOOM_connection_remove_tasks(c);
+        return;
     }
-}
-
-ZOOM_API(ZOOM_query)
-    ZOOM_query_create(void)
-{
-    ZOOM_query s = (ZOOM_query) xmalloc(sizeof(*s));
-
-    s->refcount = 1;
-    s->z_query = 0;
-    s->sort_spec = 0;
-    s->odr = odr_createmem(ODR_ENCODE);
-    s->query_string = 0;
-
-    return s;
-}
-
-ZOOM_API(void)
-    ZOOM_query_destroy(ZOOM_query s)
-{
-    if (!s)
+    if (c->sru_mode == zoom_sru_error)
+    {
+        ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
+        ZOOM_connection_remove_tasks(c);
         return;
+    }
 
-    (s->refcount)--;
-    if (s->refcount == 0)
+    if (c->odr_print)
+        odr_destroy(c->odr_print);
+    c->odr_print = 0;
+    val = ZOOM_options_get(c->options, "apdufile");
+    if (val)
     {
-        odr_destroy(s->odr);
-        xfree(s);
+        c->odr_print = odr_createmem(ODR_PRINT);
+        if (strcmp(val, "-"))
+        {
+            FILE *f = fopen(val, "a");
+            if (!f)
+            {
+                WRBUF w = wrbuf_alloc();
+                wrbuf_printf(w, "fopen: %s", val);
+                ZOOM_set_error(c, ZOOM_ERROR_INTERNAL, wrbuf_cstr(w));
+                wrbuf_destroy(w);
+                return;
+            }
+            odr_setprint(c->odr_print, f);
+        }
     }
-}
-
-ZOOM_API(int)
-    ZOOM_query_prefix(ZOOM_query s, const char *str)
-{
-    s->query_string = odr_strdup(s->odr, str);
-    s->z_query = (Z_Query *) odr_malloc(s->odr, sizeof(*s->z_query));
-    s->z_query->which = Z_Query_type_1;
-    s->z_query->u.type_1 =  p_query_rpn(s->odr, str);
-    if (!s->z_query->u.type_1)
+    else if (ZOOM_options_get_bool(c->options, "apdulog", 0))
     {
-        s->z_query = 0;
-        return -1;
+        c->odr_print = odr_createmem(ODR_PRINT);
+        odr_setprint_noclose(c->odr_print, yaz_log_file());
     }
-    return 0;
-}
 
-ZOOM_API(int)
-    ZOOM_query_cql(ZOOM_query s, const char *str)
-{
-    Z_External *ext;
-
-    s->query_string = odr_strdup(s->odr, str);
-
-    ext = (Z_External *) odr_malloc(s->odr, sizeof(*ext));
-    ext->direct_reference = odr_oiddup(s->odr, yaz_oid_userinfo_cql);
-    ext->indirect_reference = 0;
-    ext->descriptor = 0;
-    ext->which = Z_External_CQL;
-    ext->u.cql = s->query_string;
-    
-    s->z_query = (Z_Query *) odr_malloc(s->odr, sizeof(*s->z_query));
-    s->z_query->which = Z_Query_type_104;
-    s->z_query->u.type_104 =  ext;
-
-    return 0;
-}
-
-/*
- * Translate the CQL string client-side into RPN which is passed to
- * the server.  This is useful for server's that don't themselves
- * support CQL, for which ZOOM_query_cql() is useless.  `conn' is used
- * only as a place to stash diagnostics if compilation fails; if this
- * information is not needed, a null pointer may be used.
- */
-ZOOM_API(int)
-    ZOOM_query_cql2rpn(ZOOM_query s, const char *str, ZOOM_connection conn)
-{
-    char *rpn;
-    int ret;
-    ZOOM_connection freeme = 0;
-
-    if (conn == 0)
-        conn = freeme = ZOOM_connection_create(0);
-
-    rpn = cql2pqf(conn, str);
-    if (freeme != 0)
-        ZOOM_connection_destroy(freeme);
-    if (rpn == 0)
-        return -1;
-
-    ret = ZOOM_query_prefix(s, rpn);
-    xfree(rpn);
-    return ret;
-}
-
-/*
- * Analogous in every way to ZOOM_query_cql2rpn(), except that there
- * is no analogous ZOOM_query_ccl() that just sends uninterpreted CCL
- * to the server, as the YAZ GFS doesn't know how to handle this.
- */
-ZOOM_API(int)
-    ZOOM_query_ccl2rpn(ZOOM_query s, const char *str, const char *config,
-                       int *ccl_error, const char **error_string,
-                       int *error_pos)
-{
-    int ret;
-    struct ccl_rpn_node *rpn;
-    CCL_bibset bibset = ccl_qual_mk();
-
-    if (config)
-        ccl_qual_buf(bibset, config);
+    yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
+    ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 
-    rpn = ccl_find_str(bibset, str, ccl_error, error_pos);
-    if (!rpn)
-    {
-        *error_string = ccl_err_msg(*ccl_error);
-        ret = -1;
-    }
-    else
+    if (!c->async)
     {
-        WRBUF wr = wrbuf_alloc();
-        ccl_pquery(wr, rpn);
-        ccl_rpn_delete(rpn);
-        ret = ZOOM_query_prefix(s, wrbuf_cstr(wr));
-        wrbuf_destroy(wr);
+        while (ZOOM_event(1, &c))
+            ;
     }
-    ccl_qual_rm(&bibset);
-    return ret;
-}
-
-ZOOM_API(int)
-    ZOOM_query_sortby(ZOOM_query s, const char *criteria)
-{
-    s->sort_spec = yaz_sort_spec(s->odr, criteria);
-    if (!s->sort_spec)
-        return -1;
-    return 0;
 }
 
 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
 {
-#if ZOOM_RESULT_LISTS
-#else
     if (r->connection)
     {
         /* remove ourselves from the resultsets in connection */
@@ -777,53 +599,38 @@ ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
         }
         r->connection = 0;
     }
-#endif
 }
 
 ZOOM_API(void)
     ZOOM_connection_destroy(ZOOM_connection c)
 {
-#if ZOOM_RESULT_LISTS
-    ZOOM_resultsets list;
-#else
     ZOOM_resultset r;
-#endif
     if (!c)
         return;
     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
+
+    ZOOM_memcached_destroy(c);
     if (c->cs)
         cs_close(c->cs);
 
-#if ZOOM_RESULT_LISTS
-    // Remove the connection's usage of resultsets
-    list = c->resultsets;
-    while (list) {
-        ZOOM_resultsets removed = list;
-        ZOOM_resultset_destroy(list->resultset);
-        list = list->next;
-        xfree(removed);
-    }
-#else
     for (r = c->resultsets; r; r = r->next)
         r->connection = 0;
-#endif
 
     xfree(c->buf_in);
     xfree(c->addinfo);
     xfree(c->diagset);
     odr_destroy(c->odr_in);
     odr_destroy(c->odr_out);
+    if (c->odr_save)
+        odr_destroy(c->odr_save);
     if (c->odr_print)
-    {
-        odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
         odr_destroy(c->odr_print);
-    }
     ZOOM_options_destroy(c->options);
     ZOOM_connection_remove_tasks(c);
     ZOOM_connection_remove_events(c);
     xfree(c->host_port);
-    xfree(c->path);
     xfree(c->proxy);
+    xfree(c->tproxy);
     xfree(c->charset);
     xfree(c->lang);
     xfree(c->cookie_out);
@@ -833,6 +640,9 @@ ZOOM_API(void)
     xfree(c->group);
     xfree(c->password);
     xfree(c->sru_version);
+    xfree(c->location);
+    yaz_cookies_destroy(c->cookies);
+    wrbuf_destroy(c->saveAPDU_wrbuf);
     xfree(c);
 }
 
@@ -848,6 +658,27 @@ void ZOOM_resultset_addref(ZOOM_resultset r)
     }
 }
 
+static int g_resultsets = 0;
+static YAZ_MUTEX g_resultset_mutex = 0;
+
+/* TODO We need to initialize this before running threaded:
+ * call resultset_use(0)  */
+
+static int resultset_use(int delta) {
+    int resultset_count;
+    if (g_resultset_mutex == 0)
+        yaz_mutex_create(&g_resultset_mutex);
+    yaz_mutex_enter(g_resultset_mutex);
+    g_resultsets += delta;
+    resultset_count = g_resultsets;
+    yaz_mutex_leave(g_resultset_mutex);
+    return resultset_count;
+}
+
+int resultsets_count(void) {
+    return resultset_use(0);
+}
+
 ZOOM_resultset ZOOM_resultset_create(void)
 {
     int i;
@@ -858,10 +689,9 @@ ZOOM_resultset ZOOM_resultset_create(void)
     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
     r->refcount = 1;
     r->size = 0;
-    r->odr = odr_createmem(ODR_ENCODE);
+    r->odr = odr_createmem(ODR_DECODE);
     r->piggyback = 1;
     r->setname = 0;
-    r->schema = 0;
     r->step = 0;
     for (i = 0; i<RECORD_HASH_SIZE; i++)
         r->record_hash[i] = 0;
@@ -870,8 +700,9 @@ ZOOM_resultset ZOOM_resultset_create(void)
     r->connection = 0;
     r->databaseNames = 0;
     r->num_databaseNames = 0;
-    r->facets = 0;
-    r->num_facets = 0;
+    r->req_facets = 0;
+    r->res_facets = 0;
+    r->num_res_facets = 0;
     r->facets_names = 0;
     r->mutex = 0;
     yaz_mutex_create(&r->mutex);
@@ -881,6 +712,9 @@ ZOOM_resultset ZOOM_resultset_create(void)
         YAZ_SHPTR_INIT(r->record_wrbuf, w);
     }
 #endif
+    resultset_use(1);
+    r->mc_key = 0;
+    r->live_set = 0;
     return r;
 }
 
@@ -902,19 +736,17 @@ ZOOM_API(ZOOM_resultset)
 {
     ZOOM_resultset r = ZOOM_resultset_create();
     ZOOM_task task;
-    const char *cp;
     int start, count;
-    const char *syntax, *elementSetName;
-#if ZOOM_RESULT_LISTS
-    ZOOM_resultsets set;
-#endif
-
+    const char *syntax, *elementSetName, *schema;
     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
-    r->r_sort_spec = q->sort_spec;
+    r->r_sort_spec = ZOOM_query_get_sortspec(q);
     r->query = q;
+    ZOOM_query_addref(q);
 
     r->options = ZOOM_options_create_with_parent(c->options);
-    
+
+    r->req_facets = odr_strdup_null(r->odr,
+                                    ZOOM_options_get(r->options, "facets"));
     start = ZOOM_options_get_int(r->options, "start", 0);
     count = ZOOM_options_get_int(r->options, "count", 0);
     {
@@ -924,29 +756,17 @@ ZOOM_API(ZOOM_resultset)
                                        (cp != 0 ? "presentChunk": "step"), 0);
     }
     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
-    cp = ZOOM_options_get(r->options, "setname");
-    if (cp)
-        r->setname = xstrdup(cp);
-    cp = ZOOM_options_get(r->options, "schema");
-    if (cp)
-        r->schema = xstrdup(cp);
-
-    r->databaseNames = ZOOM_connection_get_databases(c, c->options, &r->num_databaseNames,
-                                         r->odr);
-    
+    r->setname = odr_strdup_null(r->odr,
+                                 ZOOM_options_get(r->options, "setname"));
+    r->databaseNames = ZOOM_connection_get_databases(c, c->options,
+                                                     &r->num_databaseNames,
+                                                     r->odr);
     r->connection = c;
-
-#if ZOOM_RESULT_LISTS
-    yaz_log(log_details, "%p ZOOM_connection_search: Adding new resultset (%p) to resultsets (%p) ", c, r, c->resultsets);
-    set = xmalloc(sizeof(*set));
-    ZOOM_resultset_addref(r);
-    set->resultset = r;
-    set->next = c->resultsets;
-    c->resultsets = set;
-#else
     r->next = c->resultsets;
     c->resultsets = r;
-#endif
+
+    ZOOM_memcached_resultset(r, q);
+
     if (c->host_port && c->proto == PROTO_HTTP)
     {
         if (!c->cs)
@@ -965,17 +785,16 @@ ZOOM_API(ZOOM_resultset)
     task->u.search.resultset = r;
     task->u.search.start = start;
     task->u.search.count = count;
-    task->u.search.recv_search_fired = 0;
 
-    syntax = ZOOM_options_get(r->options, "preferredRecordSyntax"); 
+    syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
     elementSetName = ZOOM_options_get(r->options, "elementSetName");
-    task->u.search.elementSetName = elementSetName 
-        ? xstrdup(elementSetName) : 0;
-   
-    ZOOM_resultset_addref(r);
+    task->u.search.elementSetName = elementSetName ?
+        xstrdup(elementSetName) : 0;
+    schema = ZOOM_options_get(r->options, "schema");
+    task->u.search.schema = schema ? xstrdup(schema) : 0;
 
-    (q->refcount)++;
+    ZOOM_resultset_addref(r);
 
     if (!c->async)
     {
@@ -1023,13 +842,13 @@ ZOOM_API(int)
             c->reconnect_ok = 1;
         }
     }
-    
+
     ZOOM_resultset_cache_reset(r);
     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
     task->u.sort.resultset = r;
     task->u.sort.q = newq;
 
-    ZOOM_resultset_addref(r);  
+    ZOOM_resultset_addref(r);
 
     if (!c->async)
     {
@@ -1040,38 +859,6 @@ ZOOM_API(int)
     return 0;
 }
 
-static void ZOOM_record_release(ZOOM_record rec)
-{
-    if (!rec)
-        return;
-
-#if SHPTR
-    if (rec->record_wrbuf)
-        YAZ_SHPTR_DEC(rec->record_wrbuf, wrbuf_destroy);
-#else
-    if (rec->wrbuf)
-        wrbuf_destroy(rec->wrbuf);
-#endif
-
-    if (rec->odr)
-        odr_destroy(rec->odr);
-}
-
-ZOOM_API(void)
-    ZOOM_resultset_cache_reset(ZOOM_resultset r)
-{
-    int i;
-    for (i = 0; i<RECORD_HASH_SIZE; i++)
-    {
-        ZOOM_record_cache rc;
-        for (rc = r->record_hash[i]; rc; rc = rc->next)
-        {
-            ZOOM_record_release(&rc->rec);
-        }
-        r->record_hash[i] = 0;
-    }
-}
-
 ZOOM_API(void)
     ZOOM_resultset_destroy(ZOOM_resultset r)
 {
@@ -1096,12 +883,12 @@ static void resultset_destroy(ZOOM_resultset r)
         ZOOM_query_destroy(r->query);
         ZOOM_options_destroy(r->options);
         odr_destroy(r->odr);
-        xfree(r->setname);
-        xfree(r->schema);
         yaz_mutex_destroy(&r->mutex);
 #if SHPTR
         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
 #endif
+        wrbuf_destroy(r->mc_key);
+        resultset_use(-1);
         xfree(r);
     }
     else
@@ -1161,26 +948,19 @@ static void ZOOM_resultset_retrieve(ZOOM_resultset r,
             c->reconnect_ok = 1;
         }
     }
-    task = ZOOM_connection_add_task(c, ZOOM_TASK_RETRIEVE);
-    task->u.retrieve.resultset = r;
-    task->u.retrieve.start = start;
-    task->u.retrieve.count = count;
+    task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
+    task->u.search.resultset = r;
+    task->u.search.start = start;
+    task->u.search.count = count;
 
-    syntax = ZOOM_options_get(r->options, "preferredRecordSyntax"); 
-    task->u.retrieve.syntax = syntax ? xstrdup(syntax) : 0;
+    syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
+    task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
     elementSetName = ZOOM_options_get(r->options, "elementSetName");
-    task->u.retrieve.elementSetName = elementSetName 
+    task->u.search.elementSetName = elementSetName
         ? xstrdup(elementSetName) : 0;
 
     cp = ZOOM_options_get(r->options, "schema");
-    if (cp)
-    {
-        if (!r->schema || strcmp(r->schema, cp))
-        {
-            xfree(r->schema);
-            r->schema = xstrdup(cp);
-        }
-    }
+    task->u.search.schema = cp ? xstrdup(cp) : 0;
 
     ZOOM_resultset_addref(r);
 
@@ -1211,32 +991,41 @@ ZOOM_API(void)
 }
 
 ZOOM_API(size_t)
-    ZOOM_resultset_facets_size(ZOOM_resultset r) {
-    return r->num_facets;
+    ZOOM_resultset_facets_size(ZOOM_resultset r)
+{
+    return r->num_res_facets;
 }
 
 ZOOM_API(ZOOM_facet_field)
-    ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name) {
-    int num = r->num_facets;
-    ZOOM_facet_field *facets = r->facets;
-    int index;
-    for (index = 0; index < num; index++) {
-        if (!strcmp(facets[index]->facet_name, name)) {
-            return facets[index];
-        }
-    }
+    ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name)
+{
+    int num = r->num_res_facets;
+    ZOOM_facet_field *facets = r->res_facets;
+    int i;
+    for (i = 0; i < num; i++)
+        if (!strcmp(facets[i]->facet_name, name))
+            return facets[i];
     return 0;
 }
 
+ZOOM_API(ZOOM_facet_field)
+    ZOOM_resultset_get_facet_field_by_index(ZOOM_resultset r, int idx)
+{
+    int num = r->num_res_facets;
+    ZOOM_facet_field *facets = r->res_facets;
+    if (idx >= 0 && idx < num)
+        return facets[idx];
+    return 0;
+}
 
 ZOOM_API(ZOOM_facet_field *)
     ZOOM_resultset_facets(ZOOM_resultset r)
 {
-    return r->facets;
+    return r->res_facets;
 }
 
 ZOOM_API(const char**)
-    ZOOM_resultset_facet_names(ZOOM_resultset r)
+    ZOOM_resultset_facets_names(ZOOM_resultset r)
 {
     return (const char **) r->facets_names;
 }
@@ -1254,7 +1043,8 @@ ZOOM_API(size_t)
 }
 
 ZOOM_API(const char*)
-    ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq) {
+    ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq)
+{
     *freq = field->facet_terms[idx].frequency;
     return field->facet_terms[idx].term;
 }
@@ -1264,7 +1054,7 @@ static void get_cert(ZOOM_connection c)
 {
     char *cert_buf;
     int cert_len;
-    
+
     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
     {
         ZOOM_connection_option_setl(c, "sslPeerCert",
@@ -1274,45 +1064,29 @@ static void get_cert(ZOOM_connection c)
 }
 
 static zoom_ret do_connect_host(ZOOM_connection c,
-                                const char *effective_host,
                                 const char *logical_url);
 
 static zoom_ret do_connect(ZOOM_connection c)
 {
-    const char *effective_host;
-
-    if (c->proxy)
-        effective_host = c->proxy;
-    else
-        effective_host = c->host_port;
-    return do_connect_host(c, effective_host, c->host_port);
+    return do_connect_host(c, c->host_port);
 }
 
-static zoom_ret do_connect_host(ZOOM_connection c, const char *effective_host,
-    const char *logical_url)
+static zoom_ret do_connect_host(ZOOM_connection c, const char *logical_url)
 {
     void *add;
 
-    yaz_log(c->log_details, "%p do_connect effective_host=%s", c, effective_host);
-
     if (c->cs)
         cs_close(c->cs);
-    c->cs = cs_create_host(effective_host, 0, &add);
+    c->cs = cs_create_host2(logical_url, CS_FLAGS_DNS_NO_BLOCK, &add,
+                            c->tproxy ? c->tproxy : c->proxy,
+                            &c->proxy_mode);
+    if (!c->proxy)
+        c->proxy_mode = 0;
 
     if (c->cs && c->cs->protocol == PROTO_HTTP)
     {
 #if YAZ_HAVE_XML2
-        if (logical_url)
-        {
-            const char *db = 0;
-            
-            c->proto = PROTO_HTTP;
-            cs_get_host_args(logical_url, &db);
-            xfree(c->path);
-            
-            c->path = xmalloc(strlen(db) * 3 + 2);
-            yaz_encode_sru_dbpath_buf(c->path, db);
-        }
+        c->proto = PROTO_HTTP;
 #else
         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
         ZOOM_connection_close(c);
@@ -1335,7 +1109,11 @@ static zoom_ret do_connect_host(ZOOM_connection c, const char *effective_host,
                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
                 ZOOM_connection_remove_task(c);
                 ZOOM_connection_set_mask(c, 0);
-                ZOOM_connection_exec_task(c);
+
+                if (c->cs && c->location)
+                    send_HTTP_redirect(c, c->location);
+                else
+                    ZOOM_connection_exec_task(c);
             }
             c->state = STATE_ESTABLISHED;
             return zoom_pending;
@@ -1348,7 +1126,7 @@ static zoom_ret do_connect_host(ZOOM_connection c, const char *effective_host,
             if (c->cs->io_pending & CS_WANT_READ)
                 mask += ZOOM_SELECT_READ;
             ZOOM_connection_set_mask(c, mask);
-            c->state = STATE_CONNECTING; 
+            c->state = STATE_CONNECTING;
             return zoom_pending;
         }
     }
@@ -1358,196 +1136,20 @@ static zoom_ret do_connect_host(ZOOM_connection c, const char *effective_host,
 }
 
 /* returns 1 if PDU was sent OK (still pending )
-   0 if PDU was not sent OK (nothing to wait for) 
+   0 if PDU was not sent OK (nothing to wait for)
 */
 
-#if YAZ_HAVE_XML2
-static zoom_ret send_srw(ZOOM_connection c, Z_SRW_PDU *sr)
+ZOOM_API(ZOOM_record)
+    ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
 {
-    Z_GDU *gdu;
-    ZOOM_Event event;
-    const char *database =  ZOOM_options_get(c->options, "databaseName");
-    char *fdatabase = 0;
-    
-    if (database)
-        fdatabase = yaz_encode_sru_dbpath_odr(c->odr_out, database);
-    gdu = z_get_HTTP_Request_host_path(c->odr_out, c->host_port,
-                                       fdatabase ? fdatabase : c->path);
+    const char *syntax =
+        ZOOM_options_get(s->options, "preferredRecordSyntax");
+    const char *elementSetName =
+        ZOOM_options_get(s->options, "elementSetName");
+    const char *schema =
+        ZOOM_options_get(s->options, "schema");
 
-    if (c->sru_mode == zoom_sru_get)
-    {
-        yaz_sru_get_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
-    }
-    else if (c->sru_mode == zoom_sru_post)
-    {
-        yaz_sru_post_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
-    }
-    else if (c->sru_mode == zoom_sru_soap)
-    {
-        yaz_sru_soap_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
-    }
-    if (!z_GDU(c->odr_out, &gdu, 0, 0))
-        return zoom_complete;
-    if (c->odr_print)
-        z_GDU(c->odr_print, &gdu, 0, 0);
-    c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
-        
-    event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
-    ZOOM_connection_put_event(c, event);
-    odr_reset(c->odr_out);
-    return ZOOM_send_buf(c);
-}
-#endif
-
-#if YAZ_HAVE_XML2
-static Z_SRW_PDU *ZOOM_srw_get_pdu(ZOOM_connection c, int type)
-{
-    Z_SRW_PDU *sr = yaz_srw_get_pdu(c->odr_out, type, c->sru_version);
-    sr->username = c->user;
-    sr->password = c->password;
-    return sr;
-}
-#endif
-
-#if YAZ_HAVE_XML2
-static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
-{
-    int i;
-    int *start, *count;
-    ZOOM_resultset resultset = 0;
-    Z_SRW_PDU *sr = 0;
-    const char *option_val = 0;
-
-    if (c->error)                  /* don't continue on error */
-        return zoom_complete;
-    assert(c->tasks);
-    switch(c->tasks->which)
-    {
-    case ZOOM_TASK_SEARCH:
-        resultset = c->tasks->u.search.resultset;
-        if (!resultset->setname)
-            resultset->setname = xstrdup("default");
-        ZOOM_options_set(resultset->options, "setname", resultset->setname);
-        start = &c->tasks->u.search.start;
-        count = &c->tasks->u.search.count;
-        break;
-    case ZOOM_TASK_RETRIEVE:
-        resultset = c->tasks->u.retrieve.resultset;
-
-        start = &c->tasks->u.retrieve.start;
-        count = &c->tasks->u.retrieve.count;
-
-        if (*start >= resultset->size)
-            return zoom_complete;
-        if (*start + *count > resultset->size)
-            *count = resultset->size - *start;
-
-        for (i = 0; i < *count; i++)
-        {
-            ZOOM_record rec =
-                ZOOM_record_cache_lookup(resultset, i + *start,
-                                         c->tasks->u.retrieve.syntax,
-                                         c->tasks->u.retrieve.elementSetName);
-            if (!rec)
-                break;
-            else
-            {
-                ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
-                ZOOM_connection_put_event(c, event);
-            }
-        }
-        *start += i;
-        *count -= i;
-
-        if (*count == 0)
-            return zoom_complete;
-        break;
-    default:
-        return zoom_complete;
-    }
-    assert(resultset->query);
-        
-    sr = ZOOM_srw_get_pdu(c, Z_SRW_searchRetrieve_request);
-    if (resultset->query->z_query->which == Z_Query_type_104
-        && resultset->query->z_query->u.type_104->which == Z_External_CQL)
-    {
-        sr->u.request->query_type = Z_SRW_query_type_cql;
-        sr->u.request->query.cql =resultset->query->z_query->u.type_104->u.cql;
-    }
-    else if (resultset->query->z_query->which == Z_Query_type_1 &&
-             resultset->query->z_query->u.type_1)
-    {
-        sr->u.request->query_type = Z_SRW_query_type_pqf;
-        sr->u.request->query.pqf = resultset->query->query_string;
-    }
-    else
-    {
-        ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_QUERY, 0);
-        return zoom_complete;
-    }
-    sr->u.request->startRecord = odr_intdup(c->odr_out, *start + 1);
-    sr->u.request->maximumRecords = odr_intdup(
-        c->odr_out, (resultset->step > 0 && resultset->step < *count) ? 
-        resultset->step : *count);
-    sr->u.request->recordSchema = resultset->schema;
-    
-    option_val = ZOOM_resultset_option_get(resultset, "recordPacking");
-    if (option_val)
-        sr->u.request->recordPacking = odr_strdup(c->odr_out, option_val);
-
-    option_val = ZOOM_resultset_option_get(resultset, "extraArgs");
-    yaz_encode_sru_extra(sr, c->odr_out, option_val);
-    return send_srw(c, sr);
-}
-#else
-static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
-{
-    return zoom_complete;
-}
-#endif
-
-ZOOM_API(ZOOM_record)
-    ZOOM_record_clone(ZOOM_record srec)
-{
-    char *buf;
-    int size;
-    ODR odr_enc;
-    ZOOM_record nrec;
-
-    odr_enc = odr_createmem(ODR_ENCODE);
-    if (!z_NamePlusRecord(odr_enc, &srec->npr, 0, 0))
-        return 0;
-    buf = odr_getbuf(odr_enc, &size, 0);
-    
-    nrec = (ZOOM_record) xmalloc(sizeof(*nrec));
-    yaz_log(log_details0, "ZOOM_record create");
-    nrec->odr = odr_createmem(ODR_DECODE);
-#if SHPTR
-    nrec->record_wrbuf = 0;
-#else
-    nrec->wrbuf = 0;
-#endif
-    odr_setbuf(nrec->odr, buf, size, 0);
-    z_NamePlusRecord(nrec->odr, &nrec->npr, 0, 0);
-    
-    nrec->schema = odr_strdup_null(nrec->odr, srec->schema);
-    nrec->diag_uri = odr_strdup_null(nrec->odr, srec->diag_uri);
-    nrec->diag_message = odr_strdup_null(nrec->odr, srec->diag_message);
-    nrec->diag_details = odr_strdup_null(nrec->odr, srec->diag_details);
-    nrec->diag_set = odr_strdup_null(nrec->odr, srec->diag_set);
-    odr_destroy(odr_enc);
-    return nrec;
-}
-
-ZOOM_API(ZOOM_record)
-    ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
-{
-    const char *syntax =
-        ZOOM_options_get(s->options, "preferredRecordSyntax"); 
-    const char *elementSetName =
-        ZOOM_options_get(s->options, "elementSetName");
-
-    return ZOOM_record_cache_lookup(s, pos, syntax, elementSetName);
+    return ZOOM_record_cache_lookup_i(s, pos, syntax, elementSetName, schema);
 }
 
 ZOOM_API(ZOOM_record)
@@ -1571,492 +1173,6 @@ ZOOM_API(ZOOM_record)
     return rec;
 }
 
-ZOOM_API(void)
-    ZOOM_record_destroy(ZOOM_record rec)
-{
-    ZOOM_record_release(rec);
-    yaz_log(log_details0, "ZOOM_record destroy");
-    xfree(rec);
-}
-
-
-static yaz_iconv_t iconv_create_charset(const char *record_charset)
-{
-    char to[40];
-    char from[40];
-    yaz_iconv_t cd = 0;
-
-    *from = '\0';
-    strcpy(to, "UTF-8");
-    if (record_charset && *record_charset)
-    {
-        /* Use "from,to" or just "from" */
-        const char *cp = strchr(record_charset, ',');
-        size_t clen = strlen(record_charset);
-        if (cp && cp[1])
-        {
-            strncpy( to, cp+1, sizeof(to)-1);
-            to[sizeof(to)-1] = '\0';
-            clen = cp - record_charset;
-        }
-        if (clen > sizeof(from)-1)
-            clen = sizeof(from)-1;
-        
-        if (clen)
-            strncpy(from, record_charset, clen);
-        from[clen] = '\0';
-    }
-    if (*from && *to)
-        cd = yaz_iconv_open(to, from);
-    return cd;
-}
-
-static const char *return_marc_record(WRBUF wrbuf,
-                                      int marc_type,
-                                      int *len,
-                                      const char *buf, int sz,
-                                      const char *record_charset)
-{
-    yaz_iconv_t cd = iconv_create_charset(record_charset);
-    yaz_marc_t mt = yaz_marc_create();
-    const char *ret_string = 0;
-
-    if (cd)
-        yaz_marc_iconv(mt, cd);
-    yaz_marc_xml(mt, marc_type);
-    if (yaz_marc_decode_wrbuf(mt, buf, sz, wrbuf) > 0)
-    {
-        if (len)
-            *len = wrbuf_len(wrbuf);
-        ret_string = wrbuf_cstr(wrbuf);
-    }
-    yaz_marc_destroy(mt);
-    if (cd)
-        yaz_iconv_close(cd);
-    return ret_string;
-}
-
-static const char *return_opac_record(WRBUF wrbuf,
-                                      int marc_type,
-                                      int *len,
-                                      Z_OPACRecord *opac_rec,
-                                      const char *record_charset)
-{
-    yaz_iconv_t cd = iconv_create_charset(record_charset);
-    yaz_marc_t mt = yaz_marc_create();
-
-    if (cd)
-        yaz_marc_iconv(mt, cd);
-    yaz_marc_xml(mt, marc_type);
-
-    yaz_opac_decode_wrbuf(mt, opac_rec, wrbuf);
-    yaz_marc_destroy(mt);
-
-    if (cd)
-        yaz_iconv_close(cd);
-    if (len)
-        *len = wrbuf_len(wrbuf);
-    return wrbuf_cstr(wrbuf);
-}
-
-static const char *return_string_record(WRBUF wrbuf,
-                                        int *len,
-                                        const char *buf, int sz,
-                                        const char *record_charset)
-{
-    yaz_iconv_t cd = iconv_create_charset(record_charset);
-
-    if (cd)
-    {
-        wrbuf_iconv_write(wrbuf, cd, buf, sz);
-        wrbuf_iconv_reset(wrbuf, cd);
-
-        buf = wrbuf_cstr(wrbuf);
-        sz = wrbuf_len(wrbuf);
-        yaz_iconv_close(cd);
-    }
-    if (len)
-        *len = sz;
-    return buf;
-}
-
-static const char *return_record_wrbuf(WRBUF wrbuf, int *len,
-                                       Z_NamePlusRecord *npr,
-                                       int marctype, const char *charset)
-{
-    Z_External *r = (Z_External *) npr->u.databaseRecord;
-    const Odr_oid *oid = r->direct_reference;
-
-    wrbuf_rewind(wrbuf);
-    /* render bibliographic record .. */
-    if (r->which == Z_External_OPAC)
-    {
-        return return_opac_record(wrbuf, marctype, len,
-                                  r->u.opac, charset);
-    }
-    if (r->which == Z_External_sutrs)
-        return return_string_record(wrbuf, len,
-                                    (char*) r->u.sutrs->buf,
-                                    r->u.sutrs->len,
-                                    charset);
-    else if (r->which == Z_External_octet)
-    {
-        if (yaz_oid_is_iso2709(oid))
-        {
-            const char *ret_buf = return_marc_record(
-                wrbuf, marctype, len,
-                (const char *) r->u.octet_aligned->buf,
-                r->u.octet_aligned->len,
-                charset);
-            if (ret_buf)
-                return ret_buf;
-            /* bad ISO2709. Return fail unless raw (ISO2709) is wanted */
-            if (marctype != YAZ_MARC_ISO2709)
-                return 0;
-        }
-        return return_string_record(wrbuf, len,
-                                    (const char *) r->u.octet_aligned->buf,
-                                    r->u.octet_aligned->len,
-                                    charset);
-    }
-    else if (r->which == Z_External_grs1)
-    {
-        yaz_display_grs1(wrbuf, r->u.grs1, 0);
-        return return_string_record(wrbuf, len,
-                                    wrbuf_buf(wrbuf),
-                                    wrbuf_len(wrbuf),
-                                    charset);
-    }
-    return 0;
-}
-    
-ZOOM_API(int)
-    ZOOM_record_error(ZOOM_record rec, const char **cp,
-                      const char **addinfo, const char **diagset)
-{
-    Z_NamePlusRecord *npr;
-    
-    if (!rec)
-        return 0;
-
-    npr = rec->npr;
-    if (rec->diag_uri)
-    {
-        if (cp)
-            *cp = rec->diag_message;
-        if (addinfo)
-            *addinfo = rec->diag_details;
-        if (diagset)
-            *diagset = rec->diag_set;
-        return uri_to_code(rec->diag_uri);
-    }
-    if (npr && npr->which == Z_NamePlusRecord_surrogateDiagnostic)
-    {
-        Z_DiagRec *diag_rec = npr->u.surrogateDiagnostic;
-        int error = YAZ_BIB1_UNSPECIFIED_ERROR;
-        const char *add = 0;
-
-        if (diag_rec->which == Z_DiagRec_defaultFormat)
-        {
-            Z_DefaultDiagFormat *ddf = diag_rec->u.defaultFormat;
-            oid_class oclass;
-    
-            error = *ddf->condition;
-            switch (ddf->which)
-            {
-            case Z_DefaultDiagFormat_v2Addinfo:
-                add = ddf->u.v2Addinfo;
-                break;
-            case Z_DefaultDiagFormat_v3Addinfo:
-                add = ddf->u.v3Addinfo;
-                break;
-            }
-            if (diagset)
-                *diagset =
-                    yaz_oid_to_string(yaz_oid_std(),
-                                      ddf->diagnosticSetId, &oclass);
-        }
-        else
-        {
-            if (diagset)
-                *diagset = "Bib-1";
-        }
-        if (addinfo)
-            *addinfo = add ? add : "";
-        if (cp)
-            *cp = diagbib1_str(error);
-        return error;
-    }
-    return 0;
-}
-
-static const char *get_record_format(WRBUF wrbuf, int *len,
-                                     Z_NamePlusRecord *npr,
-                                     int marctype, const char *charset,
-                                     const char *format)
-{
-    const char *res = return_record_wrbuf(wrbuf, len, npr, marctype, charset);
-#if YAZ_HAVE_XML2
-    if (*format == '1' && len)
-    {
-        /* try to XML format res */
-        xmlDocPtr doc;
-        xmlKeepBlanksDefault(0); /* get get xmlDocFormatMemory to work! */
-        doc = xmlParseMemory(res, *len);
-        if (doc)
-        {
-            xmlChar *xml_mem;
-            int xml_size;
-            xmlDocDumpFormatMemory(doc, &xml_mem, &xml_size, 1);
-            wrbuf_rewind(wrbuf);
-            wrbuf_write(wrbuf, (const char *) xml_mem, xml_size);
-            xmlFree(xml_mem);
-            xmlFreeDoc(doc);
-            res = wrbuf_cstr(wrbuf);
-            *len = wrbuf_len(wrbuf);
-        } 
-    }
-#endif
-    return res;
-}
-
-
-static const char *npr_format(Z_NamePlusRecord *npr, const char *schema,
-                              WRBUF wrbuf,
-                              const char *type_spec, int *len)
-{
-    size_t i;
-    char type[40];
-    char charset[40];
-    char format[3];
-    const char *cp = type_spec;
-
-    for (i = 0; cp[i] && cp[i] != ';' && cp[i] != ' ' && i < sizeof(type)-1;
-         i++)
-        type[i] = cp[i];
-    type[i] = '\0';
-    charset[0] = '\0';
-    format[0] = '\0';
-    while (1)
-    {
-        while (cp[i] == ' ')
-            i++;
-        if (cp[i] != ';')
-            break;
-        i++;
-        while (cp[i] == ' ')
-            i++;
-        if (!strncmp(cp + i, "charset=", 8))
-        {
-            size_t j = 0;
-            i = i + 8; /* skip charset= */
-            for (j = 0; cp[i] && cp[i] != ';' && cp[i] != ' '; i++)
-            {
-                if (j < sizeof(charset)-1)
-                    charset[j++] = cp[i];
-            }
-            charset[j] = '\0';
-        }
-        else if (!strncmp(cp + i, "format=", 7))
-        {
-            size_t j = 0; 
-            i = i + 7;
-            for (j = 0; cp[i] && cp[i] != ';' && cp[i] != ' '; i++)
-            {
-                if (j < sizeof(format)-1)
-                    format[j++] = cp[i];
-            }
-            format[j] = '\0';
-        } 
-    }
-    if (!strcmp(type, "database"))
-    {
-        if (len)
-            *len = (npr->databaseName ? strlen(npr->databaseName) : 0);
-        return npr->databaseName;
-    }
-    else if (!strcmp(type, "schema"))
-    {
-        if (len)
-            *len = schema ? strlen(schema) : 0;
-        return schema;
-    }
-    else if (!strcmp(type, "syntax"))
-    {
-        const char *desc = 0;   
-        if (npr->which == Z_NamePlusRecord_databaseRecord)
-        {
-            Z_External *r = (Z_External *) npr->u.databaseRecord;
-            desc = yaz_oid_to_string(yaz_oid_std(), r->direct_reference, 0);
-        }
-        if (!desc)
-            desc = "none";
-        if (len)
-            *len = strlen(desc);
-        return desc;
-    }
-    if (npr->which != Z_NamePlusRecord_databaseRecord)
-        return 0;
-
-    /* from now on - we have a database record .. */
-    if (!strcmp(type, "render"))
-    {
-        return get_record_format(wrbuf, len, npr, YAZ_MARC_LINE, charset, format);
-    }
-    else if (!strcmp(type, "xml"))
-    {
-        return get_record_format(wrbuf, len, npr, YAZ_MARC_MARCXML, charset,
-                                 format);
-    }
-    else if (!strcmp(type, "txml"))
-    {
-        return get_record_format(wrbuf, len, npr, YAZ_MARC_TURBOMARC, charset,
-                                 format);
-    }
-    else if (!strcmp(type, "raw"))
-    {
-        return get_record_format(wrbuf, len, npr, YAZ_MARC_ISO2709, charset,
-            format);
-    }
-    else if (!strcmp(type, "ext"))
-    {
-        if (len) *len = -1;
-        return (const char *) npr->u.databaseRecord;
-    }
-    else if (!strcmp(type, "opac"))
-    {
-        if (npr->u.databaseRecord->which == Z_External_OPAC)
-            return get_record_format(wrbuf, len, npr, YAZ_MARC_MARCXML, charset,
-                                     format);
-    }
-    return 0;
-}
-
-ZOOM_API(const char *)
-    ZOOM_record_get(ZOOM_record rec, const char *type_spec, int *len)
-{
-    WRBUF wrbuf;
-    
-    if (len)
-        *len = 0; /* default return */
-        
-    if (!rec || !rec->npr)
-        return 0;
-
-#if SHPTR
-    if (!rec->record_wrbuf)
-    {
-        WRBUF w = wrbuf_alloc();
-        YAZ_SHPTR_INIT(rec->record_wrbuf, w);
-    }
-    wrbuf = rec->record_wrbuf->ptr;
-#else
-    if (!rec->wrbuf)
-        rec->wrbuf = wrbuf_alloc();
-    wrbuf = rec->wrbuf;
-#endif
-    return npr_format(rec->npr, rec->schema, wrbuf, type_spec, len);
-}
-
-static int strcmp_null(const char *v1, const char *v2)
-{
-    if (!v1 && !v2)
-        return 0;
-    if (!v1 || !v2)
-        return -1;
-    return strcmp(v1, v2);
-}
-
-static size_t record_hash(int pos)
-{
-    if (pos < 0)
-        pos = 0;
-    return pos % RECORD_HASH_SIZE;
-}
-
-void ZOOM_record_cache_add(ZOOM_resultset r, Z_NamePlusRecord *npr, 
-                           int pos,
-                           const char *syntax, const char *elementSetName,
-                           const char *schema,
-                           Z_SRW_diagnostic *diag)
-{
-    ZOOM_record_cache rc = 0;
-    
-    ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
-    ZOOM_connection_put_event(r->connection, event);
-
-    for (rc = r->record_hash[record_hash(pos)]; rc; rc = rc->next)
-    {
-        if (pos == rc->pos 
-            && strcmp_null(r->schema, rc->schema) == 0
-            && strcmp_null(elementSetName,rc->elementSetName) == 0
-            && strcmp_null(syntax, rc->syntax) == 0)
-            break;
-    }
-    if (!rc)
-    {
-        rc = (ZOOM_record_cache) odr_malloc(r->odr, sizeof(*rc));
-        rc->rec.odr = 0;
-#if SHPTR
-        YAZ_SHPTR_INC(r->record_wrbuf);
-        rc->rec.record_wrbuf = r->record_wrbuf;
-#else
-        rc->rec.wrbuf = 0;
-#endif
-        rc->elementSetName = odr_strdup_null(r->odr, elementSetName);
-        
-        rc->syntax = odr_strdup_null(r->odr, syntax);
-        
-        rc->schema = odr_strdup_null(r->odr, r->schema);
-
-        rc->pos = pos;
-        rc->next = r->record_hash[record_hash(pos)];
-        r->record_hash[record_hash(pos)] = rc;
-    }
-    rc->rec.npr = npr;
-    rc->rec.schema = odr_strdup_null(r->odr, schema);
-    rc->rec.diag_set = 0;
-    rc->rec.diag_uri = 0;
-    rc->rec.diag_message = 0;
-    rc->rec.diag_details = 0;
-    if (diag)
-    {
-        if (diag->uri)
-        {
-            char *cp;
-            rc->rec.diag_set = odr_strdup(r->odr, diag->uri);
-            if ((cp = strrchr(rc->rec.diag_set, '/')))
-                *cp = '\0';
-            rc->rec.diag_uri = odr_strdup(r->odr, diag->uri);
-        }
-        rc->rec.diag_message = odr_strdup_null(r->odr, diag->message);            
-        rc->rec.diag_details = odr_strdup_null(r->odr, diag->details);
-    }
-}
-
-ZOOM_record ZOOM_record_cache_lookup(ZOOM_resultset r, int pos,
-                                     const char *syntax,
-                                     const char *elementSetName)
-{
-    ZOOM_record_cache rc;
-    
-    for (rc = r->record_hash[record_hash(pos)]; rc; rc = rc->next)
-    {
-        if (pos == rc->pos)
-        {
-            if (strcmp_null(r->schema, rc->schema))
-                continue;
-            if (strcmp_null(elementSetName,rc->elementSetName))
-                continue;
-            if (strcmp_null(syntax, rc->syntax))
-                continue;
-            return &rc->rec;
-        }
-    }
-    return 0;
-}
-                                             
-
 ZOOM_API(ZOOM_scanset)
     ZOOM_connection_scan(ZOOM_connection c, const char *start)
 {
@@ -2075,8 +1191,9 @@ ZOOM_API(ZOOM_scanset)
     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
 {
     ZOOM_scanset scan = 0;
+    Z_Query *z_query = ZOOM_query_get_Z_Query(q);
 
-    if (!q->z_query)
+    if (!z_query)
         return 0;
     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
     scan->connection = c;
@@ -2087,7 +1204,7 @@ ZOOM_API(ZOOM_scanset)
     scan->srw_scan_response = 0;
 
     scan->query = q;
-    (q->refcount)++;
+    ZOOM_query_addref(q);
     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
                                             &scan->num_databaseNames,
                                             scan->odr);
@@ -2096,7 +1213,7 @@ ZOOM_API(ZOOM_scanset)
     {
         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
         task->u.scan.scan = scan;
-        
+
         (scan->refcount)++;
         if (!c->async)
         {
@@ -2118,7 +1235,7 @@ ZOOM_API(void)
         ZOOM_query_destroy(scan->query);
 
         odr_destroy(scan->odr);
-        
+
         ZOOM_options_destroy(scan->options);
         xfree(scan);
     }
@@ -2132,66 +1249,16 @@ static zoom_ret send_package(ZOOM_connection c)
     if (!c->tasks)
         return zoom_complete;
     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
-    
+
     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
     ZOOM_connection_put_event(c, event);
-    
+
     c->buf_out = c->tasks->u.package->buf_out;
     c->len_out = c->tasks->u.package->len_out;
 
     return ZOOM_send_buf(c);
 }
 
-#if YAZ_HAVE_XML2
-static zoom_ret ZOOM_connection_srw_send_scan(ZOOM_connection c)
-{
-    ZOOM_scanset scan;
-    Z_SRW_PDU *sr = 0;
-    const char *option_val = 0;
-
-    if (!c->tasks)
-        return zoom_complete;
-    assert (c->tasks->which == ZOOM_TASK_SCAN);
-    scan = c->tasks->u.scan.scan;
-        
-    sr = ZOOM_srw_get_pdu(c, Z_SRW_scan_request);
-
-    /* SRU scan can only carry CQL and PQF */
-    if (scan->query->z_query->which == Z_Query_type_104)
-    {
-        sr->u.scan_request->query_type = Z_SRW_query_type_cql;
-        sr->u.scan_request->scanClause.cql = scan->query->query_string;
-    }
-    else if (scan->query->z_query->which == Z_Query_type_1
-             || scan->query->z_query->which == Z_Query_type_101)
-    {
-        sr->u.scan_request->query_type = Z_SRW_query_type_pqf;
-        sr->u.scan_request->scanClause.pqf = scan->query->query_string;
-    }
-    else
-    {
-        ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_QUERY, 0);
-        return zoom_complete;
-    }
-
-    sr->u.scan_request->maximumTerms = odr_intdup(
-        c->odr_out, ZOOM_options_get_int(scan->options, "number", 10));
-    
-    sr->u.scan_request->responsePosition = odr_intdup(
-        c->odr_out, ZOOM_options_get_int(scan->options, "position", 1));
-    
-    option_val = ZOOM_options_get(scan->options, "extraArgs");
-    yaz_encode_sru_extra(sr, c->odr_out, option_val);
-    return send_srw(c, sr);
-}
-#else
-static zoom_ret ZOOM_connection_srw_send_scan(ZOOM_connection c)
-{
-    return zoom_complete;
-}
-#endif
-
-
 ZOOM_API(size_t)
     ZOOM_scanset_size(ZOOM_scanset scan)
 {
@@ -2211,7 +1278,7 @@ static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
                                 const char **disp_term, size_t *disp_len)
 {
     size_t noent = ZOOM_scanset_size(scan);
-    
+
     *value_term = 0;
     *value_len = 0;
 
@@ -2227,7 +1294,7 @@ static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
         {
             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
-            
+
             *value_term = (const char *) t->term->u.general->buf;
             *value_len = t->term->u.general->len;
             if (t->displayTerm)
@@ -2273,7 +1340,7 @@ ZOOM_API(const char *)
 
     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
                         &disp_term, &disp_len);
-    
+
     *len = value_len;
     return value_term;
 }
@@ -2289,7 +1356,7 @@ ZOOM_API(const char *)
 
     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
                         &disp_term, &disp_len);
-    
+
     *len = disp_len;
     return disp_term;
 }
@@ -2332,7 +1399,7 @@ ZOOM_API(void)
     {
         odr_destroy(p->odr_out);
         xfree(p->buf_out);
-        
+
         ZOOM_options_destroy(p->options);
         xfree(p);
     }
@@ -2397,13 +1464,7 @@ ZOOM_API(int)
             if (c->proto == PROTO_HTTP)
                 ret = ZOOM_connection_srw_send_search(c);
             else
-                ret = ZOOM_connection_Z3950_send_search(c);
-            break;
-        case ZOOM_TASK_RETRIEVE:
-            if (c->proto == PROTO_HTTP)
-                ret = ZOOM_connection_srw_send_search(c);
-            else
-                ret = send_Z3950_present(c);
+                ret = ZOOM_connection_Z3950_search(c);
             break;
         case ZOOM_TASK_CONNECT:
             ret = do_connect(c);
@@ -2418,8 +1479,8 @@ ZOOM_API(int)
             ret = send_package(c);
             break;
         case ZOOM_TASK_SORT:
-            c->tasks->u.sort.resultset->r_sort_spec = 
-                c->tasks->u.sort.q->sort_spec;
+            c->tasks->u.sort.resultset->r_sort_spec =
+                ZOOM_query_get_sortspec(c->tasks->u.sort.q);
             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
             break;
         }
@@ -2442,248 +1503,73 @@ ZOOM_API(int)
     return 1;
 }
 
-#if YAZ_HAVE_XML2
-static zoom_ret handle_srw_response(ZOOM_connection c,
-                                    Z_SRW_searchRetrieveResponse *res)
-{
-    ZOOM_resultset resultset = 0;
-    int i;
-    NMEM nmem;
-    ZOOM_Event event;
-    int *start, *count;
-    const char *syntax, *elementSetName;
-
-    if (!c->tasks)
-        return zoom_complete;
-
-    switch(c->tasks->which)
-    {
-    case ZOOM_TASK_SEARCH:
-        resultset = c->tasks->u.search.resultset;
-        start = &c->tasks->u.search.start;
-        count = &c->tasks->u.search.count;
-        syntax = c->tasks->u.search.syntax;
-        elementSetName = c->tasks->u.search.elementSetName;        
-
-        if (!c->tasks->u.search.recv_search_fired)
-        {
-            event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
-            ZOOM_connection_put_event(c, event);
-            c->tasks->u.search.recv_search_fired = 1;
-        }
-        break;
-    case ZOOM_TASK_RETRIEVE:
-        resultset = c->tasks->u.retrieve.resultset;
-        start = &c->tasks->u.retrieve.start;
-        count = &c->tasks->u.retrieve.count;
-        syntax = c->tasks->u.retrieve.syntax;
-        elementSetName = c->tasks->u.retrieve.elementSetName;
-        break;
-    default:
-        return zoom_complete;
-    }
-
-    resultset->size = 0;
-
-    if (res->resultSetId)
-        ZOOM_resultset_option_set(resultset, "resultSetId", res->resultSetId);
-
-    yaz_log(c->log_details, "%p handle_srw_response got SRW response OK", c);
-
-    if (res->num_diagnostics > 0)
-    {
-        set_SRU_error(c, &res->diagnostics[0]);
-    }
-    else
-    {
-        if (res->numberOfRecords)
-            resultset->size = *res->numberOfRecords;
-        for (i = 0; i<res->num_records; i++)
-        {
-            int pos;
-            Z_SRW_record *sru_rec;
-            Z_SRW_diagnostic *diag = 0;
-            int num_diag;
-            
-            Z_NamePlusRecord *npr = (Z_NamePlusRecord *)
-                odr_malloc(c->odr_in, sizeof(Z_NamePlusRecord));
-            
-            if (res->records[i].recordPosition && 
-                *res->records[i].recordPosition > 0)
-                pos = *res->records[i].recordPosition - 1;
-            else
-                pos = *start + i;
-            
-            sru_rec = &res->records[i];
-            
-            npr->databaseName = 0;
-            npr->which = Z_NamePlusRecord_databaseRecord;
-            npr->u.databaseRecord = (Z_External *)
-                odr_malloc(c->odr_in, sizeof(Z_External));
-            npr->u.databaseRecord->descriptor = 0;
-            npr->u.databaseRecord->direct_reference =
-                odr_oiddup(c->odr_in, yaz_oid_recsyn_xml);
-            npr->u.databaseRecord->which = Z_External_octet;
-            
-            npr->u.databaseRecord->u.octet_aligned = (Odr_oct *)
-                odr_malloc(c->odr_in, sizeof(Odr_oct));
-            npr->u.databaseRecord->u.octet_aligned->buf = (unsigned char*)
-                sru_rec->recordData_buf;
-            npr->u.databaseRecord->u.octet_aligned->len = 
-                npr->u.databaseRecord->u.octet_aligned->size = 
-                sru_rec->recordData_len;
-            
-            if (sru_rec->recordSchema 
-                && !strcmp(sru_rec->recordSchema,
-                           "info:srw/schema/1/diagnostics-v1.1"))
-            {
-                sru_decode_surrogate_diagnostics(sru_rec->recordData_buf,
-                                                 sru_rec->recordData_len,
-                                                 &diag, &num_diag,
-                                                 resultset->odr);
-            }
-            ZOOM_record_cache_add(resultset, npr, pos, syntax, elementSetName,
-                                  sru_rec->recordSchema, diag);
-        }
-        *count -= i;
-        *start += i;
-        if (*count + *start > resultset->size)
-            *count = resultset->size - *start;
-        if (*count < 0)
-            *count = 0;
-        
-        nmem = odr_extract_mem(c->odr_in);
-        nmem_transfer(odr_getmem(resultset->odr), nmem);
-        nmem_destroy(nmem);
-
-        if (*count > 0)
-            return ZOOM_connection_srw_send_search(c);
-    }
-    return zoom_complete;
-}
-#endif
-
-#if YAZ_HAVE_XML2
-static void handle_srw_scan_response(ZOOM_connection c,
-                                     Z_SRW_scanResponse *res)
-{
-    NMEM nmem = odr_extract_mem(c->odr_in);
-    ZOOM_scanset scan;
-
-    if (!c->tasks || c->tasks->which != ZOOM_TASK_SCAN)
-        return;
-    scan = c->tasks->u.scan.scan;
-
-    if (res->num_diagnostics > 0)
-        set_SRU_error(c, &res->diagnostics[0]);
-
-    scan->scan_response = 0;
-    scan->srw_scan_response = res;
-    nmem_transfer(odr_getmem(scan->odr), nmem);
-
-    ZOOM_options_set_int(scan->options, "number", res->num_terms);
-    nmem_destroy(nmem);
-}
-#endif
-
-#if YAZ_HAVE_XML2
-static Z_GDU *get_HTTP_Request_url(ODR odr, const char *url)
-{
-    Z_GDU *p = z_get_HTTP_Request(odr);
-    const char *host = url;
-    const char *cp0 = strstr(host, "://");
-    const char *cp1 = 0;
-    if (cp0)
-        cp0 = cp0+3;
-    else
-        cp0 = host;
-    
-    cp1 = strchr(cp0, '/');
-    if (!cp1)
-        cp1 = cp0 + strlen(cp0);
-    
-    if (cp0 && cp1)
-    {
-        char *h = (char*) odr_malloc(odr, cp1 - cp0 + 1);
-        memcpy (h, cp0, cp1 - cp0);
-        h[cp1-cp0] = '\0';
-        z_HTTP_header_add(odr, &p->u.HTTP_Request->headers, "Host", h);
-    }
-    p->u.HTTP_Request->path = odr_strdup(odr, *cp1 ? cp1 : "/");
-    return p;
-}
 
-static zoom_ret send_SRW_redirect(ZOOM_connection c, const char *uri,
-                                  Z_HTTP_Response *cookie_hres)
+static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri)
 {
-    struct Z_HTTP_Header *h;
-    Z_GDU *gdu = get_HTTP_Request_url(c->odr_out, uri);
-    char *combined_cookies;
-    int combined_cookies_len = 0;
+    Z_GDU *gdu = z_get_HTTP_Request_uri(c->odr_out, uri, 0, c->proxy ? 1 : 0);
 
     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
                       "text/xml");
-
-    for (h = cookie_hres->headers; h; h = h->next)
-    {
-        if (!strcmp(h->name, "Set-Cookie"))
-        {
-            char *cp;
-
-            if (!(cp = strchr(h->value, ';')))
-                cp = h->value + strlen(h->value);
-            if (cp - h->value >= 1) {
-                combined_cookies = xrealloc(combined_cookies, combined_cookies_len + cp - h->value + 3);
-                memcpy(combined_cookies+combined_cookies_len, h->value, cp - h->value);
-                combined_cookies[combined_cookies_len + cp - h->value] = '\0';
-                strcat(combined_cookies,"; ");
-                combined_cookies_len = strlen(combined_cookies);
-            }
-        }
-    }
-
-    if (combined_cookies_len)
-    {
-        z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
-                          "Cookie", combined_cookies);
-        xfree(combined_cookies);
-    }
-
+    yaz_cookies_request(c->cookies, c->odr_out, gdu->u.HTTP_Request);
     if (c->user && c->password)
     {
         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
                                      c->user, c->password);
     }
-    if (!z_GDU(c->odr_out, &gdu, 0, 0))
+    xfree(c->location);
+    c->location = 0;
+    return ZOOM_send_GDU(c, gdu);
+}
+
+
+zoom_ret ZOOM_send_GDU(ZOOM_connection c, Z_GDU *gdu)
+{
+    ZOOM_Event event;
+
+    int r = z_GDU(c->odr_out, &gdu, 0, 0);
+    if (!r)
         return zoom_complete;
     if (c->odr_print)
         z_GDU(c->odr_print, &gdu, 0, 0);
+    if (c->odr_save)
+        z_GDU(c->odr_save, &gdu, 0, 0);
     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
-
     odr_reset(c->odr_out);
+
+    event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
+    ZOOM_connection_put_event(c, event);
+
     return ZOOM_send_buf(c);
 }
 
+void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
+                         const char *addinfo, const char *addinfo2)
+{
+    ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
+}
+
+#if YAZ_HAVE_XML2
 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
 {
     zoom_ret cret = zoom_complete;
     int ret = -1;
-    const char *addinfo = 0;
+    char *addinfo = 0;
     const char *connection_head = z_HTTP_header_lookup(hres->headers,
                                                        "Connection");
     const char *location;
 
     ZOOM_connection_set_mask(c, 0);
     yaz_log(c->log_details, "%p handle_http", c);
-    
+
+    yaz_cookies_response(c->cookies, hres);
     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
     {
         c->no_redirects++;
         if (c->no_redirects > 10)
         {
-            set_HTTP_error(c, hres->code, 0, 0);
+            ZOOM_set_HTTP_error(c, hres->code, 0, 0);
             c->no_redirects = 0;
             ZOOM_connection_close(c);
         }
@@ -2691,53 +1577,19 @@ static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
         {
             /* since redirect may change host we just reconnect. A smarter
                implementation might check whether it's the same server */
-            do_connect_host(c, location, 0);
-            send_SRW_redirect(c, location, hres);
-            /* we're OK for now. Operation is not really complete */
-            ret = 0;
-            cret = zoom_pending;
+
+            int host_change = 0;
+            location = yaz_check_location(c->odr_in, c->host_port,
+                                          location, &host_change);
+            xfree(c->location);
+            c->location = xstrdup(location);
+            do_connect_host(c, location);
+            return;
         }
     }
-    else 
-    {   /* not redirect (normal response) */
-        if (!yaz_srw_check_content_type(hres))
-            addinfo = "content-type";
-        else
-        {
-            Z_SOAP *soap_package = 0;
-            ODR o = c->odr_in;
-            Z_SOAP_Handler soap_handlers[2] = {
-                {YAZ_XMLNS_SRU_v1_1, 0, (Z_SOAP_fun) yaz_srw_codec},
-                {0, 0, 0}
-            };
-            ret = z_soap_codec(o, &soap_package,
-                               &hres->content_buf, &hres->content_len,
-                               soap_handlers);
-            if (!ret && soap_package->which == Z_SOAP_generic &&
-                soap_package->u.generic->no == 0)
-            {
-                Z_SRW_PDU *sr = (Z_SRW_PDU*) soap_package->u.generic->p;
-                
-                ZOOM_options_set(c->options, "sru_version", sr->srw_version);
-                ZOOM_options_setl(c->options, "sru_extra_response_data",
-                                  sr->extraResponseData_buf, sr->extraResponseData_len);
-                if (sr->which == Z_SRW_searchRetrieve_response)
-                    cret = handle_srw_response(c, sr->u.response);
-                else if (sr->which == Z_SRW_scan_response)
-                    handle_srw_scan_response(c, sr->u.scan_response);
-                else
-                    ret = -1;
-            }
-            else if (!ret && (soap_package->which == Z_SOAP_fault
-                              || soap_package->which == Z_SOAP_error))
-            {
-                set_HTTP_error(c, hres->code,
-                               soap_package->u.fault->fault_code,
-                               soap_package->u.fault->fault_string);
-            }
-            else
-                ret = -1;
-        }   
+    else
+    {
+        ret = ZOOM_handle_sru(c, hres, &cret, &addinfo);
         if (ret == 0)
         {
             if (c->no_redirects) /* end of redirect. change hosts again */
@@ -2748,14 +1600,18 @@ static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
     if (ret)
     {
         if (hres->code != 200)
-            set_HTTP_error(c, hres->code, 0, 0);
+            ZOOM_set_HTTP_error(c, hres->code, 0, 0);
         else
+        {
+            yaz_log(YLOG_LOG, "set error... addinfo=%s", addinfo ?
+                    addinfo : "NULL");
             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
+        }
         ZOOM_connection_close(c);
     }
     if (cret == zoom_complete)
     {
-        yaz_log(YLOG_LOG, "removing tasks in handle_http");
+        yaz_log(c->log_details, "removing tasks in handle_http");
         ZOOM_connection_remove_task(c);
     }
     {
@@ -2782,6 +1638,8 @@ static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
                 c->reconnect_ok = 0;
             }
         }
+        else
+            c->reconnect_ok = 1; /* if the server closes anyway */
     }
 }
 #endif
@@ -2790,10 +1648,10 @@ static int do_read(ZOOM_connection c)
 {
     int r, more;
     ZOOM_Event event;
-    
+
     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
     ZOOM_connection_put_event(c, event);
-    
+
     r = cs_get(c->cs, &c->buf_in, &c->len_in);
     more = cs_more(c->cs);
     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
@@ -2819,27 +1677,33 @@ static int do_read(ZOOM_connection c)
 
         if (!z_GDU(c->odr_in, &gdu, 0, 0))
         {
-            int x;
-            int err = odr_geterrorx(c->odr_in, &x);
-            char msg[100];
-            const char *element = odr_getelement(c->odr_in);
-            yaz_snprintf(msg, sizeof(msg),
-                    "ODR code %d:%d element=%s offset=%d",
-                    err, x, element ? element : "<unknown>",
-                    odr_offset(c->odr_in));
-            ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
-            if (c->log_api)
+            /* even on failures we try to re-connect (HTTP) */
+            if (!ZOOM_test_reconnect(c))
             {
-                FILE *ber_file = yaz_log_file();
-                if (ber_file)
-                    odr_dumpBER(ber_file, c->buf_in, r);
+                int x;
+                int err = odr_geterrorx(c->odr_in, &x);
+                char msg[100];
+                const char *element = odr_getelement(c->odr_in);
+                yaz_snprintf(msg, sizeof(msg),
+                             "ODR code %d:%d element=%s offset=%d",
+                             err, x, element ? element : "<unknown>",
+                             odr_offset(c->odr_in));
+                ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
+                if (c->log_api)
+                {
+                    FILE *ber_file = yaz_log_file();
+                    if (ber_file)
+                        odr_dumpBER(ber_file, c->buf_in, r);
+                }
+                ZOOM_connection_close(c);
             }
-            ZOOM_connection_close(c);
         }
         else
         {
             if (c->odr_print)
                 z_GDU(c->odr_print, &gdu, 0, 0);
+            if (c->odr_save)
+                z_GDU(c->odr_save, &gdu, 0, 0);
             if (gdu->which == Z_GDU_Z3950)
                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
             else if (gdu->which == Z_GDU_HTTP_Response)
@@ -2852,7 +1716,6 @@ static int do_read(ZOOM_connection c)
 #endif
             }
         }
-        c->reconnect_ok = 0;
     }
     return 1;
 }
@@ -2861,7 +1724,7 @@ static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
 {
     int r;
     ZOOM_Event event;
-    
+
     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
     ZOOM_connection_put_event(c, event);
 
@@ -2881,7 +1744,7 @@ static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
         return zoom_complete;
     }
     else if (r == 1)
-    {    
+    {
         int mask = ZOOM_SELECT_EXCEPT;
         if (c->cs->io_pending & CS_WANT_WRITE)
             mask += ZOOM_SELECT_WRITE;
@@ -2909,20 +1772,56 @@ zoom_ret ZOOM_send_buf(ZOOM_connection c)
 ZOOM_API(const char *)
     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
 {
-    return ZOOM_options_get(c->options, key);
+    if (!strcmp(key, "APDU"))
+    {
+        return c->saveAPDU_wrbuf ? wrbuf_cstr(c->saveAPDU_wrbuf) : "";
+    }
+    else
+        return ZOOM_options_get(c->options, key);
 }
 
 ZOOM_API(const char *)
     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
 {
-    return ZOOM_options_getl(c->options, key, lenp);
+    if (!strcmp(key, "APDU"))
+    {
+        if (c->saveAPDU_wrbuf)
+        {
+            *lenp = wrbuf_len(c->saveAPDU_wrbuf);
+            return wrbuf_cstr(c->saveAPDU_wrbuf);
+        }
+        else
+        {
+            *lenp = 0;
+            return "";
+        }
+    }
+    else
+        return ZOOM_options_getl(c->options, key, lenp);
 }
 
 ZOOM_API(void)
     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
                                const char *val)
 {
-    ZOOM_options_set(c->options, key, val);
+    if (!strcmp(key, "saveAPDU"))
+    {
+        if (val && strcmp(val, "0"))
+        {
+            if (!c->saveAPDU_wrbuf)
+                c->saveAPDU_wrbuf = wrbuf_alloc();
+            else
+                wrbuf_rewind(c->saveAPDU_wrbuf);
+        }
+        else
+        {
+            wrbuf_destroy(c->saveAPDU_wrbuf);
+            c->saveAPDU_wrbuf = 0;
+        }
+        ZOOM_connection_save_apdu_wrbuf(c, c->saveAPDU_wrbuf);
+    }
+    else
+        ZOOM_options_set(c->options, key, val);
 }
 
 ZOOM_API(void)
@@ -3013,6 +1912,14 @@ ZOOM_API(const char *)
         return "CCL configuration error";
     case ZOOM_ERROR_CCL_PARSE:
         return "CCL parsing error";
+    case ZOOM_ERROR_ES_INVALID_ACTION:
+        return "Extended Service. invalid action";
+    case ZOOM_ERROR_ES_INVALID_VERSION:
+        return "Extended Service. invalid version";
+    case ZOOM_ERROR_ES_INVALID_SYNTAX:
+        return "Extended Service. invalid syntax";
+    case ZOOM_ERROR_MEMCACHED:
+        return "Memcached";
     default:
         return diagbib1_str(error);
     }
@@ -3056,7 +1963,7 @@ static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
     int r = cs_look(c->cs);
     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
             c, mask, r);
-    
+
     if (r == CS_NONE)
     {
         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
@@ -3077,7 +1984,7 @@ static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
             if (c->cs->io_pending & CS_WANT_READ)
                 mask += ZOOM_SELECT_READ;
             ZOOM_connection_set_mask(c, mask);
-            event = ZOOM_Event_create(ZOOM_EVENT_NONE);
+            event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
             ZOOM_connection_put_event(c, event);
         }
         else if (ret == 0)
@@ -3090,10 +1997,15 @@ static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
             else
             {
                 /* no init request for SRW .. */
-                assert(c->tasks->which == ZOOM_TASK_CONNECT);
-                ZOOM_connection_remove_task(c);
-                ZOOM_connection_set_mask(c, 0);
-                ZOOM_connection_exec_task(c);
+                if (c->tasks->which == ZOOM_TASK_CONNECT)
+                {
+                    ZOOM_connection_remove_task(c);
+                    ZOOM_connection_set_mask(c, 0);
+                }
+                if (c->cs && c->location)
+                    send_HTTP_redirect(c, c->location);
+                else
+                    ZOOM_connection_exec_task(c);
             }
             c->state = STATE_ESTABLISHED;
         }
@@ -3130,69 +2042,6 @@ ZOOM_API(int)
 }
 
 
-static void cql2pqf_wrbuf_puts(const char *buf, void *client_data)
-{
-    WRBUF wrbuf = (WRBUF) client_data;
-    wrbuf_puts(wrbuf, buf);
-}
-
-/*
- * Returns an xmalloc()d string containing RPN that corresponds to the
- * CQL passed in.  On error, sets the Connection object's error state
- * and returns a null pointer.
- * ### We could cache CQL parser and/or transformer in Connection.
- */
-static char *cql2pqf(ZOOM_connection c, const char *cql)
-{
-    CQL_parser parser;
-    int error;
-    const char *cqlfile;
-    cql_transform_t trans;
-    char *result = 0;
-
-    parser = cql_parser_create();
-    if ((error = cql_parser_string(parser, cql)) != 0) {
-        cql_parser_destroy(parser);
-        ZOOM_set_error(c, ZOOM_ERROR_CQL_PARSE, cql);
-        return 0;
-    }
-
-    cqlfile = ZOOM_connection_option_get(c, "cqlfile");
-    if (cqlfile == 0) 
-    {
-        ZOOM_set_error(c, ZOOM_ERROR_CQL_TRANSFORM, "no CQL transform file");
-    }
-    else if ((trans = cql_transform_open_fname(cqlfile)) == 0) 
-    {
-        char buf[512];        
-        sprintf(buf, "can't open CQL transform file '%.200s': %.200s",
-                cqlfile, strerror(errno));
-        ZOOM_set_error(c, ZOOM_ERROR_CQL_TRANSFORM, buf);
-    }
-    else 
-    {
-        WRBUF wrbuf_result = wrbuf_alloc();
-        error = cql_transform(trans, cql_parser_result(parser),
-                              cql2pqf_wrbuf_puts, wrbuf_result);
-        if (error != 0) {
-            char buf[512];
-            const char *addinfo;
-            error = cql_transform_error(trans, &addinfo);
-            sprintf(buf, "%.200s (addinfo=%.200s)", 
-                    cql_strerror(error), addinfo);
-            ZOOM_set_error(c, ZOOM_ERROR_CQL_TRANSFORM, buf);
-        }
-        else
-        {
-            result = xstrdup(wrbuf_cstr(wrbuf_result));
-        }
-        cql_transform_close(trans);
-        wrbuf_destroy(wrbuf_result);
-    }
-    cql_parser_destroy(parser);
-    return result;
-}
-
 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
 {
     if (c->mask)
@@ -3235,7 +2084,7 @@ ZOOM_API(int)
     int i;
 
     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
-    
+
     for (i = 0; i<no; i++)
     {
         ZOOM_connection c = cs[i];
@@ -3264,7 +2113,7 @@ ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
 {
     c->mask = mask;
     if (!c->cs)
-        return -1; 
+        return -1;
     return 0;
 }