ZOOM: increase default max message to 64 MB
[yaz-moved-to-github.git] / src / zoom-c.c
index fa3e7ee..2e3e1fe 100644 (file)
@@ -1,11 +1,14 @@
 /* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2010 Index Data
+ * Copyright (C) 1995-2012 Index Data
  * See the file LICENSE for details.
  */
 /**
  * \file zoom-c.c
  * \brief Implements ZOOM C interface.
  */
+#if HAVE_CONFIG_H
+#include <config.h>
+#endif
 
 #include <assert.h>
 #include <string.h>
@@ -34,27 +37,10 @@ static int log_details0 = 0;
 static void resultset_destroy(ZOOM_resultset r);
 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
 
-ZOOM_API(const char *) ZOOM_get_event_str(int event)
-{
-    static const char *ar[] = {
-        "NONE",
-        "CONNECT",
-        "SEND_DATA",
-        "RECV_DATA",
-        "TIMEOUT",
-        "UNKNOWN",
-        "SEND_APDU",
-        "RECV_APDU",
-        "RECV_RECORD",
-        "RECV_SEARCH",
-        "END"
-    };
-    return ar[event];
-}
-
 static void initlog(void)
 {
     static int log_level_initialized = 0;
+
     if (!log_level_initialized)
     {
         log_api0 = yaz_log_module_level("zoom");
@@ -63,72 +49,6 @@ static void initlog(void)
     }
 }
 
-ZOOM_Event ZOOM_Event_create(int kind)
-{
-    ZOOM_Event event = (ZOOM_Event) xmalloc(sizeof(*event));
-    event->kind = kind;
-    event->next = 0;
-    event->prev = 0;
-    return event;
-}
-
-static void ZOOM_Event_destroy(ZOOM_Event event)
-{
-    xfree(event);
-}
-
-void ZOOM_connection_put_event(ZOOM_connection c, ZOOM_Event event)
-{
-    if (c->m_queue_back)
-    {
-        c->m_queue_back->prev = event;
-        assert(c->m_queue_front);
-    }
-    else
-    {
-        assert(!c->m_queue_front);
-        c->m_queue_front = event;
-    }
-    event->next = c->m_queue_back;
-    event->prev = 0;
-    c->m_queue_back = event;
-}
-
-static ZOOM_Event ZOOM_connection_get_event(ZOOM_connection c)
-{
-    ZOOM_Event event = c->m_queue_front;
-    if (!event)
-    {
-        c->last_event = ZOOM_EVENT_NONE;
-        return 0;
-    }
-    assert(c->m_queue_back);
-    c->m_queue_front = event->prev;
-    if (c->m_queue_front)
-    {
-        assert(c->m_queue_back);
-        c->m_queue_front->next = 0;
-    }
-    else
-        c->m_queue_back = 0;
-    c->last_event = event->kind;
-    return event;
-}
-
-static void ZOOM_connection_remove_events(ZOOM_connection c)
-{
-    ZOOM_Event event;
-    while ((event = ZOOM_connection_get_event(c)))
-        ZOOM_Event_destroy(event);
-}
-
-ZOOM_API(int) ZOOM_connection_peek_event(ZOOM_connection c)
-{
-    ZOOM_Event event = c->m_queue_front;
-
-    return event ? event->kind : ZOOM_EVENT_NONE;
-}
-
 void ZOOM_connection_remove_tasks(ZOOM_connection c);
 
 void ZOOM_set_dset_error(ZOOM_connection c, int error,
@@ -150,8 +70,9 @@ void ZOOM_set_dset_error(ZOOM_connection c, int error,
     }
     if (addinfo && addinfo2)
     {
-        c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 2);
+        c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 3);
         strcpy(c->addinfo, addinfo);
+        strcat(c->addinfo, ": ");
         strcat(c->addinfo, addinfo2);
     }
     else if (addinfo)
@@ -176,22 +97,6 @@ int ZOOM_uri_to_code(const char *uri)
 }
 
 
-#if YAZ_HAVE_XML2
-static void set_HTTP_error(ZOOM_connection c, int error,
-                           const char *addinfo, const char *addinfo2)
-{
-    ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
-}
-
-static void set_SRU_error(ZOOM_connection c, Z_SRW_diagnostic *d)
-{
-    const char *uri = d->uri;
-    if (uri)
-        ZOOM_set_dset_error(c, ZOOM_uri_to_code(uri), uri, d->details, 0);
-}
-
-#endif
-
 
 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
 {
@@ -278,7 +183,6 @@ ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
 
     task->running = 0;
     task->which = which;
-    clear_error(c);
     return task;
 }
 
@@ -332,6 +236,12 @@ void ZOOM_connection_remove_tasks(ZOOM_connection c)
         ZOOM_connection_remove_task(c);
 }
 
+static void odr_wrbuf_write(ODR o, void *handle, int type,
+                            const char *buf, int len)
+{
+    WRBUF w = (WRBUF) handle;
+    wrbuf_write(w, buf, len);
+}
 
 ZOOM_API(ZOOM_connection)
     ZOOM_connection_create(ZOOM_options options)
@@ -362,8 +272,8 @@ ZOOM_API(ZOOM_connection)
     c->options = ZOOM_options_create_with_parent(options);
 
     c->host_port = 0;
-    c->path = 0;
     c->proxy = 0;
+    c->tproxy = 0;
     
     c->charset = c->lang = 0;
 
@@ -382,6 +292,7 @@ ZOOM_API(ZOOM_connection)
     c->odr_in = odr_createmem(ODR_DECODE);
     c->odr_out = odr_createmem(ODR_ENCODE);
     c->odr_print = 0;
+    c->odr_save = 0;
 
     c->async = 0;
     c->support_named_resultsets = 0;
@@ -395,6 +306,19 @@ ZOOM_API(ZOOM_connection)
     return c;
 }
 
+ZOOM_API(void) ZOOM_connection_save_apdu_wrbuf(ZOOM_connection c, WRBUF w)
+{
+    if (c->odr_save)
+    {
+        odr_destroy(c->odr_save);
+        c->odr_save = 0;
+    }
+    if (w)
+    {
+        c->odr_save = odr_createmem(ODR_PRINT);
+        odr_set_stream(c->odr_save, w, odr_wrbuf_write, 0);
+    }
+}
 
 /* set database names. Take local databases (if set); otherwise
    take databases given in ZURL (if set); otherwise use Default */
@@ -413,7 +337,7 @@ char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
         if (cp)
             cp++;
     }
-    if (!cp)
+    if (!cp || !*cp)
         cp = "Default";
     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
     return databaseNames;
@@ -438,6 +362,8 @@ static zoom_sru_mode get_sru_mode_from_string(const char *s)
         return zoom_sru_get;
     else if (!yaz_matchstr(s, "post"))
         return zoom_sru_post;
+    else if (!yaz_matchstr(s, "solr"))
+        return zoom_sru_solr;
     return zoom_sru_error;
 }
 
@@ -446,7 +372,6 @@ ZOOM_API(void)
                             const char *host, int portnum)
 {
     const char *val;
-    ZOOM_task task;
 
     initlog();
 
@@ -485,6 +410,15 @@ ZOOM_API(void)
         c->proxy = xstrdup(val);
     }
 
+    xfree(c->tproxy);
+    c->tproxy = 0;
+    val = ZOOM_options_get(c->options, "tproxy");
+    if (val && *val)
+    {
+        yaz_log(c->log_details, "%p ZOOM_connection_connect tproxy=%s", c, val);
+        c->tproxy = xstrdup(val);
+    }
+
     xfree(c->charset);
     c->charset = 0;
     val = ZOOM_options_get(c->options, "charset");
@@ -531,20 +465,22 @@ ZOOM_API(void)
         char *pcomma;
         char *pequals;
         while ((pcomma = strchr(remainder, ',')) != 0 &&
-               (pcolon == 0 || pcomma < pcolon)) {
+               (pcolon == 0 || pcomma < pcolon))
+        {
             *pcomma = '\0';
-            if ((pequals = strchr(remainder, '=')) != 0) {
+            if ((pequals = strchr(remainder, '=')) != 0)
+            {
                 *pequals = '\0';
-                /*printf("# setting '%s'='%s'\n", remainder, pequals+1);*/
                 ZOOM_connection_option_set(c, remainder, pequals+1);
             }
             remainder = pcomma+1;
         }
 
-        if (remainder != c->host_port) {
+        if (remainder != c->host_port)
+        {
+            remainder = xstrdup(remainder);
             xfree(c->host_port);
-            c->host_port = xstrdup(remainder);
-            /*printf("# reset hp='%s'\n", remainder);*/
+            c->host_port = remainder;
         }
     }
 
@@ -598,14 +534,21 @@ ZOOM_API(void)
         c->password = xstrdup(val);
     
     c->maximum_record_size =
-        ZOOM_options_get_int(c->options, "maximumRecordSize", 1024*1024);
+        ZOOM_options_get_int(c->options, "maximumRecordSize", 64*1024*1024);
     c->preferred_message_size =
-        ZOOM_options_get_int(c->options, "preferredMessageSize", 1024*1024);
+        ZOOM_options_get_int(c->options, "preferredMessageSize", 64*1024*1024);
 
     c->async = ZOOM_options_get_bool(c->options, "async", 0);
+
+    if (c->sru_mode == zoom_sru_error)
+    {
+        ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
+        ZOOM_connection_remove_tasks(c);
+        return;
+    }
+
     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
-    task = ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
+    ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 
     if (!c->async)
     {
@@ -652,7 +595,7 @@ ZOOM_API(void)
         cs_close(c->cs);
 
 #if ZOOM_RESULT_LISTS
-    // Remove the connection's usage of resultsets
+    /* Remove the connection's usage of resultsets */
     list = c->resultsets;
     while (list) {
         ZOOM_resultsets removed = list;
@@ -670,6 +613,8 @@ ZOOM_API(void)
     xfree(c->diagset);
     odr_destroy(c->odr_in);
     odr_destroy(c->odr_out);
+    if (c->odr_save)
+        odr_destroy(c->odr_save);
     if (c->odr_print)
     {
         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
@@ -679,8 +624,8 @@ ZOOM_API(void)
     ZOOM_connection_remove_tasks(c);
     ZOOM_connection_remove_events(c);
     xfree(c->host_port);
-    xfree(c->path);
     xfree(c->proxy);
+    xfree(c->tproxy);
     xfree(c->charset);
     xfree(c->lang);
     xfree(c->cookie_out);
@@ -705,6 +650,27 @@ void ZOOM_resultset_addref(ZOOM_resultset r)
     }
 }
 
+static int g_resultsets = 0;
+static YAZ_MUTEX g_resultset_mutex = 0;
+
+/* TODO We need to initialize this before running threaded:
+ * call resultset_use(0)  */
+
+static int resultset_use(int delta) {
+    int resultset_count;
+    if (g_resultset_mutex == 0)
+        yaz_mutex_create(&g_resultset_mutex);
+    yaz_mutex_enter(g_resultset_mutex);
+    g_resultsets += delta;
+    resultset_count = g_resultsets;
+    yaz_mutex_leave(g_resultset_mutex);
+    return resultset_count;
+}
+
+int resultsets_count(void) {
+    return resultset_use(0);
+}
+
 ZOOM_resultset ZOOM_resultset_create(void)
 {
     int i;
@@ -738,6 +704,7 @@ ZOOM_resultset ZOOM_resultset_create(void)
         YAZ_SHPTR_INIT(r->record_wrbuf, w);
     }
 #endif
+    resultset_use(1);
     return r;
 }
 
@@ -927,6 +894,7 @@ static void resultset_destroy(ZOOM_resultset r)
 #if SHPTR
         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
 #endif
+        resultset_use(-1);
         xfree(r);
     }
     else
@@ -1053,6 +1021,15 @@ ZOOM_API(ZOOM_facet_field)
     return 0;
 }
 
+ZOOM_API(ZOOM_facet_field)
+    ZOOM_resultset_get_facet_field_by_index(ZOOM_resultset r, int index) {
+    int num = r->num_facets;
+    ZOOM_facet_field *facets = r->facets;
+    if (index >= 0 && index < num) {
+        return facets[index];
+    }
+    return 0;
+}
 
 ZOOM_API(ZOOM_facet_field *)
     ZOOM_resultset_facets(ZOOM_resultset r)
@@ -1061,7 +1038,7 @@ ZOOM_API(ZOOM_facet_field *)
 }
 
 ZOOM_API(const char**)
-    ZOOM_resultset_facet_names(ZOOM_resultset r)
+    ZOOM_resultset_facets_names(ZOOM_resultset r)
 {
     return (const char **) r->facets_names;
 }
@@ -1099,45 +1076,26 @@ static void get_cert(ZOOM_connection c)
 }
 
 static zoom_ret do_connect_host(ZOOM_connection c,
-                                const char *effective_host,
                                 const char *logical_url);
 
 static zoom_ret do_connect(ZOOM_connection c)
 {
-    const char *effective_host;
-
-    if (c->proxy)
-        effective_host = c->proxy;
-    else
-        effective_host = c->host_port;
-    return do_connect_host(c, effective_host, c->host_port);
+    return do_connect_host(c, c->host_port);
 }
 
-static zoom_ret do_connect_host(ZOOM_connection c, const char *effective_host,
-    const char *logical_url)
+static zoom_ret do_connect_host(ZOOM_connection c, const char *logical_url)
 {
     void *add;
 
-    yaz_log(c->log_details, "%p do_connect effective_host=%s", c, effective_host);
-
     if (c->cs)
         cs_close(c->cs);
-    c->cs = cs_create_host(effective_host, 0, &add);
-
+    c->cs = cs_create_host_proxy(logical_url, 0, &add,
+                                 c->tproxy ? c->tproxy : c->proxy);
+    
     if (c->cs && c->cs->protocol == PROTO_HTTP)
     {
 #if YAZ_HAVE_XML2
-        if (logical_url)
-        {
-            const char *db = 0;
-            
-            c->proto = PROTO_HTTP;
-            cs_get_host_args(logical_url, &db);
-            xfree(c->path);
-            
-            c->path = xmalloc(strlen(db) * 3 + 2);
-            yaz_encode_sru_dbpath_buf(c->path, db);
-        }
+        c->proto = PROTO_HTTP;
 #else
         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
         ZOOM_connection_close(c);
@@ -1186,154 +1144,6 @@ static zoom_ret do_connect_host(ZOOM_connection c, const char *effective_host,
    0 if PDU was not sent OK (nothing to wait for) 
 */
 
-#if YAZ_HAVE_XML2
-static zoom_ret send_srw(ZOOM_connection c, Z_SRW_PDU *sr)
-{
-    Z_GDU *gdu;
-    ZOOM_Event event;
-    const char *database =  ZOOM_options_get(c->options, "databaseName");
-    char *fdatabase = 0;
-    
-    if (database)
-        fdatabase = yaz_encode_sru_dbpath_odr(c->odr_out, database);
-    gdu = z_get_HTTP_Request_host_path(c->odr_out, c->host_port,
-                                       fdatabase ? fdatabase : c->path);
-
-    if (c->sru_mode == zoom_sru_get)
-    {
-        yaz_sru_get_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
-    }
-    else if (c->sru_mode == zoom_sru_post)
-    {
-        yaz_sru_post_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
-    }
-    else if (c->sru_mode == zoom_sru_soap)
-    {
-        yaz_sru_soap_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
-    }
-    if (!z_GDU(c->odr_out, &gdu, 0, 0))
-        return zoom_complete;
-    if (c->odr_print)
-        z_GDU(c->odr_print, &gdu, 0, 0);
-    c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
-        
-    event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
-    ZOOM_connection_put_event(c, event);
-    odr_reset(c->odr_out);
-    return ZOOM_send_buf(c);
-}
-#endif
-
-#if YAZ_HAVE_XML2
-static Z_SRW_PDU *ZOOM_srw_get_pdu(ZOOM_connection c, int type)
-{
-    Z_SRW_PDU *sr = yaz_srw_get_pdu(c->odr_out, type, c->sru_version);
-    sr->username = c->user;
-    sr->password = c->password;
-    return sr;
-}
-#endif
-
-#if YAZ_HAVE_XML2
-static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
-{
-    int i;
-    int *start, *count;
-    ZOOM_resultset resultset = 0;
-    Z_SRW_PDU *sr = 0;
-    const char *option_val = 0;
-    Z_Query *z_query;
-
-    if (c->error)                  /* don't continue on error */
-        return zoom_complete;
-    assert(c->tasks);
-    switch(c->tasks->which)
-    {
-    case ZOOM_TASK_SEARCH:
-        resultset = c->tasks->u.search.resultset;
-        if (!resultset->setname)
-            resultset->setname = xstrdup("default");
-        ZOOM_options_set(resultset->options, "setname", resultset->setname);
-        start = &c->tasks->u.search.start;
-        count = &c->tasks->u.search.count;
-        break;
-    case ZOOM_TASK_RETRIEVE:
-        resultset = c->tasks->u.retrieve.resultset;
-
-        start = &c->tasks->u.retrieve.start;
-        count = &c->tasks->u.retrieve.count;
-
-        if (*start >= resultset->size)
-            return zoom_complete;
-        if (*start + *count > resultset->size)
-            *count = resultset->size - *start;
-
-        for (i = 0; i < *count; i++)
-        {
-            ZOOM_record rec =
-                ZOOM_record_cache_lookup(resultset, i + *start,
-                                         c->tasks->u.retrieve.syntax,
-                                         c->tasks->u.retrieve.elementSetName);
-            if (!rec)
-                break;
-            else
-            {
-                ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
-                ZOOM_connection_put_event(c, event);
-            }
-        }
-        *start += i;
-        *count -= i;
-
-        if (*count == 0)
-            return zoom_complete;
-        break;
-    default:
-        return zoom_complete;
-    }
-    assert(resultset->query);
-        
-    sr = ZOOM_srw_get_pdu(c, Z_SRW_searchRetrieve_request);
-    z_query = ZOOM_query_get_Z_Query(resultset->query);
-
-    if (z_query->which == Z_Query_type_104
-        && z_query->u.type_104->which == Z_External_CQL)
-    {
-        sr->u.request->query_type = Z_SRW_query_type_cql;
-        sr->u.request->query.cql = z_query->u.type_104->u.cql;
-    }
-    else if (z_query->which == Z_Query_type_1 && z_query->u.type_1)
-    {
-        sr->u.request->query_type = Z_SRW_query_type_pqf;
-        sr->u.request->query.pqf =
-            ZOOM_query_get_query_string(resultset->query);
-    }
-    else
-    {
-        ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_QUERY, 0);
-        return zoom_complete;
-    }
-    sr->u.request->startRecord = odr_intdup(c->odr_out, *start + 1);
-    sr->u.request->maximumRecords = odr_intdup(
-        c->odr_out, (resultset->step > 0 && resultset->step < *count) ? 
-        resultset->step : *count);
-    sr->u.request->recordSchema = resultset->schema;
-    
-    option_val = ZOOM_resultset_option_get(resultset, "recordPacking");
-    if (option_val)
-        sr->u.request->recordPacking = odr_strdup(c->odr_out, option_val);
-
-    option_val = ZOOM_resultset_option_get(resultset, "extraArgs");
-    yaz_encode_sru_extra(sr, c->odr_out, option_val);
-    return send_srw(c, sr);
-}
-#else
-static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
-{
-    return zoom_complete;
-}
-#endif
-
 ZOOM_API(ZOOM_record)
     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
 {
@@ -1452,60 +1262,6 @@ static zoom_ret send_package(ZOOM_connection c)
     return ZOOM_send_buf(c);
 }
 
-#if YAZ_HAVE_XML2
-static zoom_ret ZOOM_connection_srw_send_scan(ZOOM_connection c)
-{
-    ZOOM_scanset scan;
-    Z_SRW_PDU *sr = 0;
-    const char *option_val = 0;
-    Z_Query *z_query;
-
-    if (!c->tasks)
-        return zoom_complete;
-    assert (c->tasks->which == ZOOM_TASK_SCAN);
-    scan = c->tasks->u.scan.scan;
-        
-    sr = ZOOM_srw_get_pdu(c, Z_SRW_scan_request);
-
-    z_query = ZOOM_query_get_Z_Query(scan->query);
-    /* SRU scan can only carry CQL and PQF */
-    if (z_query->which == Z_Query_type_104)
-    {
-        sr->u.scan_request->query_type = Z_SRW_query_type_cql;
-        sr->u.scan_request->scanClause.cql =
-            ZOOM_query_get_query_string(scan->query);
-    }
-    else if (z_query->which == Z_Query_type_1
-             || z_query->which == Z_Query_type_101)
-    {
-        sr->u.scan_request->query_type = Z_SRW_query_type_pqf;
-        sr->u.scan_request->scanClause.pqf =
-            ZOOM_query_get_query_string(scan->query);
-    }
-    else
-    {
-        ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_QUERY, 0);
-        return zoom_complete;
-    }
-
-    sr->u.scan_request->maximumTerms = odr_intdup(
-        c->odr_out, ZOOM_options_get_int(scan->options, "number", 10));
-    
-    sr->u.scan_request->responsePosition = odr_intdup(
-        c->odr_out, ZOOM_options_get_int(scan->options, "position", 1));
-    
-    option_val = ZOOM_options_get(scan->options, "extraArgs");
-    yaz_encode_sru_extra(sr, c->odr_out, option_val);
-    return send_srw(c, sr);
-}
-#else
-static zoom_ret ZOOM_connection_srw_send_scan(ZOOM_connection c)
-{
-    return zoom_complete;
-}
-#endif
-
-
 ZOOM_API(size_t)
     ZOOM_scanset_size(ZOOM_scanset scan)
 {
@@ -1757,183 +1513,14 @@ ZOOM_API(int)
 }
 
 #if YAZ_HAVE_XML2
-static zoom_ret handle_srw_response(ZOOM_connection c,
-                                    Z_SRW_searchRetrieveResponse *res)
-{
-    ZOOM_resultset resultset = 0;
-    int i;
-    NMEM nmem;
-    ZOOM_Event event;
-    int *start, *count;
-    const char *syntax, *elementSetName;
-
-    if (!c->tasks)
-        return zoom_complete;
-
-    switch(c->tasks->which)
-    {
-    case ZOOM_TASK_SEARCH:
-        resultset = c->tasks->u.search.resultset;
-        start = &c->tasks->u.search.start;
-        count = &c->tasks->u.search.count;
-        syntax = c->tasks->u.search.syntax;
-        elementSetName = c->tasks->u.search.elementSetName;        
 
-        if (!c->tasks->u.search.recv_search_fired)
-        {
-            event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
-            ZOOM_connection_put_event(c, event);
-            c->tasks->u.search.recv_search_fired = 1;
-        }
-        break;
-    case ZOOM_TASK_RETRIEVE:
-        resultset = c->tasks->u.retrieve.resultset;
-        start = &c->tasks->u.retrieve.start;
-        count = &c->tasks->u.retrieve.count;
-        syntax = c->tasks->u.retrieve.syntax;
-        elementSetName = c->tasks->u.retrieve.elementSetName;
-        break;
-    default:
-        return zoom_complete;
-    }
-
-    resultset->size = 0;
-
-    if (res->resultSetId)
-        ZOOM_resultset_option_set(resultset, "resultSetId", res->resultSetId);
-
-    yaz_log(c->log_details, "%p handle_srw_response got SRW response OK", c);
-
-    if (res->num_diagnostics > 0)
-    {
-        set_SRU_error(c, &res->diagnostics[0]);
-    }
-    else
-    {
-        if (res->numberOfRecords)
-            resultset->size = *res->numberOfRecords;
-        for (i = 0; i<res->num_records; i++)
-        {
-            int pos;
-            Z_SRW_record *sru_rec;
-            Z_SRW_diagnostic *diag = 0;
-            int num_diag;
-            
-            Z_NamePlusRecord *npr = (Z_NamePlusRecord *)
-                odr_malloc(c->odr_in, sizeof(Z_NamePlusRecord));
-            
-            if (res->records[i].recordPosition && 
-                *res->records[i].recordPosition > 0)
-                pos = *res->records[i].recordPosition - 1;
-            else
-                pos = *start + i;
-            
-            sru_rec = &res->records[i];
-            
-            npr->databaseName = 0;
-            npr->which = Z_NamePlusRecord_databaseRecord;
-            npr->u.databaseRecord = (Z_External *)
-                odr_malloc(c->odr_in, sizeof(Z_External));
-            npr->u.databaseRecord->descriptor = 0;
-            npr->u.databaseRecord->direct_reference =
-                odr_oiddup(c->odr_in, yaz_oid_recsyn_xml);
-            npr->u.databaseRecord->which = Z_External_octet;
-            
-            npr->u.databaseRecord->u.octet_aligned = (Odr_oct *)
-                odr_malloc(c->odr_in, sizeof(Odr_oct));
-            npr->u.databaseRecord->u.octet_aligned->buf = (unsigned char*)
-                sru_rec->recordData_buf;
-            npr->u.databaseRecord->u.octet_aligned->len = 
-                npr->u.databaseRecord->u.octet_aligned->size = 
-                sru_rec->recordData_len;
-            
-            if (sru_rec->recordSchema 
-                && !strcmp(sru_rec->recordSchema,
-                           "info:srw/schema/1/diagnostics-v1.1"))
-            {
-                sru_decode_surrogate_diagnostics(sru_rec->recordData_buf,
-                                                 sru_rec->recordData_len,
-                                                 &diag, &num_diag,
-                                                 resultset->odr);
-            }
-            ZOOM_record_cache_add(resultset, npr, pos, syntax, elementSetName,
-                                  sru_rec->recordSchema, diag);
-        }
-        *count -= i;
-        *start += i;
-        if (*count + *start > resultset->size)
-            *count = resultset->size - *start;
-        if (*count < 0)
-            *count = 0;
-        
-        nmem = odr_extract_mem(c->odr_in);
-        nmem_transfer(odr_getmem(resultset->odr), nmem);
-        nmem_destroy(nmem);
-
-        if (*count > 0)
-            return ZOOM_connection_srw_send_search(c);
-    }
-    return zoom_complete;
-}
-#endif
-
-#if YAZ_HAVE_XML2
-static void handle_srw_scan_response(ZOOM_connection c,
-                                     Z_SRW_scanResponse *res)
-{
-    NMEM nmem = odr_extract_mem(c->odr_in);
-    ZOOM_scanset scan;
-
-    if (!c->tasks || c->tasks->which != ZOOM_TASK_SCAN)
-        return;
-    scan = c->tasks->u.scan.scan;
-
-    if (res->num_diagnostics > 0)
-        set_SRU_error(c, &res->diagnostics[0]);
-
-    scan->scan_response = 0;
-    scan->srw_scan_response = res;
-    nmem_transfer(odr_getmem(scan->odr), nmem);
-
-    ZOOM_options_set_int(scan->options, "number", res->num_terms);
-    nmem_destroy(nmem);
-}
-#endif
-
-#if YAZ_HAVE_XML2
-static Z_GDU *get_HTTP_Request_url(ODR odr, const char *url)
-{
-    Z_GDU *p = z_get_HTTP_Request(odr);
-    const char *host = url;
-    const char *cp0 = strstr(host, "://");
-    const char *cp1 = 0;
-    if (cp0)
-        cp0 = cp0+3;
-    else
-        cp0 = host;
-    
-    cp1 = strchr(cp0, '/');
-    if (!cp1)
-        cp1 = cp0 + strlen(cp0);
-    
-    if (cp0 && cp1)
-    {
-        char *h = (char*) odr_malloc(odr, cp1 - cp0 + 1);
-        memcpy (h, cp0, cp1 - cp0);
-        h[cp1-cp0] = '\0';
-        z_HTTP_header_add(odr, &p->u.HTTP_Request->headers, "Host", h);
-    }
-    p->u.HTTP_Request->path = odr_strdup(odr, *cp1 ? cp1 : "/");
-    return p;
-}
-
-static zoom_ret send_SRW_redirect(ZOOM_connection c, const char *uri,
+static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri,
                                   Z_HTTP_Response *cookie_hres)
 {
     struct Z_HTTP_Header *h;
-    Z_GDU *gdu = get_HTTP_Request_url(c->odr_out, uri);
-    char *combined_cookies;
+    char *combined_cookies = 0;
     int combined_cookies_len = 0;
+    Z_GDU *gdu = z_get_HTTP_Request_uri(c->odr_out, uri, 0, c->proxy ? 1 : 0);
 
     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
@@ -1973,31 +1560,42 @@ static zoom_ret send_SRW_redirect(ZOOM_connection c, const char *uri,
         return zoom_complete;
     if (c->odr_print)
         z_GDU(c->odr_print, &gdu, 0, 0);
+    if (c->odr_save)
+        z_GDU(c->odr_save, &gdu, 0, 0);
     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
 
     odr_reset(c->odr_out);
     return ZOOM_send_buf(c);
 }
 
+#if YAZ_HAVE_XML2
+void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
+                         const char *addinfo, const char *addinfo2)
+{
+    ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
+}
+#endif
+
+
 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
 {
     zoom_ret cret = zoom_complete;
     int ret = -1;
-    const char *addinfo = 0;
+    char *addinfo = 0;
     const char *connection_head = z_HTTP_header_lookup(hres->headers,
                                                        "Connection");
     const char *location;
 
     ZOOM_connection_set_mask(c, 0);
     yaz_log(c->log_details, "%p handle_http", c);
-    
+
     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
     {
         c->no_redirects++;
         if (c->no_redirects > 10)
         {
-            set_HTTP_error(c, hres->code, 0, 0);
+            ZOOM_set_HTTP_error(c, hres->code, 0, 0);
             c->no_redirects = 0;
             ZOOM_connection_close(c);
         }
@@ -2005,53 +1603,15 @@ static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
         {
             /* since redirect may change host we just reconnect. A smarter
                implementation might check whether it's the same server */
-            do_connect_host(c, location, 0);
-            send_SRW_redirect(c, location, hres);
+            do_connect_host(c, location);
+            send_HTTP_redirect(c, location, hres);
             /* we're OK for now. Operation is not really complete */
-            ret = 0;
-            cret = zoom_pending;
+            return;
         }
     }
     else 
-    {   /* not redirect (normal response) */
-        if (!yaz_srw_check_content_type(hres))
-            addinfo = "content-type";
-        else
-        {
-            Z_SOAP *soap_package = 0;
-            ODR o = c->odr_in;
-            Z_SOAP_Handler soap_handlers[2] = {
-                {YAZ_XMLNS_SRU_v1_1, 0, (Z_SOAP_fun) yaz_srw_codec},
-                {0, 0, 0}
-            };
-            ret = z_soap_codec(o, &soap_package,
-                               &hres->content_buf, &hres->content_len,
-                               soap_handlers);
-            if (!ret && soap_package->which == Z_SOAP_generic &&
-                soap_package->u.generic->no == 0)
-            {
-                Z_SRW_PDU *sr = (Z_SRW_PDU*) soap_package->u.generic->p;
-                
-                ZOOM_options_set(c->options, "sru_version", sr->srw_version);
-                ZOOM_options_setl(c->options, "sru_extra_response_data",
-                                  sr->extraResponseData_buf, sr->extraResponseData_len);
-                if (sr->which == Z_SRW_searchRetrieve_response)
-                    cret = handle_srw_response(c, sr->u.response);
-                else if (sr->which == Z_SRW_scan_response)
-                    handle_srw_scan_response(c, sr->u.scan_response);
-                else
-                    ret = -1;
-            }
-            else if (!ret && (soap_package->which == Z_SOAP_fault
-                              || soap_package->which == Z_SOAP_error))
-            {
-                set_HTTP_error(c, hres->code,
-                               soap_package->u.fault->fault_code,
-                               soap_package->u.fault->fault_string);
-            }
-            else
-                ret = -1;
-        }   
+    {  
+        ret = ZOOM_handle_sru(c, hres, &cret, &addinfo);
         if (ret == 0)
         {
             if (c->no_redirects) /* end of redirect. change hosts again */
@@ -2062,14 +1622,18 @@ static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
     if (ret)
     {
         if (hres->code != 200)
-            set_HTTP_error(c, hres->code, 0, 0);
+            ZOOM_set_HTTP_error(c, hres->code, 0, 0);
         else
+        {
+            yaz_log(YLOG_LOG, "set error... addinfo=%s", addinfo ?
+                    addinfo : "NULL");
             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
+        }
         ZOOM_connection_close(c);
     }
     if (cret == zoom_complete)
     {
-        yaz_log(YLOG_LOG, "removing tasks in handle_http");
+        yaz_log(c->log_details, "removing tasks in handle_http");
         ZOOM_connection_remove_task(c);
     }
     {
@@ -2096,6 +1660,8 @@ static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
                 c->reconnect_ok = 0;
             }
         }
+        else
+            c->reconnect_ok = 1; /* if the server closes anyway */
     }
 }
 #endif
@@ -2154,6 +1720,8 @@ static int do_read(ZOOM_connection c)
         {
             if (c->odr_print)
                 z_GDU(c->odr_print, &gdu, 0, 0);
+            if (c->odr_save)
+                z_GDU(c->odr_save, &gdu, 0, 0);
             if (gdu->which == Z_GDU_Z3950)
                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
             else if (gdu->which == Z_GDU_HTTP_Response)
@@ -2166,7 +1734,6 @@ static int do_read(ZOOM_connection c)
 #endif
             }
         }
-        c->reconnect_ok = 0;
     }
     return 1;
 }
@@ -2327,6 +1894,12 @@ ZOOM_API(const char *)
         return "CCL configuration error";
     case ZOOM_ERROR_CCL_PARSE:
         return "CCL parsing error";
+    case ZOOM_ERROR_ES_INVALID_ACTION:
+        return "Extended Service. invalid action";
+    case ZOOM_ERROR_ES_INVALID_VERSION:
+        return "Extended Service. invalid version";
+    case ZOOM_ERROR_ES_INVALID_SYNTAX:
+        return "Extended Service. invalid syntax";
     default:
         return diagbib1_str(error);
     }