zomo refactor: move m_frontend_database to Frontend
[metaproxy-moved-to-github.git] / src / filter_zoom.cpp
index 018f532..a3ab833 100644 (file)
@@ -1,5 +1,5 @@
 /* This file is part of Metaproxy.
-   Copyright (C) 2005-2013 Index Data
+   Copyright (C) Index Data
 
 Metaproxy is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free
@@ -51,6 +51,8 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #include <yaz/sortspec.h>
 #include <yaz/tokenizer.h>
 #include <yaz/zoom.h>
+#include <yaz/otherinfo.h>
+#include <yaz/match_glob.h>
 
 namespace mp = metaproxy_1;
 namespace yf = mp::filter;
@@ -60,6 +62,8 @@ namespace metaproxy_1 {
         class Zoom::Searchable : boost::noncopyable {
           public:
             std::string authentication;
+            std::string authenticationMode;
+            std::string contentAuthentication;
             std::string cfAuth;
             std::string cfProxy;
             std::string cfSubDB;
@@ -78,6 +82,7 @@ namespace metaproxy_1 {
             std::string sortStrategy;
             std::string extraArgs;
             std::string rpn2cql_fname;
+            std::string retry_on_failure;
             bool use_turbomarc;
             bool piggyback;
             CCL_bibset ccl_bibset;
@@ -88,11 +93,9 @@ namespace metaproxy_1 {
         class Zoom::Backend : boost::noncopyable {
             friend class Impl;
             friend class Frontend;
-            std::string zurl;
             mp::wrbuf m_apdu_wrbuf;
             ZOOM_connection m_connection;
             ZOOM_resultset m_resultset;
-            std::string m_frontend_database;
             SearchablePtr sptr;
             xsltStylesheetPtr xsp;
             std::string cproxy_host;
@@ -101,16 +104,18 @@ namespace metaproxy_1 {
             xmlDoc *explain_doc;
             std::string m_proxy;
             cql_transform_t cqlt;
+            std::string retry_on_failure;
         public:
             Backend();
             ~Backend();
             void connect(std::string zurl, int *error, char **addinfo,
                          ODR odr);
             void search(ZOOM_query q, Odr_int *hits,
-                        int *error, char **addinfo, ODR odr);
+                        int *error, char **addinfo, Z_FacetList **fl, ODR odr);
             void present(Odr_int start, Odr_int number, ZOOM_record *recs,
                          int *error, char **addinfo, ODR odr);
             void set_option(const char *name, const char *value);
+            void set_option(const char *name, const char *value, size_t l);
             void set_option(const char *name, std::string value);
             const char *get_option(const char *name);
             void get_zoom_error(int *error, char **addinfo, ODR odr);
@@ -121,29 +126,29 @@ namespace metaproxy_1 {
             bool m_is_virtual;
             bool m_in_use;
             std::string session_realm;
+            std::string m_frontend_database;
             yazpp_1::GDU m_init_gdu;
             BackendPtr m_backend;
             void handle_package(mp::Package &package);
             void handle_search(mp::Package &package);
-
             void auth(mp::Package &package, Z_InitRequest *req,
                       int *error, char **addinfo, ODR odr);
 
-            BackendPtr explain_search(mp::Package &package,
-                                      std::string &database,
-                                      int *error,
-                                      char **addinfo,
-                                      mp::odr &odr,
-                                      std::string torus_url,
-                                      std::string &torus_db,
-                                      std::string &realm);
+            void explain_search(mp::Package &package,
+                                std::string &database,
+                                int *error,
+                                char **addinfo,
+                                mp::odr &odr,
+                                std::string torus_url,
+                                std::string &torus_db,
+                                std::string &realm);
             void handle_present(mp::Package &package);
-            BackendPtr get_backend_from_databases(mp::Package &package,
-                                                  std::string &database,
-                                                  int *error,
-                                                  char **addinfo,
-                                                  mp::odr &odr,
-                                                  int *proxy_step);
+            void get_backend_from_databases(mp::Package &package,
+                                            std::string &database,
+                                            int *error,
+                                            char **addinfo,
+                                            mp::odr &odr,
+                                            int *proxy_step);
 
             bool create_content_session(mp::Package &package,
                                         BackendPtr b,
@@ -216,7 +221,9 @@ namespace metaproxy_1 {
             std::string torus_searchable_url;
             std::string torus_content_url;
             std::string torus_auth_url;
+            std::string torus_allow_ip;
             std::string default_realm;
+            std::string torus_auth_hostname;
             std::map<std::string,std::string> fieldmap;
             std::string xsldir;
             std::string file_path;
@@ -370,15 +377,84 @@ void yf::Zoom::Backend::connect(std::string zurl,
 }
 
 void yf::Zoom::Backend::search(ZOOM_query q, Odr_int *hits,
-                               int *error, char **addinfo, ODR odr)
+                               int *error, char **addinfo, Z_FacetList **flp,
+                               ODR odr)
 {
     ZOOM_resultset_destroy(m_resultset);
+    m_resultset = 0;
+    if (*flp)
+    {
+        WRBUF w = wrbuf_alloc();
+        yaz_facet_list_to_wrbuf(w, *flp);
+        set_option("facets", wrbuf_cstr(w));
+        wrbuf_destroy(w);
+    }
     m_resultset = ZOOM_connection_search(m_connection, q);
     get_zoom_error(error, addinfo, odr);
     if (*error == 0)
         *hits = ZOOM_resultset_size(m_resultset);
     else
         *hits = 0;
+    *flp = 0;
+    size_t num_facets = ZOOM_resultset_facets_size(m_resultset);
+    if (num_facets > 0)
+    {
+        size_t i;
+        Z_FacetList *fl = (Z_FacetList *) odr_malloc(odr, sizeof(*fl));
+        fl->elements = (Z_FacetField **)
+            odr_malloc(odr, num_facets * sizeof(*fl->elements));
+        for (i = 0; i < num_facets; i++)
+        {
+            ZOOM_facet_field ff =
+                ZOOM_resultset_get_facet_field_by_index(m_resultset, i);
+            if (!ff)
+                break;
+            Z_AttributeList *al = (Z_AttributeList *)
+                odr_malloc(odr, sizeof(*al));
+            al->num_attributes = 1;
+            al->attributes = (Z_AttributeElement **)
+                odr_malloc(odr, sizeof(*al->attributes));
+            Z_AttributeElement *ae = al->attributes[0] = (Z_AttributeElement *)
+                odr_malloc(odr, sizeof(**al->attributes));
+            ae->attributeSet = 0;
+            ae->attributeType = odr_intdup(odr, 1);
+            ae->which = Z_AttributeValue_complex;
+            ae->value.complex = (Z_ComplexAttribute *)
+                odr_malloc(odr, sizeof(*ae->value.complex));
+            ae->value.complex->num_list = 1;
+            ae->value.complex->list = (Z_StringOrNumeric **)
+                odr_malloc(odr, sizeof(**ae->value.complex->list));
+            ae->value.complex->list[0] = (Z_StringOrNumeric *)
+                odr_malloc(odr, sizeof(**ae->value.complex->list));
+            ae->value.complex->list[0]->which = Z_StringOrNumeric_string;
+            ae->value.complex->list[0]->u.string =
+                odr_strdup(odr, ZOOM_facet_field_name(ff));
+            ae->value.complex->num_semanticAction = 0;
+            ae->value.complex->semanticAction = 0;
+
+            int num_terms = ZOOM_facet_field_term_count(ff);
+            fl->elements[i] = (Z_FacetField *)
+                odr_malloc(odr, sizeof(Z_FacetField));
+            fl->elements[i]->attributes = al;
+            fl->elements[i]->num_terms = num_terms;
+            fl->elements[i]->terms = (Z_FacetTerm **)
+                odr_malloc(odr, num_terms * sizeof(Z_FacetTerm *));
+            int j;
+            for (j = 0; j < num_terms; j++)
+            {
+                int freq;
+                const char *a_term = ZOOM_facet_field_get_term(ff, j, &freq);
+                Z_FacetTerm *ft = (Z_FacetTerm *) odr_malloc(odr, sizeof(*ft));
+                ft->term = z_Term_create(odr, Z_Term_general, a_term,
+                                         strlen(a_term));
+                ft->count = odr_intdup(odr, freq);
+
+                fl->elements[i]->terms[j] = ft;
+            }
+        }
+        fl->num = i;
+        *flp = fl;
+    }
 }
 
 void yf::Zoom::Backend::present(Odr_int start, Odr_int number,
@@ -389,6 +465,12 @@ void yf::Zoom::Backend::present(Odr_int start, Odr_int number,
     get_zoom_error(error, addinfo, odr);
 }
 
+
+void yf::Zoom::Backend::set_option(const char *name, const char *value, size_t l)
+{
+    ZOOM_connection_option_setl(m_connection, name, value, l);
+}
+
 void yf::Zoom::Backend::set_option(const char *name, const char *value)
 {
     ZOOM_connection_option_set(m_connection, name, value);
@@ -411,6 +493,7 @@ yf::Zoom::Searchable::Searchable(CCL_bibset base)
     piggyback = true;
     use_turbomarc = true;
     sortStrategy = "embed";
+    retry_on_failure = "1"; // existing default (should have been false)
     ccl_bibset = ccl_qual_dup(base);
 }
 
@@ -508,6 +591,16 @@ yf::Zoom::SearchablePtr yf::Zoom::Impl::parse_torus_record(const xmlNode *ptr)
             s->authentication = mp::xml::get_text(ptr);
         }
         else if (!strcmp((const char *) ptr->name,
+                         "authenticationMode"))
+        {
+            s->authenticationMode = mp::xml::get_text(ptr);
+        }
+        else if (!strcmp((const char *) ptr->name,
+                         "contentAuthentication"))
+        {
+            s->contentAuthentication = mp::xml::get_text(ptr);
+        }
+        else if (!strcmp((const char *) ptr->name,
                          "cfAuth"))
         {
             s->cfAuth = mp::xml::get_text(ptr);
@@ -617,6 +710,11 @@ yf::Zoom::SearchablePtr yf::Zoom::Impl::parse_torus_record(const xmlNode *ptr)
         }
         else if (!strcmp((const char *) ptr->name, "rpn2cql"))
             s->rpn2cql_fname = mp::xml::get_text(ptr);
+        else if (!strcmp((const char *) ptr->name,
+                          "retryOnFailure"))
+        {
+            s->retry_on_failure = mp::xml::get_text(ptr);
+        }
     }
     return s;
 }
@@ -694,8 +792,12 @@ void yf::Zoom::Impl::configure(const xmlNode *ptr, bool test_only,
                     torus_content_url = mp::xml::get_text(attr->children);
                 else if (!strcmp((const char *) attr->name, "auth_url"))
                     torus_auth_url = mp::xml::get_text(attr->children);
+                else if (!strcmp((const char *) attr->name, "allow_ip"))
+                    torus_allow_ip = mp::xml::get_text(attr->children);
                 else if (!strcmp((const char *) attr->name, "realm"))
                     default_realm = mp::xml::get_text(attr->children);
+                else if (!strcmp((const char *) attr->name, "auth_hostname"))
+                    torus_auth_hostname = mp::xml::get_text(attr->children);
                 else if (!strcmp((const char *) attr->name, "xsldir"))
                     xsldir = mp::xml::get_text(attr->children);
                 else if (!strcmp((const char *) attr->name, "element_transform"))
@@ -1010,7 +1112,7 @@ bool yf::Zoom::Frontend::create_content_session(mp::Package &package,
     return true;
 }
 
-yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
+void yf::Zoom::Frontend::get_backend_from_databases(
     mp::Package &package,
     std::string &database, int *error, char **addinfo, mp::odr &odr,
     int *proxy_step)
@@ -1018,9 +1120,8 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
     bool connection_reuse = false;
     std::string proxy;
 
-    std::list<BackendPtr>::const_iterator map_it;
     if (m_backend && !m_backend->enable_explain &&
-        m_backend->m_frontend_database == database)
+        m_frontend_database == database)
     {
         connection_reuse = true;
         proxy = m_backend->m_proxy;
@@ -1037,8 +1138,6 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
     else
         torus_db = database;
 
-    std::string authentication;
-    std::string content_authentication;
     std::string content_proxy;
     std::string realm = session_realm;
     if (realm.length() == 0)
@@ -1049,6 +1148,7 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
     const char *param_content_user = 0;
     const char *param_content_password = 0;
     const char *param_nocproxy = 0;
+    const char *param_retry = 0;
     int no_parms = 0;
 
     char **names;
@@ -1084,6 +1184,8 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
             content_proxy = value;
         else if (!strcmp(name, "nocproxy"))
             param_nocproxy = value;
+        else if (!strcmp(name, "retry"))
+            param_retry = value;
         else if (!strcmp(name, "proxy"))
         {
             char **dstr;
@@ -1138,7 +1240,7 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
             sprintf(msg, "zoom: bad database argument: %s", name);
             *addinfo = msg;
-            return notfound;
+            return;
         }
     }
     if (proxy.length())
@@ -1147,20 +1249,7 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
     if (connection_reuse)
     {
         m_backend->connect("", error, addinfo, odr);
-        return m_backend;
-    }
-
-    if (param_user)
-    {
-        authentication = std::string(param_user);
-        if (param_password)
-            authentication += "/" + std::string(param_password);
-    }
-    if (param_content_user)
-    {
-        content_authentication = std::string(param_content_user);
-        if (param_content_password)
-            content_authentication += "/" + std::string(param_content_password);
+        return;
     }
 
     if (torus_db.compare("IR-Explain---1") == 0)
@@ -1175,17 +1264,18 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
         sptr = it->second;
     else if (torus_url.length() > 0)
     {
+        std::string torus_addinfo;
         std::string torus_query = "udb==" + torus_db;
         xmlDoc *doc = mp::get_searchable(package,torus_url, torus_db,
                                          torus_query,
-                                         realm, m_p->proxy);
+                                         realm, m_p->proxy,
+                                         torus_addinfo);
         if (!doc)
         {
             *error = YAZ_BIB1_UNSPECIFIED_ERROR;
-            *addinfo = odr_strdup(odr, "Torus server unavailable or "
-                                  "incorrectly configured");
-            BackendPtr b;
-            return b;
+            if (torus_addinfo.length())
+                *addinfo = odr_strdup(odr, torus_addinfo.c_str());
+            return;
         }
         const xmlNode *ptr = xmlDocGetRootElement(doc);
         if (ptr && ptr->type == XML_ELEMENT_NODE)
@@ -1209,8 +1299,7 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
                             sprintf(*addinfo, "multiple records for udb=%s",
                                     database.c_str());
                             xmlFreeDoc(doc);
-                            BackendPtr b;
-                            return b;
+                            return;
                         }
                         sptr = m_p->parse_torus_record(ptr);
                     }
@@ -1223,8 +1312,7 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
                     odr, 40 + strlen((const char *) ptr->name));
                 sprintf(*addinfo, "bad root element for torus: %s", ptr->name);
                 xmlFreeDoc(doc);
-                BackendPtr b;
-                return b;
+                return;
             }
         }
         xmlFreeDoc(doc);
@@ -1234,8 +1322,7 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
     {
         *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
         *addinfo = odr_strdup(odr, torus_db.c_str());
-        BackendPtr b;
-        return b;
+        return;
     }
 
     xsltStylesheetPtr xsp = 0;
@@ -1248,8 +1335,7 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
             *addinfo = odr_strdup(odr, "zoom: xmlParseMemory failed "
                                   "for literalTransform XSL");
-            BackendPtr b;
-            return b;
+            return;
         }
         xsp = xsltParseStylesheetDoc(xsp_doc);
         if (!xsp)
@@ -1258,9 +1344,8 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
             *addinfo =
                 odr_strdup(odr,"zoom: xsltParseStylesheetDoc failed "
                            "for literalTransform XSL");
-            BackendPtr b;
             xmlFreeDoc(xsp_doc);
-            return b;
+            return;
         }
     }
     else if (sptr->transform_xsl_fname.length())
@@ -1285,8 +1370,7 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
                 odr_malloc(odr, 40 + sptr->transform_xsl_fname.length());
             sprintf(*addinfo, "zoom: could not open file %s",
                     sptr->transform_xsl_fname.c_str());
-            BackendPtr b;
-            return b;
+            return;
         }
         xmlDoc *xsp_doc = xmlParseFile(fname.c_str());
         if (!xsp_doc)
@@ -1295,8 +1379,7 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
             *addinfo = (char *) odr_malloc(odr, 50 + fname.length());
             sprintf(*addinfo, "zoom: xmlParseFile failed for file %s",
                     fname.c_str());
-            BackendPtr b;
-            return b;
+            return;
         }
         xsp = xsltParseStylesheetDoc(xsp_doc);
         if (!xsp)
@@ -1305,9 +1388,8 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
             *addinfo = (char *) odr_malloc(odr, 50 + fname.length());
             sprintf(*addinfo, "zoom: xsltParseStylesheetDoc failed "
                     "for file %s", fname.c_str());
-            BackendPtr b;
             xmlFreeDoc(xsp_doc);
-            return b;
+            return;
         }
     }
 
@@ -1327,9 +1409,8 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
     {
         *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
         *addinfo = odr_strdup(odr, "zoom: missing/invalid cql2rpn file");
-        BackendPtr b;
         xsltFreeStylesheet(xsp);
-        return b;
+        return;
     }
 
     m_backend.reset();
@@ -1339,14 +1420,18 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
     b->cqlt = cqlt;
     b->sptr = sptr;
     b->xsp = xsp;
-    b->m_frontend_database = database;
+    m_frontend_database = database;
     b->enable_cproxy = param_nocproxy ? false : true;
 
+    if (param_retry)
+        b->retry_on_failure = param_retry;
+    else
+        b->retry_on_failure = b->sptr->retry_on_failure;
+
     if (sptr->query_encoding.length())
         b->set_option("rpnCharset", sptr->query_encoding);
 
-    if (sptr->extraArgs.length())
-        b->set_option("extraArgs", sptr->extraArgs);
+    std::string extraArgs = sptr->extraArgs;
 
     b->set_option("timeout", m_p->zoom_timeout.c_str());
 
@@ -1358,8 +1443,20 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
                                         maximumRecords > 0 */
     b->set_option("piggyback", sptr->piggyback ? "1" : "0");
 
-    if (authentication.length() == 0)
-        authentication = sptr->authentication;
+    std::string authentication = sptr->authentication;
+    if (param_user)
+    {
+        authentication = std::string(param_user);
+        if (param_password)
+            authentication += "/" + std::string(param_password);
+    }
+    std::string content_authentication = sptr->contentAuthentication;
+    if (param_content_user)
+    {
+        content_authentication = std::string(param_content_user);
+        if (param_content_password)
+            content_authentication += "/" + std::string(param_content_password);
+    }
 
     if (proxy.length() == 0)
         proxy = sptr->cfProxy;
@@ -1369,7 +1466,17 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
     {
         // A CF target
         b->set_option("user", sptr->cfAuth);
-        if (authentication.length())
+        if (param_user)
+        {
+            out_names[no_out_args] = "user";
+            out_values[no_out_args++] = odr_strdup(odr, param_user);
+            if (param_password)
+            {
+                out_names[no_out_args] = "password";
+                out_values[no_out_args++] = odr_strdup(odr, param_password);
+            }
+        }
+        else if (authentication.length())
         {
             size_t found = authentication.find('/');
             if (found != std::string::npos)
@@ -1399,6 +1506,9 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
             out_names[no_out_args] = "subdatabase";
             out_values[no_out_args++] = odr_strdup(odr, sptr->cfSubDB.c_str());
         }
+        if (!param_nocproxy && b->sptr->contentConnector.length())
+            param_nocproxy = "1";
+
         if (param_nocproxy)
         {
             out_names[no_out_args] = "nocproxy";
@@ -1407,62 +1517,37 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
     }
     else
     {
-        if (sptr->sru.length() == 0)
-            b->set_option("user", authentication); /* Z39.50 */
+        const char *auth = authentication.c_str();
+        const char *cp1 = strchr(auth, ' ');
+        if (!cp1 && sptr->sru.length())
+            cp1 =  strchr(auth, '/');
+        if (!cp1)
+        {
+            /* Z39.50 user/password style, or no password for SRU */
+            b->set_option("user", auth);
+        }
         else
         {
-            std::string user;
-            std::string password;
-            std::string authtype;
-            {
-                const char *cstr = authentication.c_str();
-                const char *cp1 = strchr(cstr, '/');
-                const char *cp2 = 0;
-                if (cp1)
-                {
-                    cp2 = strchr(cp1 + 1, '/');
-                    if (cp2)
-                    {
-                        password.assign(cp1 + 1, cp2 - cp1 - 1);
-                        authtype.assign(cp2 + 1);
-                    }
-                    else
-                        password.assign(cp1 + 1);
-                    user.assign(cstr, cp1 - cstr);
-                }
-                else
-                    user.assign(cstr);
-            }
+            /* now consider group as well */
+            const char *cp2 = strchr(cp1 + 1, ' ');
 
-            if (authtype.compare("url") == 0)
-            {
-                /* SRU URL encoding of auth stuff */
-                ODR o = odr_createmem(ODR_ENCODE);
-                char *path = 0;
-                const char *names[3];
-                const char *values[3];
-
-                names[0] = "x-username";
-                values[0] = user.c_str();
-                names[1] = "x-password";
-                values[1] = password.c_str();
-                names[2] = 0;
-                values[2] = 0;
-
-                yaz_array_to_uri(&path, o, (char **) names, (char **) values);
-                b->set_option("extraArgs", path);
-                odr_destroy(o);
-            }
+            b->set_option("user", auth, cp1 - auth);
+            if (!cp2)
+                b->set_option("password", cp1 + 1);
             else
             {
-                b->set_option("user", user);
-                if (password.length())
-                    b->set_option("password", password);
+                b->set_option("group", cp1 + 1, cp2 - cp1 - 1);
+                b->set_option("password", cp2 + 1);
             }
         }
+        if (sptr->authenticationMode.length())
+            b->set_option("authenticationMode", sptr->authenticationMode);
         if (proxy.length())
             b->set_option("proxy", proxy);
     }
+    if (extraArgs.length())
+        b->set_option("extraArgs", extraArgs);
+
     std::string url(sptr->target);
     if (sptr->sru.length())
     {
@@ -1491,7 +1576,6 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
                                realm);
     if (*error == 0)
         m_backend = b;
-    return b;
 }
 
 void yf::Zoom::Frontend::prepare_elements(BackendPtr b,
@@ -1608,7 +1692,7 @@ Z_Records *yf::Zoom::Frontend::get_explain_records(
 
         Z_NamePlusRecord *npr =
             (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
-        npr->databaseName = odr_strdup(odr, b->m_frontend_database.c_str());
+        npr->databaseName = odr_strdup(odr, m_frontend_database.c_str());
         npr->which = Z_NamePlusRecord_databaseRecord;
         npr->u.databaseRecord =
             z_ext_record_xml(odr,
@@ -1671,8 +1755,19 @@ Z_Records *yf::Zoom::Frontend::get_records(mp::Package &package,
     if (!*error)
     {
         for (i = 0; i < number_to_present; i++)
+        {
             if (!recs[i])
                 break;
+
+            const char *addinfo;
+            int sur_error = ZOOM_record_error(recs[i], 0 /* msg */,
+                                              &addinfo, 0 /* diagset */);
+            if (sur_error ==
+                YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS && addinfo &&
+                !strcmp(addinfo,
+                        "ZOOM C generated. Present phase and no records"))
+                break;
+        }
     }
     if (i > 0)
     {  // only return records if no error and at least one record
@@ -1695,15 +1790,14 @@ Z_Records *yf::Zoom::Frontend::get_records(mp::Package &package,
             xsl_parms[0] = 0;
         }
 
-        char *odr_database = odr_strdup(odr,
-                                        b->m_frontend_database.c_str());
+        char *odr_database = odr_strdup(odr, m_frontend_database.c_str());
         Z_NamePlusRecordList *npl = (Z_NamePlusRecordList *)
             odr_malloc(odr, sizeof(*npl));
         *number_of_records_returned = i;
         npl->num_records = i;
         npl->records = (Z_NamePlusRecord **)
             odr_malloc(odr, i * sizeof(*npl->records));
-        for (i = 0; i < number_to_present; i++)
+        for (i = 0; i < npl->num_records; i++)
         {
             Z_NamePlusRecord *npr = 0;
             const char *addinfo;
@@ -1922,20 +2016,20 @@ void yf::Zoom::Frontend::log_diagnostic(mp::Package &package,
                     error, err_msg);
 }
 
-yf::Zoom::BackendPtr yf::Zoom::Frontend::explain_search(mp::Package &package,
-                                                        std::string &database,
-                                                        int *error,
-                                                        char **addinfo,
-                                                        mp::odr &odr,
-                                                        std::string torus_url,
-                                                        std::string &torus_db,
-                                                        std::string &realm)
+void yf::Zoom::Frontend::explain_search(mp::Package &package,
+                                        std::string &database,
+                                        int *error,
+                                        char **addinfo,
+                                        mp::odr &odr,
+                                        std::string torus_url,
+                                        std::string &torus_db,
+                                        std::string &realm)
 {
     m_backend.reset();
 
     BackendPtr b(new Backend);
 
-    b->m_frontend_database = database;
+    m_frontend_database = database;
     b->enable_explain = true;
 
     Z_GDU *gdu = package.request().get();
@@ -1949,15 +2043,17 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::explain_search(mp::Package &package,
         *addinfo =
             odr_strdup(odr, "IR-Explain---1 unsupported. "
                        "Torus explain_xsl not defined");
-        return m_backend;
+        return;
     }
     else if (query->which == Z_Query_type_104 &&
         query->u.type_104->which == Z_External_CQL)
     {
+        std::string torus_addinfo;
         std::string torus_query(query->u.type_104->u.cql);
         xmlDoc *doc = mp::get_searchable(package, torus_url, "",
                                          torus_query,
-                                         realm, m_p->proxy);
+                                         realm, m_p->proxy,
+                                         torus_addinfo);
         if (m_p->explain_xsp)
         {
             xmlDoc *rec_res =  xsltApplyStylesheet(m_p->explain_xsp, doc, 0);
@@ -1968,8 +2064,8 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::explain_search(mp::Package &package,
         if (!doc)
         {
             *error = YAZ_BIB1_UNSPECIFIED_ERROR;
-            *addinfo = odr_strdup(odr, "Torus server unavailable or "
-                                  "incorrectly configured");
+            if (torus_addinfo.length())
+                *addinfo = odr_strdup(odr, torus_addinfo.c_str());
         }
         else
         {
@@ -1986,13 +2082,11 @@ yf::Zoom::BackendPtr yf::Zoom::Frontend::explain_search(mp::Package &package,
         if (b->explain_doc)
             xmlFreeDoc(b->explain_doc);
         b->explain_doc = doc;
-        return m_backend;
     }
     else
     {
         *error = YAZ_BIB1_QUERY_TYPE_UNSUPP;
         *addinfo = odr_strdup(odr, "IR-Explain---1 only supports CQL");
-        return m_backend;
     }
 }
 
@@ -2001,7 +2095,7 @@ static bool wait_conn(COMSTACK cs, int secs)
     struct yaz_poll_fd pfd;
 
     yaz_poll_add(pfd.input_mask, yaz_poll_except);
-    if (cs->io_pending && CS_WANT_WRITE)
+    if (cs->io_pending & CS_WANT_WRITE)
         yaz_poll_add(pfd.input_mask, yaz_poll_write);
     if (cs->io_pending & CS_WANT_READ)
         yaz_poll_add(pfd.input_mask, yaz_poll_read);
@@ -2073,7 +2167,8 @@ bool yf::Zoom::Frontend::retry(mp::Package &package,
         error = YAZ_BIB1_PROXY_FAILURE;
         *addinfo = odr_strdup(odr, b->m_proxy.c_str());
     }
-    else if (same_retries == 0 && proxy_retries == 0)
+    else if (b && b->retry_on_failure.compare("0")
+             && same_retries == 0 && proxy_retries == 0)
     {
         log_diagnostic(package, error, *addinfo);
         same_retries++;
@@ -2110,8 +2205,9 @@ next_proxy:
     char *addinfo = 0;
     std::string db(sr->databaseNames[0]);
 
-    BackendPtr b = get_backend_from_databases(package, db, &error,
-                                              &addinfo, odr, &proxy_step);
+    get_backend_from_databases(package, db, &error,
+                               &addinfo, odr, &proxy_step);
+    BackendPtr b = m_backend;
     if (error)
     {
         if (retry(package, odr, b, error, &addinfo, proxy_step,
@@ -2308,48 +2404,47 @@ next_proxy:
     ZOOM_query q = ZOOM_query_create();
     ZOOM_query_sortby2(q, b->sptr->sortStrategy.c_str(), sortkeys.c_str());
 
+    Z_FacetList *fl = 0;
+
+    // Facets for request.. And later for reponse
+    if (!fl)
+        fl = yaz_oi_get_facetlist(&sr->otherInfo);
+    if (!fl)
+        fl = yaz_oi_get_facetlist(&sr->additionalSearchInfo);
+
     if (b->get_option("sru"))
     {
-        int status = 0;
         Z_RPNQuery *zquery;
         zquery = p_query_rpn(odr, wrbuf_cstr(pqf_wrbuf));
-        mp::wrbuf wrb;
+        mp::wrbuf wrb_cql;
+        mp::wrbuf wrb_addinfo;
 
         if (!strcmp(b->get_option("sru"), "solr"))
-        {
-            solr_transform_t cqlt = solr_transform_create();
-
-            status = solr_transform_rpn2solr_wrbuf(cqlt, wrb, zquery);
-
-            solr_transform_close(cqlt);
-        }
+            error = solr_transform_rpn2solr_stream_r(b->cqlt, wrb_addinfo,
+                                                     wrbuf_vp_puts, wrb_cql,
+                                                     zquery);
         else
+            error = cql_transform_rpn2cql_stream_r(b->cqlt, wrb_addinfo,
+                                                   wrbuf_vp_puts, wrb_cql,
+                                                   zquery);
+        if (error)
         {
-            status = cql_transform_rpn2cql_wrbuf(b->cqlt, wrb, zquery);
-        }
-        if (status == 0)
-        {
-            ZOOM_query_cql(q, wrbuf_cstr(wrb));
-            package.log("zoom", YLOG_LOG, "CQL: %s", wrbuf_cstr(wrb));
-            b->search(q, &hits, &error, &addinfo, odr);
-        }
-        ZOOM_query_destroy(q);
-
-        if (status)
-        {
-            error = YAZ_BIB1_MALFORMED_QUERY;
-            const char *addinfo = "can not convert from RPN to CQL/Solr";
-            log_diagnostic(package, error, addinfo);
-            apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
+            log_diagnostic(package, error, wrb_addinfo.c_str_null());
+            apdu_res = odr.create_searchResponse(apdu_req, error,
+                                                 wrb_addinfo.c_str_null());
             package.response() = apdu_res;
             return;
         }
+        ZOOM_query_cql(q, wrb_cql.c_str());
+        package.log("zoom", YLOG_LOG, "search CQL: %s", wrb_cql.c_str());
+        b->search(q, &hits, &error, &addinfo, &fl, odr);
+        ZOOM_query_destroy(q);
     }
     else
     {
-        ZOOM_query_prefix(q, wrbuf_cstr(pqf_wrbuf));
-        package.log("zoom", YLOG_LOG, "search PQF: %s", wrbuf_cstr(pqf_wrbuf));
-        b->search(q, &hits, &error, &addinfo, odr);
+        ZOOM_query_prefix(q, pqf_wrbuf.c_str());
+        package.log("zoom", YLOG_LOG, "search PQF: %s", pqf_wrbuf.c_str());
+        b->search(q, &hits, &error, &addinfo, &fl, odr);
         ZOOM_query_destroy(q);
     }
 
@@ -2381,6 +2476,9 @@ next_proxy:
             odr_intdup(odr, number_of_records_returned);
     }
     apdu_res->u.searchResponse->resultCount = odr_intdup(odr, hits);
+    if (fl)
+        yaz_oi_set_facetlist(&apdu_res->u.searchResponse->additionalSearchInfo,
+                             odr, fl);
     package.response() = apdu_res;
 }
 
@@ -2545,9 +2643,32 @@ void yf::Zoom::Frontend::auth(mp::Package &package, Z_InitRequest *req,
         }
     }
 
-    std::string ip = package.origin().get_address();
+    Z_OtherInformation **oi = &req->otherInfo;
+    const char *ip_cstr =
+        yaz_oi_get_string_oid(oi, yaz_oid_userinfo_client_ip, 1, 0);
+    std::string ip;
+    if (ip_cstr)
+        ip = ip_cstr;
+    else
+        ip = package.origin().get_address();
+
     yaz_log(YLOG_LOG, "IP=%s", ip.c_str());
 
+    {
+        NMEM nmem = nmem_create();
+        char **darray;
+        int i, num;
+        nmem_strsplit_blank(nmem, m_p->torus_allow_ip.c_str(), &darray, &num);
+        for (i = 0; i < num; i++)
+        {
+            yaz_log(YLOG_LOG, "check against %s+%s", darray[i], ip.c_str());
+            if (yaz_match_glob(darray[i], ip.c_str()))
+                break;
+        }
+        nmem_destroy(nmem);
+        if (i < num)
+            return;  /* allow this IP */
+    }
     std::string torus_query;
     int failure_code;
 
@@ -2559,27 +2680,31 @@ void yf::Zoom::Frontend::auth(mp::Package &package, Z_InitRequest *req,
     }
     else
     {
-        const char *ip_cstr = ip.c_str();
-        const char *cp = strchr(ip_cstr, ':');
-        if (cp)
-            ip_cstr = cp + 1;
-
-        torus_query = "ip encloses/net.ipaddress \"";
-        torus_query += escape_cql_term(std::string(ip_cstr));
+        torus_query = "ipRanges encloses/net.ipaddress \"";
+        torus_query += escape_cql_term(std::string(ip));
         torus_query += "\"";
+
+        if (m_p->torus_auth_hostname.length())
+        {
+            torus_query += " AND hostName == \"";
+            torus_query += escape_cql_term(m_p->torus_auth_hostname);
+            torus_query += "\"";
+        }
         failure_code = YAZ_BIB1_INIT_AC_BLOCKED_NETWORK_ADDRESS;
     }
 
     std::string dummy_db;
     std::string dummy_realm;
+    std::string torus_addinfo;
     xmlDoc *doc = mp::get_searchable(package, m_p->torus_auth_url, dummy_db,
-                                     torus_query, dummy_realm, m_p->proxy);
+                                     torus_query, dummy_realm, m_p->proxy,
+                                     torus_addinfo);
     if (!doc)
     {
         // something fundamental broken in lookup.
         *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
-        *addinfo = odr_strdup(odr, "zoom: torus server unavailable or "
-                              "incorrectly configured.");
+        if (torus_addinfo.length())
+            *addinfo = odr_strdup(odr, torus_addinfo.c_str());
         return;
     }
     const xmlNode *ptr = xmlDocGetRootElement(doc);