Fix multi: hideunavailable don't hide targets that timeout MP-615
[metaproxy-moved-to-github.git] / src / filter_multi.cpp
index 1165cbd..5c581e5 100644 (file)
@@ -1,5 +1,5 @@
 /* This file is part of Metaproxy.
-   Copyright (C) 2005-2011 Index Data
+   Copyright (C) Index Data
 
 Metaproxy is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free
@@ -35,6 +35,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #include <yaz/otherinfo.h>
 #include <yaz/diagbib1.h>
 #include <yaz/match_glob.h>
+#include <yaz/oid_db.h>
 
 #include <vector>
 #include <algorithm>
@@ -70,7 +71,7 @@ namespace metaproxy_1 {
                 BackendPtr m_backend;
                 int m_pos; // position for backend (1=first, 2=second,..
                 int m_start; // present request start
-                PresentJob(BackendPtr ptr, int pos) : 
+                PresentJob(BackendPtr ptr, int pos) :
                     m_backend(ptr), m_pos(pos), m_start(0) {};
             };
             FrontendSet(std::string setname);
@@ -88,6 +89,7 @@ namespace metaproxy_1 {
             std::string m_backend_database;
             std::string m_vhost;
             std::string m_route;
+            std::string m_auth;
             void operator() (void);  // thread operation
         };
         struct Multi::Frontend {
@@ -103,20 +105,27 @@ namespace metaproxy_1 {
             void close(Package &package);
             void search(Package &package, Z_APDU *apdu);
             void present(Package &package, Z_APDU *apdu);
-            void scan1(Package &package, Z_APDU *apdu);
-            void scan2(Package &package, Z_APDU *apdu);
+            void scan(Package &package, Z_APDU *apdu);
+            void relay_apdu(Package &package, Z_APDU *apdu);
+            void record_diagnostics(Z_Records *records, 
+                                    Z_DiagRecs * &z_diag,
+                                    ODR odr,
+                                    int &no_successful);
             Rep *m_p;
-        };            
+        };
         class Multi::Map {
             std::string m_target_pattern;
             std::string m_route;
+            std::string m_auth;
         public:
-            Map(std::string pattern, std::string route) : 
-                m_target_pattern(pattern), m_route(route) {};
-            bool match(const std::string target, std::string *ret) const {
+            Map(std::string pattern, std::string route, std::string auth) :
+                m_target_pattern(pattern), m_route(route), m_auth(auth) {};
+            bool match(const std::string target, std::string *ret,
+                       std::string *auth) const {
                 if (yaz_match_glob(m_target_pattern.c_str(), target.c_str()))
                 {
                     *ret = m_route;
+                    *auth = m_auth;
                     return true;
                 }
                 return false;
@@ -125,7 +134,7 @@ namespace metaproxy_1 {
         class Multi::Rep {
             friend class Multi;
             friend struct Frontend;
-            
+
             Rep();
             FrontendPtr get_frontend(Package &package);
             void release_frontend(Package &package);
@@ -135,6 +144,7 @@ namespace metaproxy_1 {
             boost::condition m_cond_session_ready;
             std::map<mp::Session, FrontendPtr> m_clients;
             bool m_hide_unavailable;
+            bool m_hide_errors;
             multi_merge_type m_merge_type;
         };
     }
@@ -143,6 +153,7 @@ namespace metaproxy_1 {
 yf::Multi::Rep::Rep()
 {
     m_hide_unavailable = false;
+    m_hide_errors = false;
     m_merge_type = round_robin;
 }
 
@@ -166,13 +177,13 @@ yf::Multi::FrontendPtr yf::Multi::Rep::get_frontend(mp::Package &package)
     boost::mutex::scoped_lock lock(m_mutex);
 
     std::map<mp::Session,yf::Multi::FrontendPtr>::iterator it;
-    
+
     while(true)
     {
         it = m_clients.find(package.session());
         if (it == m_clients.end())
             break;
-        
+
         if (!it->second->m_in_use)
         {
             it->second->m_in_use = true;
@@ -190,7 +201,7 @@ void yf::Multi::Rep::release_frontend(mp::Package &package)
 {
     boost::mutex::scoped_lock lock(m_mutex);
     std::map<mp::Session,yf::Multi::FrontendPtr>::iterator it;
-    
+
     it = m_clients.find(package.session());
     if (it != m_clients.end())
     {
@@ -230,7 +241,7 @@ yf::Multi::~Multi() {
 }
 
 
-void yf::Multi::Backend::operator() (void) 
+void yf::Multi::Backend::operator() (void)
 {
     m_package->move(m_route);
 }
@@ -270,7 +281,7 @@ void yf::Multi::FrontendSet::serve_order(int start, int number,
         std::list<BackendSet>::const_iterator bsit;
         int voffset = 0;
         int offset = start + i - 1;
-        for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); 
+        for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end();
              bsit++)
         {
             if (offset >= voffset && offset < voffset + bsit->m_count)
@@ -320,7 +331,7 @@ void yf::Multi::FrontendSet::round_robin(int start, int number,
             // Yes. skip until start.. Rounding off is deliberate!
             min = (start-p) / no_left;
             p += no_left * min;
-            
+
             // update positions in each set..
             std::list<int>::iterator psit = pos.begin();
             for (psit = pos.begin(); psit != pos.end(); psit++)
@@ -329,11 +340,11 @@ void yf::Multi::FrontendSet::round_robin(int start, int number,
         }
         // skip on each set.. before "present range"..
         p = p + skip;
-        
+
         std::list<int>::iterator psit = pos.begin();
         for (psit = pos.begin(); psit != pos.end(); psit++)
             *psit += min;
-        
+
         omin = min; // update so we consider next class (with higher count)
     }
 #endif
@@ -392,7 +403,7 @@ void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu)
         std::list<Multi::Map>::const_iterator it =
             m_p->m_route_patterns.begin();
         while (it != m_p->m_route_patterns.end()) {
-            if (it->match(*t_it, &b->m_route))
+            if (it->match(*t_it, &b->m_route, &b->m_auth))
                 break;
             it++;
         }
@@ -404,14 +415,14 @@ void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu)
     }
     m_is_multi = true;
 
-    // create init request 
+    // create init request
     std::list<BackendPtr>::iterator bit;
     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
     {
         mp::odr odr;
         BackendPtr b = *bit;
         Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest);
-        
+
         std::list<std::string>vhost_one;
         vhost_one.push_back(b->m_vhost);
         mp::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo,
@@ -420,17 +431,36 @@ void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu)
 
         Z_InitRequest *breq = init_apdu->u.initRequest;
 
-        breq->idAuthentication = req->idAuthentication;
-        
+        if (b->m_auth.length())
+        {
+            breq->idAuthentication =
+                (Z_IdAuthentication *)
+                odr_malloc(odr, sizeof(*breq->idAuthentication));
+            breq->idAuthentication->which = Z_IdAuthentication_open;
+            breq->idAuthentication->u.open = odr_strdup(odr, b->m_auth.c_str());
+        }
+        else
+            breq->idAuthentication = req->idAuthentication;
+
+        *breq->preferredMessageSize = *req->preferredMessageSize;
+        *breq->maximumRecordSize = *req->maximumRecordSize;
+
+
+        const char *peer_name = yaz_oi_get_string_oid(
+            &req->otherInfo, yaz_oid_userinfo_client_ip, 1, 0);
+        if (peer_name)
+            yaz_oi_set_string_oid(&breq->otherInfo, odr,
+                                  yaz_oid_userinfo_client_ip, 1, peer_name);
+
         ODR_MASK_SET(breq->options, Z_Options_search);
         ODR_MASK_SET(breq->options, Z_Options_present);
         ODR_MASK_SET(breq->options, Z_Options_namedResultSets);
         ODR_MASK_SET(breq->options, Z_Options_scan);
-        
+
         ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_1);
         ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_2);
         ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_3);
-        
+
         b->m_package->request() = init_apdu;
 
         b->m_package->copy_filter(package);
@@ -447,17 +477,20 @@ void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu)
     ODR_MASK_SET(f_resp->options, Z_Options_present);
     ODR_MASK_SET(f_resp->options, Z_Options_namedResultSets);
     ODR_MASK_SET(f_resp->options, Z_Options_scan);
-    
+
     ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_1);
     ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_2);
     ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_3);
 
     int no_failed = 0;
     int no_succeeded = 0;
+
+    Odr_int preferredMessageSize = *req->preferredMessageSize;
+    Odr_int maximumRecordSize = *req->maximumRecordSize;
     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); )
     {
         PackagePtr p = (*bit)->m_package;
-        
+
         if (p->session().is_closed())
         {
             // failed. Remove from list and increment number of failed
@@ -484,14 +517,28 @@ void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu)
                 if (!ODR_MASK_GET(b_resp->protocolVersion, i))
                     ODR_MASK_CLEAR(f_resp->protocolVersion, i);
             if (*b_resp->result)
+            {
                 no_succeeded++;
+                if (preferredMessageSize > *b_resp->preferredMessageSize)
+                    preferredMessageSize = *b_resp->preferredMessageSize;
+                if (maximumRecordSize > *b_resp->maximumRecordSize)
+                    maximumRecordSize = *b_resp->maximumRecordSize;
+            }
             else
+            {
+                if (!f_resp->userInformationField
+                    && b_resp->userInformationField)
+                    f_resp->userInformationField = b_resp->userInformationField;
                 no_failed++;
+            }
         }
         else
             no_failed++;
         bit++;
     }
+    *f_resp->preferredMessageSize = preferredMessageSize;
+    *f_resp->maximumRecordSize = maximumRecordSize;
+
     if (m_p->m_hide_unavailable)
     {
         if (no_succeeded == 0)
@@ -511,38 +558,93 @@ void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu)
     package.response() = f_apdu;
 }
 
-void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
+void yf::Multi::Frontend::record_diagnostics(Z_Records *records, 
+                                             Z_DiagRecs * &z_diag,
+                                             ODR odr,
+                                             int &no_successful)
 {
-    // create search request 
-    Z_SearchRequest *req = apdu_req->u.searchRequest;
+    // see we get any errors (AKA diagnstics)
+    if (records)
+    {
+        if (records->which == Z_Records_NSD)
+        {
+            if (!z_diag)
+            {
+                z_diag = (Z_DiagRecs *)
+                    odr_malloc(odr, sizeof(*z_diag));
+                z_diag->num_diagRecs = 0;
+                z_diag->diagRecs = (Z_DiagRec**)
+                    odr_malloc(odr, sizeof(*z_diag->diagRecs));
+            }
+            else
+            {
+                Z_DiagRec **n = (Z_DiagRec **)
+                    odr_malloc(odr,
+                               (1 + z_diag->num_diagRecs) * sizeof(*n));
+                memcpy(n, z_diag->diagRecs, z_diag->num_diagRecs
+                       * sizeof(*n));
+                z_diag->diagRecs = n;
+            }
+            Z_DiagRec *nr = (Z_DiagRec *) odr_malloc(odr, sizeof(*nr));
+            nr->which = Z_DiagRec_defaultFormat;
+            nr->u.defaultFormat =
+                records->u.nonSurrogateDiagnostic;
+            z_diag->diagRecs[z_diag->num_diagRecs++] = nr;
+        }
+        else if (records->which == Z_Records_multipleNSD)
+        {
+            Z_DiagRecs * dr =records->u.multipleNonSurDiagnostics;
 
-    // save these for later
-    int smallSetUpperBound = *req->smallSetUpperBound;
-    int largeSetLowerBound = *req->largeSetLowerBound;
-    int mediumSetPresentNumber = *req->mediumSetPresentNumber;
-    
-    // they are altered now - to disable piggyback
-    *req->smallSetUpperBound = 0;
-    *req->largeSetLowerBound = 1;
-    *req->mediumSetPresentNumber = 1;
+            if (!z_diag)
+            {
+                z_diag = (Z_DiagRecs *) odr_malloc(odr, sizeof(*z_diag));
+                z_diag->num_diagRecs = 0;
+                z_diag->diagRecs = 0;
+            }
+            Z_DiagRec **n = (Z_DiagRec **)
+                odr_malloc(odr,
+                           (dr->num_diagRecs + z_diag->num_diagRecs) * 
+                           sizeof(*n));
+            if (z_diag->num_diagRecs)
+                memcpy(n, z_diag->diagRecs, z_diag->num_diagRecs * sizeof(*n));
+            memcpy(n + z_diag->num_diagRecs,
+                   dr->diagRecs, dr->num_diagRecs * sizeof(*n));
+            z_diag->diagRecs = n;
+            z_diag->num_diagRecs += dr->num_diagRecs;
+        }
+        else
+            no_successful++; // probably piggyback
+    }
+    else
+        no_successful++;  // no records and no diagnostics
+}
 
-    int default_num_db = req->num_databaseNames;
-    char **default_db = req->databaseNames;
+void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
+{
+    // create search request
+    Z_SearchRequest *req = apdu_req->u.searchRequest;
 
     std::list<BackendPtr>::const_iterator bit;
     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
     {
         PackagePtr p = (*bit)->m_package;
+        yazpp_1::GDU gdu1(apdu_req);
         mp::odr odr;
-    
+        Z_SearchRequest *req1 = gdu1.get()->u.z3950->u.searchRequest;
+
+        // they are altered now - to disable piggyback
+        *req1->smallSetUpperBound = 0;
+        *req1->largeSetLowerBound = 1;
+        *req1->mediumSetPresentNumber = 0;
+
         if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
-                                                &req->num_databaseNames,
-                                                &req->databaseNames))
+                                               &req1->num_databaseNames,
+                                               &req1->databaseNames))
         {
-            req->num_databaseNames = default_num_db;
-            req->databaseNames = default_db;
+            req1->num_databaseNames = req->num_databaseNames;
+            req1->databaseNames = req->databaseNames;
         }
-        p->request() = apdu_req;
+        p->request() = gdu1;
         p->copy_filter(package);
     }
     multi_move(m_backend_list);
@@ -550,67 +652,79 @@ void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
     // look at each response
     FrontendSet resultSet(std::string(req->resultSetName));
 
-    int result_set_size = 0;
-    Z_Records *z_records_diag = 0;  // no diagnostics (yet)
+    mp::odr odr;
+    Odr_int result_set_size = 0;
+    Z_DiagRecs *z_diag = 0;
+    int no_successful = 0;
+    PackagePtr close_p;
+
     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
     {
         PackagePtr p = (*bit)->m_package;
-        
-        if (p->session().is_closed()) // if any backend closes, close frontend
-            package.session().close();
-        
+
+        // save closing package for at least one target
+        if (p->session().is_closed())
+            close_p = p;
+
         Z_GDU *gdu = p->response().get();
         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
             Z_APDU_searchResponse)
         {
             Z_APDU *b_apdu = gdu->u.z3950;
             Z_SearchResponse *b_resp = b_apdu->u.searchResponse;
-         
-            // see we get any errors (AKA diagnstics)
-            if (b_resp->records)
-            {
-                if (b_resp->records->which == Z_Records_NSD
-                    || b_resp->records->which == Z_Records_multipleNSD)
-                    z_records_diag = b_resp->records;
-                // we may set this multiple times (TOO BAD!)
-            }
+
+            record_diagnostics(b_resp->records, z_diag, odr, no_successful);
+
             BackendSet backendSet;
             backendSet.m_backend = *bit;
             backendSet.m_count = *b_resp->resultCount;
             result_set_size += *b_resp->resultCount;
             resultSet.m_backend_sets.push_back(backendSet);
         }
-        else
-        {
-            // if any target does not return search response - return that 
-            package.response() = p->response();
-            return;
-        }
     }
 
-    mp::odr odr;
     Z_APDU *f_apdu = odr.create_searchResponse(apdu_req, 0, 0);
     Z_SearchResponse *f_resp = f_apdu->u.searchResponse;
 
+    yaz_log(YLOG_DEBUG, "no_successful=%d is_closed=%s hide_errors=%s",
+            no_successful,
+            close_p ? "true" : "false",
+            m_p->m_hide_errors ? "true" : "false");
     *f_resp->resultCount = result_set_size;
-    if (z_records_diag)
+    if (close_p && (no_successful == 0 || !m_p->m_hide_unavailable))
     {
-        // search error
-        f_resp->records = z_records_diag;
-        package.response() = f_apdu;
+        package.session().close();
+        package.response() = close_p->response();
         return;
     }
+    if (z_diag && (no_successful == 0 || !m_p->m_hide_errors))
+    {
+        f_resp->records = (Z_Records *)
+            odr_malloc(odr, sizeof(*f_resp->records));
+        if (z_diag->num_diagRecs > 1)
+        {
+            f_resp->records->which = Z_Records_multipleNSD;
+            f_resp->records->u.multipleNonSurDiagnostics = z_diag;
+        }
+        else
+        {
+            f_resp->records->which = Z_Records_NSD;
+            f_resp->records->u.nonSurrogateDiagnostic =
+                z_diag->diagRecs[0]->u.defaultFormat;
+        }
+    }
     // assume OK
     m_sets[resultSet.m_setname] = resultSet;
 
-    int number;
-    mp::util::piggyback(smallSetUpperBound,
-                         largeSetLowerBound,
-                         mediumSetPresentNumber,
-                         result_set_size,
-                         number);
+    Odr_int number;
+    mp::util::piggyback(*req->smallSetUpperBound,
+                        *req->largeSetLowerBound,
+                        *req->mediumSetPresentNumber,
+                        0, 0,
+                        result_set_size,
+                        number, 0);
     Package pp(package.session(), package.origin());
-    if (number > 0)
+    if (z_diag == 0 && number > 0)
     {
         pp.copy_filter(package);
         Z_APDU *p_apdu = zget_APDU(odr, Z_APDU_presentRequest);
@@ -621,24 +735,24 @@ void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
         *p_req->numberOfRecordsRequested = number;
         pp.request() = p_apdu;
         present(pp, p_apdu);
-        
+
         if (pp.session().is_closed())
             package.session().close();
-        
+
         Z_GDU *gdu = pp.response().get();
         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
             Z_APDU_presentResponse)
         {
             Z_PresentResponse *p_res = gdu->u.z3950->u.presentResponse;
             f_resp->records = p_res->records;
-            *f_resp->numberOfRecordsReturned = 
+            *f_resp->numberOfRecordsReturned =
                 *p_res->numberOfRecordsReturned;
-            *f_resp->nextResultSetPosition = 
+            *f_resp->nextResultSetPosition =
                 *p_res->nextResultSetPosition;
         }
-        else 
+        else
         {
-            package.response() = pp.response(); 
+            package.response() = pp.response();
             return;
         }
     }
@@ -647,7 +761,7 @@ void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
 
 void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
 {
-    // create present request 
+    // create present request
     Z_PresentRequest *req = apdu_req->u.presentRequest;
 
     Sets_it it;
@@ -655,7 +769,7 @@ void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
     if (it == m_sets.end())
     {
         mp::odr odr;
-        Z_APDU *apdu = 
+        Z_APDU *apdu =
             odr.create_presentResponse(
                 apdu_req,
                 YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
@@ -677,7 +791,7 @@ void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
         std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
         for (jit = jobs.begin(); jit != jobs.end(); jit++)
         {
-            yaz_log(YLOG_LOG, "job pos=%d", jit->m_pos);
+            yaz_log(YLOG_DEBUG, "job pos=%d", jit->m_pos);
         }
     }
 
@@ -718,7 +832,7 @@ void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
 
             *req->resultSetStartPoint = start;
             *req->numberOfRecordsRequested = end - start + 1;
-            
+
             p->request() = apdu_req;
             p->copy_filter(package);
 
@@ -728,51 +842,59 @@ void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
     multi_move(present_backend_list);
 
     // look at each response
-    Z_Records *z_records_diag = 0;
+    Z_DiagRecs *z_diag = 0;
+    int no_successful = 0;
+    mp::odr odr;
+    PackagePtr close_p;
 
     std::list<BackendPtr>::const_iterator pbit = present_backend_list.begin();
     for (; pbit != present_backend_list.end(); pbit++)
     {
         PackagePtr p = (*pbit)->m_package;
-        
-        if (p->session().is_closed()) // if any backend closes, close frontend
-            package.session().close();
-        
+
+        // save closing package for at least one target
+        if (p->session().is_closed())
+            close_p = p;
+
         Z_GDU *gdu = p->response().get();
         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
             Z_APDU_presentResponse)
         {
             Z_APDU *b_apdu = gdu->u.z3950;
             Z_PresentResponse *b_resp = b_apdu->u.presentResponse;
-         
-            // see we get any errors (AKA diagnstics)
-            if (b_resp->records)
-            {
-                if (b_resp->records->which != Z_Records_DBOSD)
-                    z_records_diag = b_resp->records;
-                // we may set this multiple times (TOO BAD!)
-            }
-        }
-        else
-        {
-            // if any target does not return present response - return that 
-            package.response() = p->response();
-            return;
+
+            record_diagnostics(b_resp->records, z_diag, odr, no_successful);
         }
     }
 
-    mp::odr odr;
     Z_APDU *f_apdu = odr.create_presentResponse(apdu_req, 0, 0);
     Z_PresentResponse *f_resp = f_apdu->u.presentResponse;
 
-    if (z_records_diag)
+    if (close_p && (no_successful == 0 || !m_p->m_hide_errors))
     {
-        f_resp->records = z_records_diag;
-        *f_resp->presentStatus = Z_PresentStatus_failure;
+        package.session().close();
+        package.response() = close_p->response();
+        return;
+    }
+    if (z_diag && (no_successful == 0 || !m_p->m_hide_errors))
+    {
+        f_resp->records = (Z_Records *)
+            odr_malloc(odr, sizeof(*f_resp->records));
+        if (z_diag->num_diagRecs > 1)
+        {
+            f_resp->records->which = Z_Records_multipleNSD;
+            f_resp->records->u.multipleNonSurDiagnostics = z_diag;
+        }
+        else
+        {
+            f_resp->records->which = Z_Records_NSD;
+            f_resp->records->u.nonSurrogateDiagnostic =
+                z_diag->diagRecs[0]->u.defaultFormat;
+        }
     }
     else if (number < 0 || (size_t) number > jobs.size())
     {
-        f_apdu = 
+        f_apdu =
             odr.create_presentResponse(
                 apdu_req,
                 YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
@@ -792,24 +914,27 @@ void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
             odr_malloc(odr, sizeof(Z_NamePlusRecord *) * nprl->num_records);
         int i = 0;
         std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
-        for (jit = jobs.begin(); jit != jobs.end(); jit++, i++)
+        for (jit = jobs.begin(); jit != jobs.end(); jit++)
         {
             PackagePtr p = jit->m_backend->m_package;
-            
+
             Z_GDU *gdu = p->response().get();
             Z_APDU *b_apdu = gdu->u.z3950;
-            Z_PresentResponse *b_resp = b_apdu->u.presentResponse;
-
-            nprl->records[i] = (Z_NamePlusRecord*)
-                odr_malloc(odr, sizeof(Z_NamePlusRecord));
             int inside_pos = jit->m_pos - jit->m_start;
-            if (inside_pos >= b_resp->records->
-                u.databaseOrSurDiagnostics->num_records)
-                break;
-           *nprl->records[i] = *b_resp->records->
-                u.databaseOrSurDiagnostics->records[inside_pos];
-            nprl->records[i]->databaseName =
+            Z_Records *records = b_apdu->u.presentResponse->records;
+
+            if (records && records->which == Z_Records_DBOSD
+                && inside_pos < 
+                records->u.databaseOrSurDiagnostics->num_records)
+            {
+                nprl->records[i] = (Z_NamePlusRecord*)
+                    odr_malloc(odr, sizeof(Z_NamePlusRecord));
+                *nprl->records[i] = *records->
+                    u.databaseOrSurDiagnostics->records[inside_pos];
+                nprl->records[i]->databaseName =
                     odr_strdup(odr, jit->m_backend->m_vhost.c_str());
+                i++;
+            }
         }
         nprl->num_records = i; // usually same as jobs.size();
         *f_resp->nextResultSetPosition = start + i;
@@ -818,63 +943,6 @@ void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
     package.response() = f_apdu;
 }
 
-void yf::Multi::Frontend::scan1(mp::Package &package, Z_APDU *apdu_req)
-{
-    if (m_backend_list.size() > 1)
-    {
-        mp::odr odr;
-        Z_APDU *f_apdu = 
-            odr.create_scanResponse(
-                apdu_req, YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP, 0);
-        package.response() = f_apdu;
-        return;
-    }
-    Z_ScanRequest *req = apdu_req->u.scanRequest;
-
-    int default_num_db = req->num_databaseNames;
-    char **default_db = req->databaseNames;
-
-    std::list<BackendPtr>::const_iterator bit;
-    for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
-    {
-        PackagePtr p = (*bit)->m_package;
-        mp::odr odr;
-    
-        if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
-                                                &req->num_databaseNames,
-                                                &req->databaseNames))
-        {
-            req->num_databaseNames = default_num_db;
-            req->databaseNames = default_db;
-        }
-        p->request() = apdu_req;
-        p->copy_filter(package);
-    }
-    multi_move(m_backend_list);
-
-    for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
-    {
-        PackagePtr p = (*bit)->m_package;
-        
-        if (p->session().is_closed()) // if any backend closes, close frontend
-            package.session().close();
-        
-        Z_GDU *gdu = p->response().get();
-        if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
-            Z_APDU_scanResponse)
-        {
-            package.response() = p->response();
-            break;
-        }
-        else
-        {
-            // if any target does not return scan response - return that 
-            package.response() = p->response();
-            return;
-        }
-    }
-}
-
 bool yf::Multi::ScanTermInfo::operator < (const ScanTermInfo &k) const
 {
     return m_norm_term < k.m_norm_term;
@@ -897,39 +965,57 @@ Z_Entry *yf::Multi::ScanTermInfo::get_entry(ODR odr)
     t->byAttributes = 0;
     t->otherTermInfo = 0;
     t->globalOccurrences = odr_intdup(odr, m_count);
-    t->term = (Z_Term *)
-        odr_malloc(odr, sizeof(*t->term));
+    t->term = (Z_Term *) odr_malloc(odr, sizeof(*t->term));
     t->term->which = Z_Term_general;
-    Odr_oct *o;
-    t->term->u.general = o = (Odr_oct *)odr_malloc(odr, sizeof(Odr_oct));
-
-    o->len = o->size = m_norm_term.size();
-    o->buf = (unsigned char *) odr_malloc(odr, o->len);
-    memcpy(o->buf, m_norm_term.c_str(), o->len);
+    t->term->u.general = odr_create_Odr_oct(odr,
+                                 m_norm_term.c_str(), m_norm_term.size());
     return e;
 }
 
-void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req)
+void yf::Multi::Frontend::relay_apdu(mp::Package &package, Z_APDU *apdu_req)
 {
-    Z_ScanRequest *req = apdu_req->u.scanRequest;
+    std::list<BackendPtr>::const_iterator bit;
+    for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
+    {
+        PackagePtr p = (*bit)->m_package;
+        mp::odr odr;
+
+        p->request() = apdu_req;
+        p->copy_filter(package);
+    }
+    multi_move(m_backend_list);
+    for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
+    {
+        PackagePtr p = (*bit)->m_package;
+
+        if (p->session().is_closed()) // if any backend closes, close frontend
+            package.session().close();
+
+        package.response() = p->response();
+    }
+}
+
 
-    int default_num_db = req->num_databaseNames;
-    char **default_db = req->databaseNames;
+void yf::Multi::Frontend::scan(mp::Package &package, Z_APDU *apdu_req)
+{
+    Z_ScanRequest *req = apdu_req->u.scanRequest;
 
     std::list<BackendPtr>::const_iterator bit;
     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
     {
         PackagePtr p = (*bit)->m_package;
+        yazpp_1::GDU gdu1(apdu_req);
         mp::odr odr;
-    
+        Z_ScanRequest *req1 = gdu1.get()->u.z3950->u.scanRequest;
+
         if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
-                                                &req->num_databaseNames,
-                                                &req->databaseNames))
+                                               &req1->num_databaseNames,
+                                               &req1->databaseNames))
         {
-            req->num_databaseNames = default_num_db;
-            req->databaseNames = default_db;
+            req1->num_databaseNames = req->num_databaseNames;
+            req1->databaseNames = req->databaseNames;
         }
-        p->request() = apdu_req;
+        p->request() = gdu1;
         p->copy_filter(package);
     }
     multi_move(m_backend_list);
@@ -942,10 +1028,10 @@ void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req)
     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
     {
         PackagePtr p = (*bit)->m_package;
-        
+
         if (p->session().is_closed()) // if any backend closes, close frontend
             package.session().close();
-        
+
         Z_GDU *gdu = p->response().get();
         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
             Z_APDU_scanResponse)
@@ -959,9 +1045,9 @@ void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req)
                 Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 1, 0);
                 Z_ScanResponse *f_res = f_apdu->u.scanResponse;
 
-                f_res->entries->nonsurrogateDiagnostics = 
+                f_res->entries->nonsurrogateDiagnostics =
                     res->entries->nonsurrogateDiagnostics;
-                f_res->entries->num_nonsurrogateDiagnostics = 
+                f_res->entries->num_nonsurrogateDiagnostics =
                     res->entries->num_nonsurrogateDiagnostics;
 
                 package.response() = f_apdu;
@@ -1000,11 +1086,11 @@ void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req)
                         }
                         if (my.m_norm_term.length())
                         {
-                            ScanTermInfoList::iterator it = 
+                            ScanTermInfoList::iterator it =
                                 entries_before.begin();
                             while (it != entries_before.end() && my <*it)
                                 it++;
-                            if (my == *it)
+                            if (it != entries_before.end() && my == *it)
                             {
                                 it->m_count += my.m_count;
                             }
@@ -1041,11 +1127,11 @@ void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req)
                         }
                         if (my.m_norm_term.length())
                         {
-                            ScanTermInfoList::iterator it = 
+                            ScanTermInfoList::iterator it =
                                 entries_after.begin();
                             while (it != entries_after.end() && *it < my)
                                 it++;
-                            if (my == *it)
+                            if (it != entries_after.end() && my == *it)
                             {
                                 it->m_count += my.m_count;
                             }
@@ -1058,11 +1144,11 @@ void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req)
                     }
                 }
 
-            }                
+            }
         }
         else
         {
-            // if any target does not return scan response - return that 
+            // if any target does not return scan response - return that
             package.response() = p->response();
             return;
         }
@@ -1076,7 +1162,7 @@ void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req)
         {
             std::cout << " " << it->m_norm_term << " " << it->m_count << "\n";
         }
-        
+
         std::cout << "AFTER\n";
         it = entries_after.begin();
         for(; it != entries_after.end(); it++)
@@ -1096,10 +1182,10 @@ void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req)
         mp::odr odr;
         Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 0, 0);
         Z_ScanResponse *resp = f_apdu->u.scanResponse;
-        
+
         int number_returned = *req->numberOfTermsRequested;
         int position_returned = *req->preferredPositionInResponse;
-        
+
         resp->entries->num_entries = number_returned;
         resp->entries->entries = (Z_Entry**)
             odr_malloc(odr, sizeof(Z_Entry*) * number_returned);
@@ -1142,7 +1228,7 @@ void yf::Multi::process(mp::Package &package) const
     FrontendPtr f = m_p->get_frontend(package);
 
     Z_GDU *gdu = package.request().get();
-    
+
     if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
         Z_APDU_initRequest && !f->m_is_multi)
     {
@@ -1156,12 +1242,12 @@ void yf::Multi::process(mp::Package &package) const
         if (apdu->which == Z_APDU_initRequest)
         {
             mp::odr odr;
-            
+
             package.response() = odr.create_close(
                 apdu,
                 Z_Close_protocolError,
                 "double init");
-            
+
             package.session().close();
         }
         else if (apdu->which == Z_APDU_searchRequest)
@@ -1174,23 +1260,28 @@ void yf::Multi::process(mp::Package &package) const
         }
         else if (apdu->which == Z_APDU_scanRequest)
         {
-            f->scan2(package, apdu);
+            f->scan(package, apdu);
+        }
+        else if (apdu->which == Z_APDU_close)
+        {
+            f->relay_apdu(package, apdu);
         }
         else
         {
             mp::odr odr;
-            
+
             package.response() = odr.create_close(
                 apdu, Z_Close_protocolError,
                 "unsupported APDU in filter multi");
-            
+
             package.session().close();
         }
     }
     m_p->release_frontend(package);
 }
 
-void mp::filter::Multi::configure(const xmlNode * ptr, bool test_only)
+void mp::filter::Multi::configure(const xmlNode * ptr, bool test_only,
+                                  const char *path)
 {
     for (ptr = ptr->children; ptr; ptr = ptr->next)
     {
@@ -1198,14 +1289,21 @@ void mp::filter::Multi::configure(const xmlNode * ptr, bool test_only)
             continue;
         if (!strcmp((const char *) ptr->name, "target"))
         {
-            std::string route = mp::xml::get_route(ptr);
+            std::string auth;
+            std::string route = mp::xml::get_route(ptr, auth);
             std::string target = mp::xml::get_text(ptr);
-            m_p->m_route_patterns.push_back(Multi::Map(target, route));
+            if (target.length() == 0)
+                target = route;
+            m_p->m_route_patterns.push_back(Multi::Map(target, route, auth));
         }
         else if (!strcmp((const char *) ptr->name, "hideunavailable"))
         {
             m_p->m_hide_unavailable = true;
         }
+        else if (!strcmp((const char *) ptr->name, "hideerrors"))
+        {
+            m_p->m_hide_errors = true;
+        }
         else if (!strcmp((const char *) ptr->name, "mergetype"))
         {
             std::string mergetype = mp::xml::get_text(ptr);
@@ -1221,7 +1319,7 @@ void mp::filter::Multi::configure(const xmlNode * ptr, bool test_only)
         else
         {
             throw mp::filter::FilterException
-                ("Bad element " 
+                ("Bad element "
                  + std::string((const char *) ptr->name)
                  + " in multi filter");
         }