Fix multi: hideunavailable don't hide targets that timeout MP-615
[metaproxy-moved-to-github.git] / src / filter_multi.cpp
index ccefccf..5c581e5 100644 (file)
@@ -1,5 +1,5 @@
 /* This file is part of Metaproxy.
-   Copyright (C) 2005-2012 Index Data
+   Copyright (C) Index Data
 
 Metaproxy is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free
@@ -35,6 +35,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #include <yaz/otherinfo.h>
 #include <yaz/diagbib1.h>
 #include <yaz/match_glob.h>
+#include <yaz/oid_db.h>
 
 #include <vector>
 #include <algorithm>
@@ -104,9 +105,12 @@ namespace metaproxy_1 {
             void close(Package &package);
             void search(Package &package, Z_APDU *apdu);
             void present(Package &package, Z_APDU *apdu);
-            void scan1(Package &package, Z_APDU *apdu);
-            void scan2(Package &package, Z_APDU *apdu);
+            void scan(Package &package, Z_APDU *apdu);
             void relay_apdu(Package &package, Z_APDU *apdu);
+            void record_diagnostics(Z_Records *records, 
+                                    Z_DiagRecs * &z_diag,
+                                    ODR odr,
+                                    int &no_successful);
             Rep *m_p;
         };
         class Multi::Map {
@@ -441,6 +445,13 @@ void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu)
         *breq->preferredMessageSize = *req->preferredMessageSize;
         *breq->maximumRecordSize = *req->maximumRecordSize;
 
+
+        const char *peer_name = yaz_oi_get_string_oid(
+            &req->otherInfo, yaz_oid_userinfo_client_ip, 1, 0);
+        if (peer_name)
+            yaz_oi_set_string_oid(&breq->otherInfo, odr,
+                                  yaz_oid_userinfo_client_ip, 1, peer_name);
+
         ODR_MASK_SET(breq->options, Z_Options_search);
         ODR_MASK_SET(breq->options, Z_Options_present);
         ODR_MASK_SET(breq->options, Z_Options_namedResultSets);
@@ -547,38 +558,93 @@ void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu)
     package.response() = f_apdu;
 }
 
+void yf::Multi::Frontend::record_diagnostics(Z_Records *records, 
+                                             Z_DiagRecs * &z_diag,
+                                             ODR odr,
+                                             int &no_successful)
+{
+    // see we get any errors (AKA diagnstics)
+    if (records)
+    {
+        if (records->which == Z_Records_NSD)
+        {
+            if (!z_diag)
+            {
+                z_diag = (Z_DiagRecs *)
+                    odr_malloc(odr, sizeof(*z_diag));
+                z_diag->num_diagRecs = 0;
+                z_diag->diagRecs = (Z_DiagRec**)
+                    odr_malloc(odr, sizeof(*z_diag->diagRecs));
+            }
+            else
+            {
+                Z_DiagRec **n = (Z_DiagRec **)
+                    odr_malloc(odr,
+                               (1 + z_diag->num_diagRecs) * sizeof(*n));
+                memcpy(n, z_diag->diagRecs, z_diag->num_diagRecs
+                       * sizeof(*n));
+                z_diag->diagRecs = n;
+            }
+            Z_DiagRec *nr = (Z_DiagRec *) odr_malloc(odr, sizeof(*nr));
+            nr->which = Z_DiagRec_defaultFormat;
+            nr->u.defaultFormat =
+                records->u.nonSurrogateDiagnostic;
+            z_diag->diagRecs[z_diag->num_diagRecs++] = nr;
+        }
+        else if (records->which == Z_Records_multipleNSD)
+        {
+            Z_DiagRecs * dr =records->u.multipleNonSurDiagnostics;
+
+            if (!z_diag)
+            {
+                z_diag = (Z_DiagRecs *) odr_malloc(odr, sizeof(*z_diag));
+                z_diag->num_diagRecs = 0;
+                z_diag->diagRecs = 0;
+            }
+            Z_DiagRec **n = (Z_DiagRec **)
+                odr_malloc(odr,
+                           (dr->num_diagRecs + z_diag->num_diagRecs) * 
+                           sizeof(*n));
+            if (z_diag->num_diagRecs)
+                memcpy(n, z_diag->diagRecs, z_diag->num_diagRecs * sizeof(*n));
+            memcpy(n + z_diag->num_diagRecs,
+                   dr->diagRecs, dr->num_diagRecs * sizeof(*n));
+            z_diag->diagRecs = n;
+            z_diag->num_diagRecs += dr->num_diagRecs;
+        }
+        else
+            no_successful++; // probably piggyback
+    }
+    else
+        no_successful++;  // no records and no diagnostics
+}
+
 void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
 {
     // create search request
     Z_SearchRequest *req = apdu_req->u.searchRequest;
 
-    // save these for later
-    Odr_int smallSetUpperBound = *req->smallSetUpperBound;
-    Odr_int largeSetLowerBound = *req->largeSetLowerBound;
-    Odr_int mediumSetPresentNumber = *req->mediumSetPresentNumber;
-
-    // they are altered now - to disable piggyback
-    *req->smallSetUpperBound = 0;
-    *req->largeSetLowerBound = 1;
-    *req->mediumSetPresentNumber = 0;
-
-    int default_num_db = req->num_databaseNames;
-    char **default_db = req->databaseNames;
-
     std::list<BackendPtr>::const_iterator bit;
     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
     {
         PackagePtr p = (*bit)->m_package;
+        yazpp_1::GDU gdu1(apdu_req);
         mp::odr odr;
+        Z_SearchRequest *req1 = gdu1.get()->u.z3950->u.searchRequest;
+
+        // they are altered now - to disable piggyback
+        *req1->smallSetUpperBound = 0;
+        *req1->largeSetLowerBound = 1;
+        *req1->mediumSetPresentNumber = 0;
 
         if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
-                                                &req->num_databaseNames,
-                                                &req->databaseNames))
+                                               &req1->num_databaseNames,
+                                               &req1->databaseNames))
         {
-            req->num_databaseNames = default_num_db;
-            req->databaseNames = default_db;
+            req1->num_databaseNames = req->num_databaseNames;
+            req1->databaseNames = req->databaseNames;
         }
-        p->request() = apdu_req;
+        p->request() = gdu1;
         p->copy_filter(package);
     }
     multi_move(m_backend_list);
@@ -607,44 +673,8 @@ void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
             Z_APDU *b_apdu = gdu->u.z3950;
             Z_SearchResponse *b_resp = b_apdu->u.searchResponse;
 
-            // see we get any errors (AKA diagnstics)
-            if (b_resp->records)
-            {
-                if (b_resp->records->which == Z_Records_NSD)
-                {
-                    if (!z_diag)
-                    {
-                        z_diag = (Z_DiagRecs *)
-                            odr_malloc(odr, sizeof(*z_diag));
-                        z_diag->num_diagRecs = 0;
-                        z_diag->diagRecs = (Z_DiagRec**)
-                            odr_malloc(odr, sizeof(*z_diag->diagRecs));
-                    }
-                    else
-                    {
-                        Z_DiagRec **n = (Z_DiagRec **)
-                            odr_malloc(odr,
-                                       (1+z_diag->num_diagRecs) * sizeof(*n));
-                        memcpy(n, z_diag->diagRecs, z_diag->num_diagRecs
-                               * sizeof(*n));
-                        z_diag->diagRecs = n;
-                    }
-                    Z_DiagRec *nr = (Z_DiagRec *) odr_malloc(odr, sizeof(*nr));
-                    nr->which = Z_DiagRec_defaultFormat;
-                    nr->u.defaultFormat =
-                        b_resp->records->u.nonSurrogateDiagnostic;
-                    z_diag->diagRecs[z_diag->num_diagRecs++] = nr;
-                }
-                else if (b_resp->records->which == Z_Records_multipleNSD)
-                {
-                    // we may set this multiple times (TOO BAD!)
-                    z_diag = b_resp->records->u.multipleNonSurDiagnostics;
-                }
-                else
-                    no_successful++; // probably piggyback
-            }
-            else
-                no_successful++;  // no records and no diagnostics
+            record_diagnostics(b_resp->records, z_diag, odr, no_successful);
+
             BackendSet backendSet;
             backendSet.m_backend = *bit;
             backendSet.m_count = *b_resp->resultCount;
@@ -656,12 +686,12 @@ void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
     Z_APDU *f_apdu = odr.create_searchResponse(apdu_req, 0, 0);
     Z_SearchResponse *f_resp = f_apdu->u.searchResponse;
 
-    yaz_log(YLOG_LOG, "no_successful=%d is_closed=%s hide_errors=%s",
+    yaz_log(YLOG_DEBUG, "no_successful=%d is_closed=%s hide_errors=%s",
             no_successful,
             close_p ? "true" : "false",
             m_p->m_hide_errors ? "true" : "false");
     *f_resp->resultCount = result_set_size;
-    if (close_p && (no_successful == 0 || !m_p->m_hide_errors))
+    if (close_p && (no_successful == 0 || !m_p->m_hide_unavailable))
     {
         package.session().close();
         package.response() = close_p->response();
@@ -687,9 +717,9 @@ void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
     m_sets[resultSet.m_setname] = resultSet;
 
     Odr_int number;
-    mp::util::piggyback(smallSetUpperBound,
-                        largeSetLowerBound,
-                        mediumSetPresentNumber,
+    mp::util::piggyback(*req->smallSetUpperBound,
+                        *req->largeSetLowerBound,
+                        *req->mediumSetPresentNumber,
                         0, 0,
                         result_set_size,
                         number, 0);
@@ -761,7 +791,7 @@ void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
         std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
         for (jit = jobs.begin(); jit != jobs.end(); jit++)
         {
-            yaz_log(YLOG_LOG, "job pos=%d", jit->m_pos);
+            yaz_log(YLOG_DEBUG, "job pos=%d", jit->m_pos);
         }
     }
 
@@ -812,15 +842,19 @@ void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
     multi_move(present_backend_list);
 
     // look at each response
-    Z_Records *z_records_diag = 0;
+    Z_DiagRecs *z_diag = 0;
+    int no_successful = 0;
+    mp::odr odr;
+    PackagePtr close_p;
 
     std::list<BackendPtr>::const_iterator pbit = present_backend_list.begin();
     for (; pbit != present_backend_list.end(); pbit++)
     {
         PackagePtr p = (*pbit)->m_package;
 
-        if (p->session().is_closed()) // if any backend closes, close frontend
-            package.session().close();
+        // save closing package for at least one target
+        if (p->session().is_closed())
+            close_p = p;
 
         Z_GDU *gdu = p->response().get();
         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
@@ -829,30 +863,34 @@ void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
             Z_APDU *b_apdu = gdu->u.z3950;
             Z_PresentResponse *b_resp = b_apdu->u.presentResponse;
 
-            // see we get any errors (AKA diagnstics)
-            if (b_resp->records)
-            {
-                if (b_resp->records->which != Z_Records_DBOSD)
-                    z_records_diag = b_resp->records;
-                // we may set this multiple times (TOO BAD!)
-            }
-        }
-        else
-        {
-            // if any target does not return present response - return that
-            package.response() = p->response();
-            return;
+            record_diagnostics(b_resp->records, z_diag, odr, no_successful);
         }
     }
 
-    mp::odr odr;
     Z_APDU *f_apdu = odr.create_presentResponse(apdu_req, 0, 0);
     Z_PresentResponse *f_resp = f_apdu->u.presentResponse;
 
-    if (z_records_diag)
+    if (close_p && (no_successful == 0 || !m_p->m_hide_errors))
     {
-        f_resp->records = z_records_diag;
-        *f_resp->presentStatus = Z_PresentStatus_failure;
+        package.session().close();
+        package.response() = close_p->response();
+        return;
+    }
+    if (z_diag && (no_successful == 0 || !m_p->m_hide_errors))
+    {
+        f_resp->records = (Z_Records *)
+            odr_malloc(odr, sizeof(*f_resp->records));
+        if (z_diag->num_diagRecs > 1)
+        {
+            f_resp->records->which = Z_Records_multipleNSD;
+            f_resp->records->u.multipleNonSurDiagnostics = z_diag;
+        }
+        else
+        {
+            f_resp->records->which = Z_Records_NSD;
+            f_resp->records->u.nonSurrogateDiagnostic =
+                z_diag->diagRecs[0]->u.defaultFormat;
+        }
     }
     else if (number < 0 || (size_t) number > jobs.size())
     {
@@ -876,24 +914,27 @@ void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
             odr_malloc(odr, sizeof(Z_NamePlusRecord *) * nprl->num_records);
         int i = 0;
         std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
-        for (jit = jobs.begin(); jit != jobs.end(); jit++, i++)
+        for (jit = jobs.begin(); jit != jobs.end(); jit++)
         {
             PackagePtr p = jit->m_backend->m_package;
 
             Z_GDU *gdu = p->response().get();
             Z_APDU *b_apdu = gdu->u.z3950;
-            Z_PresentResponse *b_resp = b_apdu->u.presentResponse;
-
-            nprl->records[i] = (Z_NamePlusRecord*)
-                odr_malloc(odr, sizeof(Z_NamePlusRecord));
             int inside_pos = jit->m_pos - jit->m_start;
-            if (!b_resp->records || inside_pos >= b_resp->records->
-                u.databaseOrSurDiagnostics->num_records)
-                break;
-           *nprl->records[i] = *b_resp->records->
-                u.databaseOrSurDiagnostics->records[inside_pos];
-            nprl->records[i]->databaseName =
+            Z_Records *records = b_apdu->u.presentResponse->records;
+
+            if (records && records->which == Z_Records_DBOSD
+                && inside_pos < 
+                records->u.databaseOrSurDiagnostics->num_records)
+            {
+                nprl->records[i] = (Z_NamePlusRecord*)
+                    odr_malloc(odr, sizeof(Z_NamePlusRecord));
+                *nprl->records[i] = *records->
+                    u.databaseOrSurDiagnostics->records[inside_pos];
+                nprl->records[i]->databaseName =
                     odr_strdup(odr, jit->m_backend->m_vhost.c_str());
+                i++;
+            }
         }
         nprl->num_records = i; // usually same as jobs.size();
         *f_resp->nextResultSetPosition = start + i;
@@ -902,63 +943,6 @@ void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
     package.response() = f_apdu;
 }
 
-void yf::Multi::Frontend::scan1(mp::Package &package, Z_APDU *apdu_req)
-{
-    if (m_backend_list.size() > 1)
-    {
-        mp::odr odr;
-        Z_APDU *f_apdu =
-            odr.create_scanResponse(
-                apdu_req, YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP, 0);
-        package.response() = f_apdu;
-        return;
-    }
-    Z_ScanRequest *req = apdu_req->u.scanRequest;
-
-    int default_num_db = req->num_databaseNames;
-    char **default_db = req->databaseNames;
-
-    std::list<BackendPtr>::const_iterator bit;
-    for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
-    {
-        PackagePtr p = (*bit)->m_package;
-        mp::odr odr;
-
-        if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
-                                                &req->num_databaseNames,
-                                                &req->databaseNames))
-        {
-            req->num_databaseNames = default_num_db;
-            req->databaseNames = default_db;
-        }
-        p->request() = apdu_req;
-        p->copy_filter(package);
-    }
-    multi_move(m_backend_list);
-
-    for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
-    {
-        PackagePtr p = (*bit)->m_package;
-
-        if (p->session().is_closed()) // if any backend closes, close frontend
-            package.session().close();
-
-        Z_GDU *gdu = p->response().get();
-        if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
-            Z_APDU_scanResponse)
-        {
-            package.response() = p->response();
-            break;
-        }
-        else
-        {
-            // if any target does not return scan response - return that
-            package.response() = p->response();
-            return;
-        }
-    }
-}
-
 bool yf::Multi::ScanTermInfo::operator < (const ScanTermInfo &k) const
 {
     return m_norm_term < k.m_norm_term;
@@ -981,15 +965,10 @@ Z_Entry *yf::Multi::ScanTermInfo::get_entry(ODR odr)
     t->byAttributes = 0;
     t->otherTermInfo = 0;
     t->globalOccurrences = odr_intdup(odr, m_count);
-    t->term = (Z_Term *)
-        odr_malloc(odr, sizeof(*t->term));
+    t->term = (Z_Term *) odr_malloc(odr, sizeof(*t->term));
     t->term->which = Z_Term_general;
-    Odr_oct *o;
-    t->term->u.general = o = (Odr_oct *)odr_malloc(odr, sizeof(Odr_oct));
-
-    o->len = o->size = m_norm_term.size();
-    o->buf = (unsigned char *) odr_malloc(odr, o->len);
-    memcpy(o->buf, m_norm_term.c_str(), o->len);
+    t->term->u.general = odr_create_Odr_oct(odr,
+                                 m_norm_term.c_str(), m_norm_term.size());
     return e;
 }
 
@@ -1017,27 +996,26 @@ void yf::Multi::Frontend::relay_apdu(mp::Package &package, Z_APDU *apdu_req)
 }
 
 
-void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req)
+void yf::Multi::Frontend::scan(mp::Package &package, Z_APDU *apdu_req)
 {
     Z_ScanRequest *req = apdu_req->u.scanRequest;
 
-    int default_num_db = req->num_databaseNames;
-    char **default_db = req->databaseNames;
-
     std::list<BackendPtr>::const_iterator bit;
     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
     {
         PackagePtr p = (*bit)->m_package;
+        yazpp_1::GDU gdu1(apdu_req);
         mp::odr odr;
+        Z_ScanRequest *req1 = gdu1.get()->u.z3950->u.scanRequest;
 
         if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
-                                                &req->num_databaseNames,
-                                                &req->databaseNames))
+                                               &req1->num_databaseNames,
+                                               &req1->databaseNames))
         {
-            req->num_databaseNames = default_num_db;
-            req->databaseNames = default_db;
+            req1->num_databaseNames = req->num_databaseNames;
+            req1->databaseNames = req->databaseNames;
         }
-        p->request() = apdu_req;
+        p->request() = gdu1;
         p->copy_filter(package);
     }
     multi_move(m_backend_list);
@@ -1282,7 +1260,7 @@ void yf::Multi::process(mp::Package &package) const
         }
         else if (apdu->which == Z_APDU_scanRequest)
         {
-            f->scan2(package, apdu);
+            f->scan(package, apdu);
         }
         else if (apdu->which == Z_APDU_close)
         {