X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Ffilter_multi.cpp;h=b0bf21f9f37b48427f964e38f8cbf52d79f32974;hb=d0123337d70b0fb97b578cc57467bb94980f1014;hp=a339959e920023138dff6146983cdc24f4f0cdc2;hpb=992092493a6c094605d83cd6c5823614f384a779;p=metaproxy-moved-to-github.git diff --git a/src/filter_multi.cpp b/src/filter_multi.cpp index a339959..b0bf21f 100644 --- a/src/filter_multi.cpp +++ b/src/filter_multi.cpp @@ -1,8 +1,22 @@ -/* $Id: filter_multi.cpp,v 1.9 2006-01-18 14:10:47 adam Exp $ - Copyright (c) 2005, Index Data. +/* This file is part of Metaproxy. + Copyright (C) 2005-2009 Index Data -%LICENSE% - */ +Metaproxy is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include #include "config.hpp" @@ -26,11 +40,15 @@ #include #include -namespace yf = yp2::filter; +namespace mp = metaproxy_1; +namespace yf = mp::filter; -namespace yp2 { +namespace metaproxy_1 { namespace filter { - + enum multi_merge_type { + round_robin, + serve_order + }; struct Multi::BackendSet { BackendPtr m_backend; int m_count; @@ -46,16 +64,20 @@ namespace yp2 { Z_Entry *get_entry(ODR odr); }; struct Multi::FrontendSet { - struct PresentJob { + class PresentJob { + public: BackendPtr m_backend; - int m_pos; - int m_inside_pos; + int m_pos; // position for backend (1=first, 2=second,.. + int m_start; // present request start + PresentJob(BackendPtr ptr, int pos) : + m_backend(ptr), m_pos(pos), m_start(0) {}; }; FrontendSet(std::string setname); FrontendSet(); ~FrontendSet(); void round_robin(int pos, int number, std::list &job); + void serve_order(int pos, int number, std::list &job); std::list m_backend_sets; std::string m_setname; @@ -92,22 +114,27 @@ namespace yp2 { }; class Multi::Rep { friend class Multi; - friend class Frontend; + friend struct Frontend; + Rep(); FrontendPtr get_frontend(Package &package); void release_frontend(Package &package); private: - boost::mutex m_sessions_mutex; - std::mapm_maps; std::map m_target_route; boost::mutex m_mutex; boost::condition m_cond_session_ready; - std::map m_clients; + std::map m_clients; + bool m_hide_unavailable; + multi_merge_type m_merge_type; }; } } -using namespace yp2; +yf::Multi::Rep::Rep() +{ + m_hide_unavailable = false; + m_merge_type = round_robin; +} bool yf::Multi::BackendSet::operator < (const BackendSet &k) const { @@ -124,11 +151,11 @@ yf::Multi::Frontend::~Frontend() { } -yf::Multi::FrontendPtr yf::Multi::Rep::get_frontend(Package &package) +yf::Multi::FrontendPtr yf::Multi::Rep::get_frontend(mp::Package &package) { boost::mutex::scoped_lock lock(m_mutex); - std::map::iterator it; + std::map::iterator it; while(true) { @@ -149,10 +176,10 @@ yf::Multi::FrontendPtr yf::Multi::Rep::get_frontend(Package &package) return f; } -void yf::Multi::Rep::release_frontend(Package &package) +void yf::Multi::Rep::release_frontend(mp::Package &package) { boost::mutex::scoped_lock lock(m_mutex); - std::map::iterator it; + std::map::iterator it; it = m_clients.find(package.session()); if (it != m_clients.end()) @@ -202,20 +229,13 @@ yf::Multi::~Multi() { } -void yf::Multi::add_map_host2hosts(std::string host, - std::list hosts, - std::string route) -{ - m_p->m_maps[host] = Multi::Map(hosts, route); -} - void yf::Multi::Backend::operator() (void) { m_package->move(m_route); } -void yf::Multi::Frontend::close(Package &package) +void yf::Multi::Frontend::close(mp::Package &package) { std::list::const_iterator bit; for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++) @@ -240,66 +260,120 @@ void yf::Multi::Frontend::multi_move(std::list &blist) g.join_all(); } +void yf::Multi::FrontendSet::serve_order(int start, int number, + std::list &jobs) +{ + int i; + for (i = 0; i < number; i++) + { + std::list::const_iterator bsit; + int voffset = 0; + int offset = start + i - 1; + for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); + bsit++) + { + if (offset >= voffset && offset < voffset + bsit->m_count) + { + PresentJob job(bsit->m_backend, offset - voffset + 1); + jobs.push_back(job); + break; + } + voffset += bsit->m_count; + } + } +} void yf::Multi::FrontendSet::round_robin(int start, int number, std::list &jobs) { - int fetched = 0; - int p = 1; - bool eof = true; - std::list pos; - std::list inside_pos; std::list::const_iterator bsit; for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); bsit++) { pos.push_back(1); - inside_pos.push_back(0); } - std::list::iterator psit = pos.begin(); - std::list::iterator esit = inside_pos.begin(); - bsit = m_backend_sets.begin(); - while (fetched < number) + int p = 1; +#if 1 + // optimization step! + int omin = 0; + while(true) { - if (bsit == m_backend_sets.end()) + int min = 0; + int no_left = 0; + // find min count for each set which is > omin + for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); bsit++) { - psit = pos.begin(); - esit = inside_pos.begin(); - bsit = m_backend_sets.begin(); - if (eof) - break; - eof = true; + if (bsit->m_count > omin) + { + if (no_left == 0 || bsit->m_count < min) + min = bsit->m_count; + no_left++; + } } - if (*psit <= bsit->m_count) + if (no_left == 0) // if nothing greater than omin, bail out. + break; + int skip = no_left * min; + if (p + skip > start) // step gets us "into" present range? + { + // Yes. skip until start.. Rounding off is deliberate! + min = (start-p) / no_left; + p += no_left * min; + + // update positions in each set.. + std::list::iterator psit = pos.begin(); + for (psit = pos.begin(); psit != pos.end(); psit++) + *psit += min; + break; + } + // skip on each set.. before "present range".. + p = p + skip; + + std::list::iterator psit = pos.begin(); + for (psit = pos.begin(); psit != pos.end(); psit++) + *psit += min; + + omin = min; // update so we consider next class (with higher count) + } +#endif + int fetched = 0; + bool more = true; + while (more) + { + more = false; + std::list::iterator psit = pos.begin(); + bsit = m_backend_sets.begin(); + + for (; bsit != m_backend_sets.end(); psit++,bsit++) { - if (p >= start) + if (fetched >= number) { - PresentJob job; - job.m_backend = bsit->m_backend; - job.m_pos = *psit; - job.m_inside_pos = *esit; - jobs.push_back(job); - (*esit)++; - fetched++; + more = false; + break; + } + if (*psit <= bsit->m_count) + { + if (p >= start) + { + PresentJob job(bsit->m_backend, *psit); + jobs.push_back(job); + fetched++; + } + (*psit)++; + p++; + more = true; } - (*psit)++; - p++; - eof = false; } - psit++; - esit++; - bsit++; } } -void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu) +void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu) { Z_InitRequest *req = gdu->u.z3950->u.initRequest; std::list targets; - yp2::util::get_vhost_otherinfo(&req->otherInfo, false, targets); + mp::util::get_vhost_otherinfo(req->otherInfo, targets); if (targets.size() < 1) { @@ -323,16 +397,16 @@ void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu) m_is_multi = true; // create init request - std::list::const_iterator bit; + std::list::iterator bit; for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++) { - yp2::odr odr; + mp::odr odr; BackendPtr b = *bit; Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest); std::listvhost_one; vhost_one.push_back(b->m_vhost); - yp2::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo, + mp::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo, odr, vhost_one); Z_InitRequest *req = init_apdu->u.initRequest; @@ -353,7 +427,7 @@ void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu) multi_move(m_backend_list); // create the frontend init response based on each backend init response - yp2::odr odr; + mp::odr odr; Z_APDU *f_apdu = odr.create_initResponse(gdu->u.z3950, 0, 0); Z_InitResponse *f_resp = f_apdu->u.initResponse; @@ -366,12 +440,21 @@ void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu) ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_2); ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_3); - for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++) + int no_failed = 0; + int no_succeeded = 0; + for (bit = m_backend_list.begin(); bit != m_backend_list.end(); ) { PackagePtr p = (*bit)->m_package; - if (p->session().is_closed()) // if any backend closes, close frontend - package.session().close(); + if (p->session().is_closed()) + { + // failed. Remove from list and increment number of failed + no_failed++; + bit = m_backend_list.erase(bit); + continue; + } + no_succeeded++; + Z_GDU *gdu = p->response().get(); if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which == Z_APDU_initResponse) @@ -401,11 +484,22 @@ void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu) package.response() = p->response(); return; } + bit++; + } + if (m_p->m_hide_unavailable) + { + if (no_succeeded == 0) + package.session().close(); + } + else + { + if (no_failed) + package.session().close(); } package.response() = f_apdu; } -void yf::Multi::Frontend::search(Package &package, Z_APDU *apdu_req) +void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req) { // create search request Z_SearchRequest *req = apdu_req->u.searchRequest; @@ -427,9 +521,9 @@ void yf::Multi::Frontend::search(Package &package, Z_APDU *apdu_req) for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++) { PackagePtr p = (*bit)->m_package; - yp2::odr odr; + mp::odr odr; - if (!yp2::util::set_databases_from_zurl(odr, (*bit)->m_vhost, + if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost, &req->num_databaseNames, &req->databaseNames)) { @@ -482,7 +576,7 @@ void yf::Multi::Frontend::search(Package &package, Z_APDU *apdu_req) } } - yp2::odr odr; + mp::odr odr; Z_APDU *f_apdu = odr.create_searchResponse(apdu_req, 0, 0); Z_SearchResponse *f_resp = f_apdu->u.searchResponse; @@ -498,7 +592,7 @@ void yf::Multi::Frontend::search(Package &package, Z_APDU *apdu_req) m_sets[resultSet.m_setname] = resultSet; int number; - yp2::util::piggyback(smallSetUpperBound, + mp::util::piggyback(smallSetUpperBound, largeSetLowerBound, mediumSetPresentNumber, result_set_size, @@ -539,7 +633,7 @@ void yf::Multi::Frontend::search(Package &package, Z_APDU *apdu_req) package.response() = f_apdu; // in this scope because of p } -void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req) +void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req) { // create present request Z_PresentRequest *req = apdu_req->u.presentRequest; @@ -548,7 +642,7 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req) it = m_sets.find(std::string(req->resultSetId)); if (it == m_sets.end()) { - yp2::odr odr; + mp::odr odr; Z_APDU *apdu = odr.create_presentResponse( apdu_req, @@ -560,7 +654,20 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req) std::list jobs; int start = *req->resultSetStartPoint; int number = *req->numberOfRecordsRequested; - it->second.round_robin(start, number, jobs); + + if (m_p->m_merge_type == round_robin) + it->second.round_robin(start, number, jobs); + else if (m_p->m_merge_type == serve_order) + it->second.serve_order(start, number, jobs); + + if (0) + { + std::list::const_iterator jit; + for (jit = jobs.begin(); jit != jobs.end(); jit++) + { + yaz_log(YLOG_LOG, "job pos=%d", jit->m_pos); + } + } std::list present_backend_list; @@ -568,22 +675,33 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req) bsit = it->second.m_backend_sets.begin(); for (; bsit != it->second.m_backend_sets.end(); bsit++) { - std::list::const_iterator jit; int start = -1; int end = -1; - - for (jit = jobs.begin(); jit != jobs.end(); jit++) { - if (jit->m_backend == bsit->m_backend) + std::list::const_iterator jit; + for (jit = jobs.begin(); jit != jobs.end(); jit++) { - if (start == -1 || jit->m_pos < start) - start = jit->m_pos; - if (end == -1 || jit->m_pos > end) - end = jit->m_pos; + if (jit->m_backend == bsit->m_backend) + { + if (start == -1 || jit->m_pos < start) + start = jit->m_pos; + if (end == -1 || jit->m_pos > end) + end = jit->m_pos; + } } } if (start != -1) { + std::list::iterator jit; + for (jit = jobs.begin(); jit != jobs.end(); jit++) + { + if (jit->m_backend == bsit->m_backend) + { + if (jit->m_pos >= start && jit->m_pos <= end) + jit->m_start = start; + } + } + PackagePtr p = bsit->m_backend->m_package; *req->resultSetStartPoint = start; @@ -631,7 +749,7 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req) } } - yp2::odr odr; + mp::odr odr; Z_APDU *f_apdu = odr.create_presentResponse(apdu_req, 0, 0); Z_PresentResponse *f_resp = f_apdu->u.presentResponse; @@ -654,7 +772,7 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req) odr_malloc(odr, sizeof(Z_NamePlusRecord *) * nprl->num_records); int i = 0; std::list::const_iterator jit; - for (jit = jobs.begin(); jit != jobs.end(); jit++) + for (jit = jobs.begin(); jit != jobs.end(); jit++, i++) { PackagePtr p = jit->m_backend->m_package; @@ -662,21 +780,29 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req) Z_APDU *b_apdu = gdu->u.z3950; Z_PresentResponse *b_resp = b_apdu->u.presentResponse; - nprl->records[i++] = - b_resp->records->u.databaseOrSurDiagnostics-> - records[jit->m_inside_pos]; + nprl->records[i] = (Z_NamePlusRecord*) + odr_malloc(odr, sizeof(Z_NamePlusRecord)); + int inside_pos = jit->m_pos - jit->m_start; + if (inside_pos >= b_resp->records-> + u.databaseOrSurDiagnostics->num_records) + break; + *nprl->records[i] = *b_resp->records-> + u.databaseOrSurDiagnostics->records[inside_pos]; + nprl->records[i]->databaseName = + odr_strdup(odr, jit->m_backend->m_vhost.c_str()); } + nprl->num_records = i; // usually same as jobs.size(); *f_resp->nextResultSetPosition = start + i; *f_resp->numberOfRecordsReturned = i; } package.response() = f_apdu; } -void yf::Multi::Frontend::scan1(Package &package, Z_APDU *apdu_req) +void yf::Multi::Frontend::scan1(mp::Package &package, Z_APDU *apdu_req) { if (m_backend_list.size() > 1) { - yp2::odr odr; + mp::odr odr; Z_APDU *f_apdu = odr.create_scanResponse( apdu_req, YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP, 0); @@ -692,9 +818,9 @@ void yf::Multi::Frontend::scan1(Package &package, Z_APDU *apdu_req) for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++) { PackagePtr p = (*bit)->m_package; - yp2::odr odr; + mp::odr odr; - if (!yp2::util::set_databases_from_zurl(odr, (*bit)->m_vhost, + if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost, &req->num_databaseNames, &req->databaseNames)) { @@ -763,10 +889,9 @@ Z_Entry *yf::Multi::ScanTermInfo::get_entry(ODR odr) return e; } -void yf::Multi::Frontend::scan2(Package &package, Z_APDU *apdu_req) +void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req) { Z_ScanRequest *req = apdu_req->u.scanRequest; - int no_targets = 0; int default_num_db = req->num_databaseNames; char **default_db = req->databaseNames; @@ -775,9 +900,9 @@ void yf::Multi::Frontend::scan2(Package &package, Z_APDU *apdu_req) for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++) { PackagePtr p = (*bit)->m_package; - yp2::odr odr; + mp::odr odr; - if (!yp2::util::set_databases_from_zurl(odr, (*bit)->m_vhost, + if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost, &req->num_databaseNames, &req->databaseNames)) { @@ -786,7 +911,6 @@ void yf::Multi::Frontend::scan2(Package &package, Z_APDU *apdu_req) } p->request() = apdu_req; p->copy_filter(package); - no_targets++; } multi_move(m_backend_list); @@ -807,6 +931,23 @@ void yf::Multi::Frontend::scan2(Package &package, Z_APDU *apdu_req) Z_APDU_scanResponse) { Z_ScanResponse *res = gdu->u.z3950->u.scanResponse; + + if (res->entries && res->entries->nonsurrogateDiagnostics) + { + // failure + mp::odr odr; + Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 1, 0); + Z_ScanResponse *f_res = f_apdu->u.scanResponse; + + f_res->entries->nonsurrogateDiagnostics = + res->entries->nonsurrogateDiagnostics; + f_res->entries->num_nonsurrogateDiagnostics = + res->entries->num_nonsurrogateDiagnostics; + + package.response() = f_apdu; + return; + } + if (res->entries && res->entries->entries) { Z_Entry **entries = res->entries->entries; @@ -856,7 +997,11 @@ void yf::Multi::Frontend::scan2(Package &package, Z_APDU *apdu_req) } } // after - for (i = position-1; iu.scanResponse; int number_returned = *req->numberOfTermsRequested; int position_returned = *req->preferredPositionInResponse; - resp->positionOfTerm = odr_intdup(odr, position_returned); - resp->numberOfEntriesReturned = odr_intdup(odr, number_returned); - resp->entries->num_entries = number_returned; resp->entries->entries = (Z_Entry**) odr_malloc(odr, sizeof(Z_Entry*) * number_returned); int i; - + + int lbefore = entries_before.size(); + if (lbefore < position_returned-1) + position_returned = lbefore+1; + ScanTermInfoList::iterator it = entries_before.begin(); - for (i = 0; ientries->entries[i] = it->get_entry(odr); + resp->entries->entries[position_returned-2-i] = it->get_entry(odr); } + it = entries_after.begin(); - for (i = position_returned-1; ientries->entries[i] = it->get_entry(odr); } + + number_returned = i; + + resp->positionOfTerm = odr_intdup(odr, position_returned); + resp->numberOfEntriesReturned = odr_intdup(odr, number_returned); + resp->entries->num_entries = number_returned; + package.response() = f_apdu; } } -void yf::Multi::process(Package &package) const +void yf::Multi::process(mp::Package &package) const { FrontendPtr f = m_p->get_frontend(package); @@ -977,7 +1135,7 @@ void yf::Multi::process(Package &package) const Z_APDU *apdu = gdu->u.z3950; if (apdu->which == Z_APDU_initRequest) { - yp2::odr odr; + mp::odr odr; package.response() = odr.create_close( apdu, @@ -1000,7 +1158,7 @@ void yf::Multi::process(Package &package) const } else { - yp2::odr odr; + mp::odr odr; package.response() = odr.create_close( apdu, Z_Close_protocolError, @@ -1012,7 +1170,7 @@ void yf::Multi::process(Package &package) const m_p->release_frontend(package); } -void yp2::filter::Multi::configure(const xmlNode * ptr) +void mp::filter::Multi::configure(const xmlNode * ptr, bool test_only) { for (ptr = ptr->children; ptr; ptr = ptr->next) { @@ -1020,58 +1178,43 @@ void yp2::filter::Multi::configure(const xmlNode * ptr) continue; if (!strcmp((const char *) ptr->name, "target")) { - std::string route = yp2::xml::get_route(ptr); - std::string target = yp2::xml::get_text(ptr); - std::cout << "route=" << route << " target=" << target << "\n"; + std::string route = mp::xml::get_route(ptr); + std::string target = mp::xml::get_text(ptr); m_p->m_target_route[target] = route; } - else if (!strcmp((const char *) ptr->name, "virtual")) + else if (!strcmp((const char *) ptr->name, "hideunavailable")) { - std::list targets; - std::string vhost; - xmlNode *v_node = ptr->children; - for (; v_node; v_node = v_node->next) - { - if (v_node->type != XML_ELEMENT_NODE) - continue; - - if (yp2::xml::is_element_yp2(v_node, "vhost")) - vhost = yp2::xml::get_text(v_node); - else if (yp2::xml::is_element_yp2(v_node, "target")) - targets.push_back(yp2::xml::get_text(v_node)); - else - throw yp2::filter::FilterException - ("Bad element " - + std::string((const char *) v_node->name) - + " in virtual section" - ); - } - std::string route = yp2::xml::get_route(ptr); - add_map_host2hosts(vhost, targets, route); - std::list::const_iterator it; - for (it = targets.begin(); it != targets.end(); it++) - { - std::cout << "Add " << vhost << "->" << *it - << "," << route << "\n"; - } + m_p->m_hide_unavailable = true; + } + else if (!strcmp((const char *) ptr->name, "mergetype")) + { + std::string mergetype = mp::xml::get_text(ptr); + if (mergetype == "roundrobin") + m_p->m_merge_type = round_robin; + else if (mergetype == "serveorder") + m_p->m_merge_type = serve_order; + else + throw mp::filter::FilterException + ("Bad mergetype " + mergetype + " in multi filter"); + } else { - throw yp2::filter::FilterException + throw mp::filter::FilterException ("Bad element " + std::string((const char *) ptr->name) - + " in virt_db filter"); + + " in multi filter"); } } } -static yp2::filter::Base* filter_creator() +static mp::filter::Base* filter_creator() { - return new yp2::filter::Multi; + return new mp::filter::Multi; } extern "C" { - struct yp2_filter_struct yp2_filter_multi = { + struct metaproxy_1_filter_struct metaproxy_1_filter_multi = { 0, "multi", filter_creator @@ -1082,8 +1225,9 @@ extern "C" { /* * Local variables: * c-basic-offset: 4 + * c-file-style: "Stroustrup" * indent-tabs-mode: nil - * c-file-style: "stroustrup" * End: * vim: shiftwidth=4 tabstop=8 expandtab */ +