X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Ffilter_multi.cpp;h=33de6a4235563ea70456cf9e143d61ba533136d8;hb=4bfec20c393570a81471c644fad6268eed3dba29;hp=fcf3f5b6260d1e6cca21089955fad7f41893122b;hpb=0f91085e7a561979199a348aba002b265b8e4b48;p=metaproxy-moved-to-github.git diff --git a/src/filter_multi.cpp b/src/filter_multi.cpp index fcf3f5b..33de6a4 100644 --- a/src/filter_multi.cpp +++ b/src/filter_multi.cpp @@ -1,25 +1,40 @@ -/* $Id: filter_multi.cpp,v 1.17 2006-05-15 13:22:02 adam Exp $ - Copyright (c) 2005-2006, Index Data. +/* This file is part of Metaproxy. + Copyright (C) 2005-2011 Index Data -%LICENSE% - */ +Metaproxy is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include #include "config.hpp" -#include "filter.hpp" -#include "package.hpp" +#include +#include #include #include #include #include -#include "util.hpp" +#include #include "filter_multi.hpp" #include #include #include +#include #include #include @@ -31,7 +46,10 @@ namespace yf = mp::filter; namespace metaproxy_1 { namespace filter { - + enum multi_merge_type { + round_robin, + serve_order + }; struct Multi::BackendSet { BackendPtr m_backend; int m_count; @@ -47,16 +65,20 @@ namespace metaproxy_1 { Z_Entry *get_entry(ODR odr); }; struct Multi::FrontendSet { - struct PresentJob { + class PresentJob { + public: BackendPtr m_backend; - int m_pos; - int m_inside_pos; + int m_pos; // position for backend (1=first, 2=second,.. + int m_start; // present request start + PresentJob(BackendPtr ptr, int pos) : + m_backend(ptr), m_pos(pos), m_start(0) {}; }; FrontendSet(std::string setname); FrontendSet(); ~FrontendSet(); void round_robin(int pos, int number, std::list &job); + void serve_order(int pos, int number, std::list &job); std::list m_backend_sets; std::string m_setname; @@ -83,29 +105,48 @@ namespace metaproxy_1 { void present(Package &package, Z_APDU *apdu); void scan1(Package &package, Z_APDU *apdu); void scan2(Package &package, Z_APDU *apdu); + void relay_apdu(Package &package, Z_APDU *apdu); Rep *m_p; }; - struct Multi::Map { - Map(std::list hosts, std::string route); - Map(); - std::list m_hosts; + class Multi::Map { + std::string m_target_pattern; std::string m_route; + public: + Map(std::string pattern, std::string route) : + m_target_pattern(pattern), m_route(route) {}; + bool match(const std::string target, std::string *ret) const { + if (yaz_match_glob(m_target_pattern.c_str(), target.c_str())) + { + *ret = m_route; + return true; + } + return false; + }; }; class Multi::Rep { friend class Multi; friend struct Frontend; + Rep(); FrontendPtr get_frontend(Package &package); void release_frontend(Package &package); private: - std::map m_target_route; + std::list m_route_patterns; boost::mutex m_mutex; boost::condition m_cond_session_ready; std::map m_clients; + bool m_hide_unavailable; + multi_merge_type m_merge_type; }; } } +yf::Multi::Rep::Rep() +{ + m_hide_unavailable = false; + m_merge_type = round_robin; +} + bool yf::Multi::BackendSet::operator < (const BackendSet &k) const { return m_count < k.m_count; @@ -121,7 +162,7 @@ yf::Multi::Frontend::~Frontend() { } -yf::Multi::FrontendPtr yf::Multi::Rep::get_frontend(Package &package) +yf::Multi::FrontendPtr yf::Multi::Rep::get_frontend(mp::Package &package) { boost::mutex::scoped_lock lock(m_mutex); @@ -146,7 +187,7 @@ yf::Multi::FrontendPtr yf::Multi::Rep::get_frontend(Package &package) return f; } -void yf::Multi::Rep::release_frontend(Package &package) +void yf::Multi::Rep::release_frontend(mp::Package &package) { boost::mutex::scoped_lock lock(m_mutex); std::map::iterator it; @@ -182,15 +223,6 @@ yf::Multi::FrontendSet::~FrontendSet() { } -yf::Multi::Map::Map(std::list hosts, std::string route) - : m_hosts(hosts), m_route(route) -{ -} - -yf::Multi::Map::Map() -{ -} - yf::Multi::Multi() : m_p(new Multi::Rep) { } @@ -205,7 +237,7 @@ void yf::Multi::Backend::operator() (void) } -void yf::Multi::Frontend::close(Package &package) +void yf::Multi::Frontend::close(mp::Package &package) { std::list::const_iterator bit; for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++) @@ -230,16 +262,37 @@ void yf::Multi::Frontend::multi_move(std::list &blist) g.join_all(); } +void yf::Multi::FrontendSet::serve_order(int start, int number, + std::list &jobs) +{ + int i; + for (i = 0; i < number; i++) + { + std::list::const_iterator bsit; + int voffset = 0; + int offset = start + i - 1; + for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); + bsit++) + { + if (offset >= voffset && offset < voffset + bsit->m_count) + { + PresentJob job(bsit->m_backend, offset - voffset + 1); + jobs.push_back(job); + break; + } + voffset += bsit->m_count; + } + } +} + void yf::Multi::FrontendSet::round_robin(int start, int number, std::list &jobs) { std::list pos; - std::list inside_pos; std::list::const_iterator bsit; for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); bsit++) { pos.push_back(1); - inside_pos.push_back(0); } int p = 1; @@ -278,8 +331,6 @@ void yf::Multi::FrontendSet::round_robin(int start, int number, // skip on each set.. before "present range".. p = p + skip; - std::cout << "\nSKIP min=" << min << " no_left=" << no_left << "\n\n"; - std::list::iterator psit = pos.begin(); for (psit = pos.begin(); psit != pos.end(); psit++) *psit += min; @@ -293,10 +344,9 @@ void yf::Multi::FrontendSet::round_robin(int start, int number, { more = false; std::list::iterator psit = pos.begin(); - std::list::iterator esit = inside_pos.begin(); bsit = m_backend_sets.begin(); - for (; bsit != m_backend_sets.end(); psit++,esit++,bsit++) + for (; bsit != m_backend_sets.end(); psit++,bsit++) { if (fetched >= number) { @@ -307,12 +357,8 @@ void yf::Multi::FrontendSet::round_robin(int start, int number, { if (p >= start) { - PresentJob job; - job.m_backend = bsit->m_backend; - job.m_pos = *psit; - job.m_inside_pos = *esit; + PresentJob job(bsit->m_backend, *psit); jobs.push_back(job); - (*esit)++; fetched++; } (*psit)++; @@ -323,13 +369,13 @@ void yf::Multi::FrontendSet::round_robin(int start, int number, } } -void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu) +void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu) { Z_InitRequest *req = gdu->u.z3950->u.initRequest; std::list targets; - mp::util::get_vhost_otherinfo(&req->otherInfo, false, targets); + mp::util::get_vhost_otherinfo(req->otherInfo, targets); if (targets.size() < 1) { @@ -344,7 +390,14 @@ void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu) Backend *b = new Backend; b->m_vhost = *t_it; - b->m_route = m_p->m_target_route[*t_it]; + std::list::const_iterator it = + m_p->m_route_patterns.begin(); + while (it != m_p->m_route_patterns.end()) { + if (it->match(*t_it, &b->m_route)) + break; + it++; + } + // b->m_route = m_p->m_target_route[*t_it]; // b->m_route unset b->m_package = PackagePtr(new Package(s, package.origin())); @@ -353,7 +406,7 @@ void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu) m_is_multi = true; // create init request - std::list::const_iterator bit; + std::list::iterator bit; for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++) { mp::odr odr; @@ -365,16 +418,22 @@ void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu) mp::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo, odr, vhost_one); - Z_InitRequest *req = init_apdu->u.initRequest; + + Z_InitRequest *breq = init_apdu->u.initRequest; + + breq->idAuthentication = req->idAuthentication; - ODR_MASK_SET(req->options, Z_Options_search); - ODR_MASK_SET(req->options, Z_Options_present); - ODR_MASK_SET(req->options, Z_Options_namedResultSets); - ODR_MASK_SET(req->options, Z_Options_scan); + *breq->preferredMessageSize = *req->preferredMessageSize; + *breq->maximumRecordSize = *req->maximumRecordSize; + + ODR_MASK_SET(breq->options, Z_Options_search); + ODR_MASK_SET(breq->options, Z_Options_present); + ODR_MASK_SET(breq->options, Z_Options_namedResultSets); + ODR_MASK_SET(breq->options, Z_Options_scan); - ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1); - ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2); - ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3); + ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_1); + ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_2); + ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_3); b->m_package->request() = init_apdu; @@ -391,17 +450,28 @@ void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu) ODR_MASK_SET(f_resp->options, Z_Options_search); ODR_MASK_SET(f_resp->options, Z_Options_present); ODR_MASK_SET(f_resp->options, Z_Options_namedResultSets); + ODR_MASK_SET(f_resp->options, Z_Options_scan); ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_1); ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_2); ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_3); - for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++) + int no_failed = 0; + int no_succeeded = 0; + + Odr_int preferredMessageSize = *req->preferredMessageSize; + Odr_int maximumRecordSize = *req->maximumRecordSize; + for (bit = m_backend_list.begin(); bit != m_backend_list.end(); ) { PackagePtr p = (*bit)->m_package; - if (p->session().is_closed()) // if any backend closes, close frontend - package.session().close(); + if (p->session().is_closed()) + { + // failed. Remove from list and increment number of failed + no_failed++; + bit = m_backend_list.erase(bit); + continue; + } Z_GDU *gdu = p->response().get(); if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which == Z_APDU_initResponse) @@ -420,30 +490,52 @@ void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu) for (i = 0; i <= Z_ProtocolVersion_3; i++) if (!ODR_MASK_GET(b_resp->protocolVersion, i)) ODR_MASK_CLEAR(f_resp->protocolVersion, i); - // reject if any of the backends reject - if (!*b_resp->result) - *f_resp->result = 0; + if (*b_resp->result) + { + no_succeeded++; + if (preferredMessageSize > *b_resp->preferredMessageSize) + preferredMessageSize = *b_resp->preferredMessageSize; + if (maximumRecordSize > *b_resp->maximumRecordSize) + maximumRecordSize = *b_resp->maximumRecordSize; + } + else + no_failed++; } else + no_failed++; + bit++; + } + *f_resp->preferredMessageSize = preferredMessageSize; + *f_resp->maximumRecordSize = maximumRecordSize; + + if (m_p->m_hide_unavailable) + { + if (no_succeeded == 0) { - // if any target does not return init return that (close or - // similar ) - package.response() = p->response(); - return; + *f_resp->result = 0; + package.session().close(); + } + } + else + { + if (no_failed) + { + *f_resp->result = 0; + package.session().close(); } } package.response() = f_apdu; } -void yf::Multi::Frontend::search(Package &package, Z_APDU *apdu_req) +void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req) { // create search request Z_SearchRequest *req = apdu_req->u.searchRequest; // save these for later - int smallSetUpperBound = *req->smallSetUpperBound; - int largeSetLowerBound = *req->largeSetLowerBound; - int mediumSetPresentNumber = *req->mediumSetPresentNumber; + Odr_int smallSetUpperBound = *req->smallSetUpperBound; + Odr_int largeSetLowerBound = *req->largeSetLowerBound; + Odr_int mediumSetPresentNumber = *req->mediumSetPresentNumber; // they are altered now - to disable piggyback *req->smallSetUpperBound = 0; @@ -474,7 +566,7 @@ void yf::Multi::Frontend::search(Package &package, Z_APDU *apdu_req) // look at each response FrontendSet resultSet(std::string(req->resultSetName)); - int result_set_size = 0; + Odr_int result_set_size = 0; Z_Records *z_records_diag = 0; // no diagnostics (yet) for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++) { @@ -527,12 +619,13 @@ void yf::Multi::Frontend::search(Package &package, Z_APDU *apdu_req) // assume OK m_sets[resultSet.m_setname] = resultSet; - int number; + Odr_int number; mp::util::piggyback(smallSetUpperBound, - largeSetLowerBound, - mediumSetPresentNumber, - result_set_size, - number); + largeSetLowerBound, + mediumSetPresentNumber, + 0, 0, + result_set_size, + number, 0); Package pp(package.session(), package.origin()); if (number > 0) { @@ -569,7 +662,7 @@ void yf::Multi::Frontend::search(Package &package, Z_APDU *apdu_req) package.response() = f_apdu; // in this scope because of p } -void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req) +void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req) { // create present request Z_PresentRequest *req = apdu_req->u.presentRequest; @@ -590,7 +683,20 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req) std::list jobs; int start = *req->resultSetStartPoint; int number = *req->numberOfRecordsRequested; - it->second.round_robin(start, number, jobs); + + if (m_p->m_merge_type == round_robin) + it->second.round_robin(start, number, jobs); + else if (m_p->m_merge_type == serve_order) + it->second.serve_order(start, number, jobs); + + if (0) + { + std::list::const_iterator jit; + for (jit = jobs.begin(); jit != jobs.end(); jit++) + { + yaz_log(YLOG_LOG, "job pos=%d", jit->m_pos); + } + } std::list present_backend_list; @@ -598,22 +704,33 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req) bsit = it->second.m_backend_sets.begin(); for (; bsit != it->second.m_backend_sets.end(); bsit++) { - std::list::const_iterator jit; int start = -1; int end = -1; - - for (jit = jobs.begin(); jit != jobs.end(); jit++) { - if (jit->m_backend == bsit->m_backend) + std::list::const_iterator jit; + for (jit = jobs.begin(); jit != jobs.end(); jit++) { - if (start == -1 || jit->m_pos < start) - start = jit->m_pos; - if (end == -1 || jit->m_pos > end) - end = jit->m_pos; + if (jit->m_backend == bsit->m_backend) + { + if (start == -1 || jit->m_pos < start) + start = jit->m_pos; + if (end == -1 || jit->m_pos > end) + end = jit->m_pos; + } } } if (start != -1) { + std::list::iterator jit; + for (jit = jobs.begin(); jit != jobs.end(); jit++) + { + if (jit->m_backend == bsit->m_backend) + { + if (jit->m_pos >= start && jit->m_pos <= end) + jit->m_start = start; + } + } + PackagePtr p = bsit->m_backend->m_package; *req->resultSetStartPoint = start; @@ -670,6 +787,14 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req) f_resp->records = z_records_diag; *f_resp->presentStatus = Z_PresentStatus_failure; } + else if (number < 0 || (size_t) number > jobs.size()) + { + f_apdu = + odr.create_presentResponse( + apdu_req, + YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, + 0); + } else { f_resp->records = (Z_Records *) odr_malloc(odr, sizeof(Z_Records)); @@ -684,7 +809,7 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req) odr_malloc(odr, sizeof(Z_NamePlusRecord *) * nprl->num_records); int i = 0; std::list::const_iterator jit; - for (jit = jobs.begin(); jit != jobs.end(); jit++) + for (jit = jobs.begin(); jit != jobs.end(); jit++, i++) { PackagePtr p = jit->m_backend->m_package; @@ -692,17 +817,25 @@ void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req) Z_APDU *b_apdu = gdu->u.z3950; Z_PresentResponse *b_resp = b_apdu->u.presentResponse; - nprl->records[i++] = - b_resp->records->u.databaseOrSurDiagnostics-> - records[jit->m_inside_pos]; + nprl->records[i] = (Z_NamePlusRecord*) + odr_malloc(odr, sizeof(Z_NamePlusRecord)); + int inside_pos = jit->m_pos - jit->m_start; + if (inside_pos >= b_resp->records-> + u.databaseOrSurDiagnostics->num_records) + break; + *nprl->records[i] = *b_resp->records-> + u.databaseOrSurDiagnostics->records[inside_pos]; + nprl->records[i]->databaseName = + odr_strdup(odr, jit->m_backend->m_vhost.c_str()); } + nprl->num_records = i; // usually same as jobs.size(); *f_resp->nextResultSetPosition = start + i; *f_resp->numberOfRecordsReturned = i; } package.response() = f_apdu; } -void yf::Multi::Frontend::scan1(Package &package, Z_APDU *apdu_req) +void yf::Multi::Frontend::scan1(mp::Package &package, Z_APDU *apdu_req) { if (m_backend_list.size() > 1) { @@ -793,7 +926,31 @@ Z_Entry *yf::Multi::ScanTermInfo::get_entry(ODR odr) return e; } -void yf::Multi::Frontend::scan2(Package &package, Z_APDU *apdu_req) +void yf::Multi::Frontend::relay_apdu(mp::Package &package, Z_APDU *apdu_req) +{ + std::list::const_iterator bit; + for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++) + { + PackagePtr p = (*bit)->m_package; + mp::odr odr; + + p->request() = apdu_req; + p->copy_filter(package); + } + multi_move(m_backend_list); + for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++) + { + PackagePtr p = (*bit)->m_package; + + if (p->session().is_closed()) // if any backend closes, close frontend + package.session().close(); + + package.response() = p->response(); + } +} + + +void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req) { Z_ScanRequest *req = apdu_req->u.scanRequest; @@ -872,7 +1029,7 @@ void yf::Multi::Frontend::scan2(Package &package, Z_APDU *apdu_req) { ScanTermInfo my; - int *occur = ent->u.termInfo->globalOccurrences; + Odr_int *occur = ent->u.termInfo->globalOccurrences; my.m_count = occur ? *occur : 0; if (ent->u.termInfo->term->which == Z_Term_general) @@ -913,7 +1070,7 @@ void yf::Multi::Frontend::scan2(Package &package, Z_APDU *apdu_req) { ScanTermInfo my; - int *occur = ent->u.termInfo->globalOccurrences; + Odr_int *occur = ent->u.termInfo->globalOccurrences; my.m_count = occur ? *occur : 0; if (ent->u.termInfo->term->which == Z_Term_general) @@ -952,7 +1109,7 @@ void yf::Multi::Frontend::scan2(Package &package, Z_APDU *apdu_req) } } - if (true) + if (false) { std::cout << "BEFORE\n"; ScanTermInfoList::iterator it = entries_before.begin(); @@ -1021,7 +1178,7 @@ void yf::Multi::Frontend::scan2(Package &package, Z_APDU *apdu_req) } -void yf::Multi::process(Package &package) const +void yf::Multi::process(mp::Package &package) const { FrontendPtr f = m_p->get_frontend(package); @@ -1060,6 +1217,10 @@ void yf::Multi::process(Package &package) const { f->scan2(package, apdu); } + else if (apdu->which == Z_APDU_close) + { + f->relay_apdu(package, apdu); + } else { mp::odr odr; @@ -1074,7 +1235,8 @@ void yf::Multi::process(Package &package) const m_p->release_frontend(package); } -void mp::filter::Multi::configure(const xmlNode * ptr) +void mp::filter::Multi::configure(const xmlNode * ptr, bool test_only, + const char *path) { for (ptr = ptr->children; ptr; ptr = ptr->next) { @@ -1084,15 +1246,30 @@ void mp::filter::Multi::configure(const xmlNode * ptr) { std::string route = mp::xml::get_route(ptr); std::string target = mp::xml::get_text(ptr); - std::cout << "route=" << route << " target=" << target << "\n"; - m_p->m_target_route[target] = route; + m_p->m_route_patterns.push_back(Multi::Map(target, route)); + } + else if (!strcmp((const char *) ptr->name, "hideunavailable")) + { + m_p->m_hide_unavailable = true; + } + else if (!strcmp((const char *) ptr->name, "mergetype")) + { + std::string mergetype = mp::xml::get_text(ptr); + if (mergetype == "roundrobin") + m_p->m_merge_type = round_robin; + else if (mergetype == "serveorder") + m_p->m_merge_type = serve_order; + else + throw mp::filter::FilterException + ("Bad mergetype " + mergetype + " in multi filter"); + } else { throw mp::filter::FilterException ("Bad element " + std::string((const char *) ptr->name) - + " in virt_db filter"); + + " in multi filter"); } } } @@ -1114,8 +1291,9 @@ extern "C" { /* * Local variables: * c-basic-offset: 4 + * c-file-style: "Stroustrup" * indent-tabs-mode: nil - * c-file-style: "stroustrup" * End: * vim: shiftwidth=4 tabstop=8 expandtab */ +