From 63dda9fef8b647d491dd2baef8ec4f4f88ee1b11 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Mon, 16 Jan 2006 15:51:56 +0000 Subject: [PATCH] Filter virt_db allows multiple databases.. Each of these is mapped to a target. Fiter multi is able to multi search these. --- etc/config-zurg.xml | 57 ++++++++++++++ src/ex_filter_frontend_net.cpp | 8 +- src/filter_multi.cpp | 58 ++++++++------ src/filter_virt_db.cpp | 167 +++++++++++++++++++++++----------------- src/filter_virt_db.hpp | 10 ++- src/util.cpp | 42 +++++++++- src/util.hpp | 4 +- 7 files changed, 244 insertions(+), 102 deletions(-) create mode 100644 etc/config-zurg.xml diff --git a/etc/config-zurg.xml b/etc/config-zurg.xml new file mode 100644 index 0000000..1c03451 --- /dev/null +++ b/etc/config-zurg.xml @@ -0,0 +1,57 @@ + + + + + + + + + + 10 + @:9000 + + + F + + + + + bdsbkm21 + zurg:9901/bdsbkm21 + + + blconm21 + zurg:9902/blconm21 + + + lcbkm21 + zurg:9903/lcbkm21 + + + radmarc + zurg:9904/radmarc + + + books + zurg:9901/bdsbkm21 + zurg:9902/blconm21 + zurg:9903/lcbkm21 + zurg:9904/radmarc + + + + M + + + + + + B + + + 30 + + + + + diff --git a/src/ex_filter_frontend_net.cpp b/src/ex_filter_frontend_net.cpp index 18df43d..15e6b55 100644 --- a/src/ex_filter_frontend_net.cpp +++ b/src/ex_filter_frontend_net.cpp @@ -1,4 +1,4 @@ -/* $Id: ex_filter_frontend_net.cpp,v 1.24 2006-01-13 15:09:35 adam Exp $ +/* $Id: ex_filter_frontend_net.cpp,v 1.25 2006-01-16 15:51:56 adam Exp $ Copyright (c) 2005, Index Data. %LICENSE% @@ -104,11 +104,11 @@ int main(int argc, char **argv) // put Virt db filter in router yp2::filter::Virt_db filter_virt_db; - filter_virt_db.add_map_db2vhost("gils", "indexdata.dk/gils", + filter_virt_db.add_map_db2target("gils", "indexdata.dk/gils", ""); - filter_virt_db.add_map_db2vhost("Default", "localhost:9999/Default", + filter_virt_db.add_map_db2target("Default", "localhost:9999/Default", ""); - filter_virt_db.add_map_db2vhost("2", "localhost:9999/Slow", ""); + filter_virt_db.add_map_db2target("2", "localhost:9999/Slow", ""); router.append(filter_virt_db); yp2::filter::SessionShared filter_session_shared; diff --git a/src/filter_multi.cpp b/src/filter_multi.cpp index 71343c6..468ccdf 100644 --- a/src/filter_multi.cpp +++ b/src/filter_multi.cpp @@ -1,4 +1,4 @@ -/* $Id: filter_multi.cpp,v 1.2 2006-01-16 01:10:19 adam Exp $ +/* $Id: filter_multi.cpp,v 1.3 2006-01-16 15:51:56 adam Exp $ Copyright (c) 2005, Index Data. %LICENSE% @@ -284,34 +284,39 @@ void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu) { Z_InitRequest *req = gdu->u.z3950->u.initRequest; - // empty or non-existang vhost is the same.. - const char *vhost_cstr = - yaz_oi_get_string_oidval(&req->otherInfo, VAL_PROXY, 1, 0); - std::string vhost; - if (vhost_cstr) - vhost = std::string(vhost_cstr); + std::list targets; - std::map::const_iterator it; - it = m_p->m_maps.find(std::string(vhost)); - if (it == m_p->m_maps.end()) + int no_targets = 0; + while (true) + { + const char *vhost_cstr = + yaz_oi_get_string_oidval(&req->otherInfo, VAL_PROXY, no_targets+1, + 0); + if (!vhost_cstr) + break; + no_targets++; + if (no_targets > 1000) + return; + targets.push_back(vhost_cstr); + } + if (no_targets < 2) { - // might return diagnostics if no match package.move(); return; } - std::list::const_iterator hit = it->second.m_hosts.begin(); - for (; hit != it->second.m_hosts.end(); hit++) + + std::list::const_iterator t_it = targets.begin(); + for (; t_it != targets.end(); t_it++) { Session s; Backend *b = new Backend; - b->m_vhost = *hit; - b->m_route = it->second.m_route; + b->m_vhost = *t_it; + + // b->m_route unset b->m_package = PackagePtr(new Package(s, package.origin())); m_backend_list.push_back(BackendPtr(b)); } - // we're going to deal with this for sure.. - m_is_multi = true; // create init request @@ -344,8 +349,6 @@ void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu) // create the frontend init response based on each backend init response yp2::odr odr; - int i; - Z_APDU *f_apdu = odr.create_initResponse(gdu->u.z3950, 0, 0); Z_InitResponse *f_resp = f_apdu->u.initResponse; @@ -367,6 +370,7 @@ void yf::Multi::Frontend::init(Package &package, Z_GDU *gdu) if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which == Z_APDU_initResponse) { + int i; Z_APDU *b_apdu = gdu->u.z3950; Z_InitResponse *b_resp = b_apdu->u.initResponse; @@ -399,18 +403,28 @@ void yf::Multi::Frontend::search(Package &package, Z_APDU *apdu_req) { // create search request Z_SearchRequest *req = apdu_req->u.searchRequest; - + // deal with piggy back (for now disable) *req->smallSetUpperBound = 0; *req->largeSetLowerBound = 1; *req->mediumSetPresentNumber = 1; + int default_num_db = req->num_databaseNames; + char **default_db = req->databaseNames; + std::list::const_iterator bit; for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++) { PackagePtr p = (*bit)->m_package; - // we don't modify database name yet! - + yp2::odr odr; + + if (!yp2::util::set_databases_from_zurl(odr, (*bit)->m_vhost, + &req->num_databaseNames, + &req->databaseNames)) + { + req->num_databaseNames = default_num_db; + req->databaseNames = default_db; + } p->request() = apdu_req; p->copy_filter(package); } diff --git a/src/filter_virt_db.cpp b/src/filter_virt_db.cpp index fa2e5a1..1ad040c 100644 --- a/src/filter_virt_db.cpp +++ b/src/filter_virt_db.cpp @@ -1,4 +1,4 @@ -/* $Id: filter_virt_db.cpp,v 1.25 2006-01-16 01:10:19 adam Exp $ +/* $Id: filter_virt_db.cpp,v 1.26 2006-01-16 15:51:56 adam Exp $ Copyright (c) 2005, Index Data. %LICENSE% @@ -37,16 +37,18 @@ namespace yp2 { std::string m_setname; }; struct Virt_db::Map { - Map(std::string vhost, std::string route); + Map(std::list targets, std::string route); Map(); - std::string m_vhost; + std::list m_targets; std::string m_route; }; struct Virt_db::Backend { yp2::Session m_backend_session; +#if 0 std::string m_backend_database; - std::string m_frontend_database; - std::string m_vhost; +#endif + std::list m_frontend_databases; + std::list m_targets; std::string m_route; bool m_named_result_sets; int m_number_of_sets; @@ -67,10 +69,13 @@ namespace yp2 { void close(Package &package); typedef std::map::iterator Sets_it; - BackendPtr lookup_backend_from_database(std::string database); - BackendPtr create_backend_from_database(std::string database); + BackendPtr lookup_backend_from_databases( + std::list databases); + BackendPtr create_backend_from_databases( + std::list databases); - BackendPtr init_backend(std::string database, Package &package, + BackendPtr init_backend(std::list database, + Package &package, int &error_code, std::string &addinfo); Rep *m_p; }; @@ -95,54 +100,67 @@ namespace yp2 { using namespace yp2; -yf::Virt_db::BackendPtr yf::Virt_db::Frontend::lookup_backend_from_database( - std::string database) +yf::Virt_db::BackendPtr yf::Virt_db::Frontend::lookup_backend_from_databases( + std::list databases) { std::list::const_iterator map_it; map_it = m_backend_list.begin(); for (; map_it != m_backend_list.end(); map_it++) - if ((*map_it)->m_frontend_database == database) + if ((*map_it)->m_frontend_databases == databases) return *map_it; BackendPtr null; return null; } -yf::Virt_db::BackendPtr yf::Virt_db::Frontend::create_backend_from_database( - std::string database) +yf::Virt_db::BackendPtr yf::Virt_db::Frontend::create_backend_from_databases( + std::list databases) { - std::map::iterator map_it; - map_it = m_p->m_maps.find(database); - if (map_it == m_p->m_maps.end()) - { - BackendPtr ptr; - return ptr; - } BackendPtr b(new Backend); + std::list::const_iterator db_it = databases.begin(); b->m_number_of_sets = 0; - b->m_frontend_database = database; + b->m_frontend_databases = databases; b->m_named_result_sets = false; - b->m_route = map_it->second.m_route; - b->m_vhost = map_it->second.m_vhost; + + std::map targets_dedup; + for (; db_it != databases.end(); db_it++) + { + std::map::iterator map_it; + map_it = m_p->m_maps.find(*db_it); + if (map_it == m_p->m_maps.end()) // database not found + { + BackendPtr ptr; + return ptr; + } + std::list::const_iterator t_it = + map_it->second.m_targets.begin(); + for (; t_it != map_it->second.m_targets.end(); t_it++) + targets_dedup[*t_it] = true; + b->m_route = map_it->second.m_route; + } + std::map::const_iterator tm_it = targets_dedup.begin(); + for (; tm_it != targets_dedup.end(); tm_it++) + b->m_targets.push_back(tm_it->first); +#if 0 const char *sep = strchr(b->m_vhost.c_str(), '/'); std::string backend_database; if (sep) b->m_backend_database = std::string(sep+1); else b->m_backend_database = database; - +#endif return b; } yf::Virt_db::BackendPtr yf::Virt_db::Frontend::init_backend( - std::string database, Package &package, + std::list databases, Package &package, int &error_code, std::string &addinfo) { - BackendPtr b = create_backend_from_database(database); + BackendPtr b = create_backend_from_databases(databases); if (!b) { error_code = YAZ_BIB1_DATABASE_UNAVAILABLE; - addinfo = database; + // addinfo = database; return b; } Package init_package(b->m_backend_session, package.origin()); @@ -152,9 +170,13 @@ yf::Virt_db::BackendPtr yf::Virt_db::Frontend::init_backend( Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest); - yaz_oi_set_string_oidval(&init_apdu->u.initRequest->otherInfo, odr, - VAL_PROXY, 1, b->m_vhost.c_str()); - + std::list::const_iterator t_it = b->m_targets.begin(); + int cat = 1; + for (; t_it != b->m_targets.end(); t_it++, cat++) + { + yaz_oi_set_string_oidval(&init_apdu->u.initRequest->otherInfo, odr, + VAL_PROXY, cat, t_it->c_str()); + } Z_InitRequest *req = init_apdu->u.initRequest; ODR_MASK_SET(req->options, Z_Options_search); @@ -173,7 +195,7 @@ yf::Virt_db::BackendPtr yf::Virt_db::Frontend::init_backend( if (init_package.session().is_closed()) { error_code = YAZ_BIB1_DATABASE_UNAVAILABLE; - addinfo = database; + // addinfo = database; BackendPtr null; return null; } @@ -191,7 +213,7 @@ yf::Virt_db::BackendPtr yf::Virt_db::Frontend::init_backend( else { error_code = YAZ_BIB1_DATABASE_UNAVAILABLE; - addinfo = database; + // addinfo = database; BackendPtr null; return null; } @@ -206,17 +228,10 @@ void yf::Virt_db::Frontend::search(Package &package, Z_APDU *apdu_req) std::string resultSetId = req->resultSetName; yp2::odr odr; - // only one datatabase for now - if (req->num_databaseNames != 1) - { // exactly one database must be specified - Z_APDU *apdu = - odr.create_searchResponse( - apdu_req, YAZ_BIB1_TOO_MANY_DATABASES_SPECIFIED, 0); - package.response() = apdu; - - return; - } - std::string database = std::string(req->databaseNames[0]); + std::list databases; + int i; + for (i = 0; inum_databaseNames; i++) + databases.push_back(req->databaseNames[i]); BackendPtr b; // null for now Sets_it sets_it = m_sets.find(req->resultSetName); @@ -244,7 +259,7 @@ void yf::Virt_db::Frontend::search(Package &package, Z_APDU *apdu_req) for (; map_it != m_backend_list.end(); map_it++) { BackendPtr tmp = *map_it; - if (tmp->m_frontend_database == database) + if (tmp->m_frontend_databases == databases) break; } if (map_it != m_backend_list.end()) @@ -261,7 +276,7 @@ void yf::Virt_db::Frontend::search(Package &package, Z_APDU *apdu_req) for (; map_it != m_backend_list.end(); map_it++) { BackendPtr tmp = *map_it; - if (tmp->m_frontend_database == database && + if (tmp->m_frontend_databases == databases && (tmp->m_named_result_sets || tmp->m_number_of_sets == 0)) break; @@ -273,7 +288,7 @@ void yf::Virt_db::Frontend::search(Package &package, Z_APDU *apdu_req) { int error_code; std::string addinfo; - b = init_backend(database, package, error_code, addinfo); + b = init_backend(databases, package, error_code, addinfo); if (!b) { // did not get a backend (unavailable somehow?) @@ -302,8 +317,10 @@ void yf::Virt_db::Frontend::search(Package &package, Z_APDU *apdu_req) req->resultSetName = odr_strdup(odr, backend_setname.c_str()); } +#if 0 const char *backend_database = b->m_backend_database.c_str(); req->databaseNames[0] = odr_strdup(odr, backend_database); +#endif *req->replaceIndicator = 1; @@ -315,8 +332,7 @@ void yf::Virt_db::Frontend::search(Package &package, Z_APDU *apdu_req) { Z_APDU *apdu = odr.create_searchResponse( - apdu_req, - YAZ_BIB1_DATABASE_UNAVAILABLE, database.c_str()); + apdu_req, YAZ_BIB1_DATABASE_UNAVAILABLE, 0); package.response() = apdu; return; } @@ -412,8 +428,8 @@ yf::Virt_db::Set::~Set() { } -yf::Virt_db::Map::Map(std::string vhost, std::string route) - : m_vhost(vhost), m_route(route) +yf::Virt_db::Map::Map(std::list targets, std::string route) + : m_targets(targets), m_route(route) { } @@ -480,18 +496,11 @@ void yf::Virt_db::Frontend::scan(Package &package, Z_APDU *apdu_req) std::string vhost; yp2::odr odr; - // only one datatabase for now - if (req->num_databaseNames != 1) - { // exactly one database must be specified - Z_APDU *apdu = - odr.create_scanResponse( - apdu_req, - YAZ_BIB1_TOO_MANY_DATABASES_SPECIFIED, 0); - package.response() = apdu; - return; - } - std::string database = std::string(req->databaseNames[0]); - + std::list databases; + int i; + for (i = 0; inum_databaseNames; i++) + databases.push_back(req->databaseNames[i]); + BackendPtr b; // pick up any existing backend with a database match std::list::const_iterator map_it; @@ -499,7 +508,7 @@ void yf::Virt_db::Frontend::scan(Package &package, Z_APDU *apdu_req) for (; map_it != m_backend_list.end(); map_it++) { BackendPtr tmp = *map_it; - if (tmp->m_frontend_database == database) + if (tmp->m_frontend_databases == databases) break; } if (map_it != m_backend_list.end()) @@ -508,7 +517,7 @@ void yf::Virt_db::Frontend::scan(Package &package, Z_APDU *apdu_req) { int error_code; std::string addinfo; - b = init_backend(database, package, error_code, addinfo); + b = init_backend(databases, package, error_code, addinfo); if (!b) { // did not get a backend (unavailable somehow?) @@ -525,8 +534,10 @@ void yf::Virt_db::Frontend::scan(Package &package, Z_APDU *apdu_req) scan_package.copy_filter(package); +#if 0 const char *backend_database = b->m_backend_database.c_str(); req->databaseNames[0] = odr_strdup(odr, backend_database); +#endif scan_package.request() = yazpp_1::GDU(apdu_req); @@ -536,7 +547,7 @@ void yf::Virt_db::Frontend::scan(Package &package, Z_APDU *apdu_req) { Z_APDU *apdu = odr.create_scanResponse( - apdu_req, YAZ_BIB1_DATABASE_UNAVAILABLE, database.c_str()); + apdu_req, YAZ_BIB1_DATABASE_UNAVAILABLE, 0); package.response() = apdu; return; } @@ -544,10 +555,22 @@ void yf::Virt_db::Frontend::scan(Package &package, Z_APDU *apdu_req) } -void yf::Virt_db::add_map_db2vhost(std::string db, std::string vhost, - std::string route) +void yf::Virt_db::add_map_db2targets(std::string db, + std::list targets, + std::string route) { - m_p->m_maps[db] = Virt_db::Map(vhost, route); + m_p->m_maps[db] = Virt_db::Map(targets, route); +} + + +void yf::Virt_db::add_map_db2target(std::string db, + std::string target, + std::string route) +{ + std::list targets; + targets.push_back(target); + + m_p->m_maps[db] = Virt_db::Map(targets, route); } void yf::Virt_db::process(Package &package) const @@ -651,7 +674,7 @@ void yp2::filter::Virt_db::configure(const xmlNode * ptr) if (!strcmp((const char *) ptr->name, "virtual")) { std::string database; - std::string target; + std::list targets; xmlNode *v_node = ptr->children; for (; v_node; v_node = v_node->next) { @@ -661,7 +684,7 @@ void yp2::filter::Virt_db::configure(const xmlNode * ptr) if (yp2::xml::is_element_yp2(v_node, "database")) database = yp2::xml::get_text(v_node); else if (yp2::xml::is_element_yp2(v_node, "target")) - target = yp2::xml::get_text(v_node); + targets.push_back(yp2::xml::get_text(v_node)); else throw yp2::filter::FilterException ("Bad element " @@ -670,9 +693,11 @@ void yp2::filter::Virt_db::configure(const xmlNode * ptr) ); } std::string route = yp2::xml::get_route(ptr); - add_map_db2vhost(database, target, route); + add_map_db2targets(database, targets, route); +#if 0 std::cout << "Add " << database << "->" << target << "," << route << "\n"; +#endif } else { diff --git a/src/filter_virt_db.hpp b/src/filter_virt_db.hpp index 6bd733c..e16fc34 100644 --- a/src/filter_virt_db.hpp +++ b/src/filter_virt_db.hpp @@ -1,4 +1,4 @@ -/* $Id: filter_virt_db.hpp,v 1.12 2006-01-15 20:03:14 adam Exp $ +/* $Id: filter_virt_db.hpp,v 1.13 2006-01-16 15:51:56 adam Exp $ Copyright (c) 2005, Index Data. %LICENSE% @@ -29,8 +29,12 @@ namespace yp2 { Virt_db(); void process(yp2::Package & package) const; void configure(const xmlNode * ptr); - void add_map_db2vhost(std::string db, std::string vhost, - std::string route); + void add_map_db2targets(std::string db, + std::list targets, + std::string route); + void add_map_db2target(std::string db, + std::string target, + std::string route); private: boost::scoped_ptr m_p; }; diff --git a/src/util.cpp b/src/util.cpp index fc304ca..da9eb53 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -1,4 +1,4 @@ -/* $Id: util.cpp,v 1.4 2006-01-13 15:09:35 adam Exp $ +/* $Id: util.cpp,v 1.5 2006-01-16 15:51:56 adam Exp $ Copyright (c) 2005, Index Data. %LICENSE% @@ -10,6 +10,7 @@ #include #include "util.hpp" + bool yp2::util::pqf(ODR odr, Z_APDU *apdu, const std::string &q) { YAZ_PQF_Parser pqf_parser = yaz_pqf_create(); @@ -28,6 +29,45 @@ bool yp2::util::pqf(ODR odr, Z_APDU *apdu, const std::string &q) { return true; } + +bool yp2::util::set_databases_from_zurl(ODR odr, std::string zurl, + int *db_num, char ***db_strings) +{ + const char *sep = strchr(zurl.c_str(), '/'); + if (!sep) + return false; + + int num = 0; + const char *cp1 = sep+1; + while(1) + { + const char *cp2 = strchr(cp1, '+'); + if (!cp2) + break; + cp1 = cp2+1; + num++; + } + *db_num = num+1; + *db_strings = (char **) odr_malloc(odr, sizeof(char*) * (*db_num)); + + num = 0; + cp1 = sep+1; + while(1) + { + const char *cp2 = strchr(cp1, '+'); + if (cp2) + (*db_strings)[num] = odr_strdupn(odr, cp1, cp2-cp1-1); + else + { + (*db_strings)[num] = odr_strdup(odr, cp1); + break; + } + cp1 = cp2+1; + num++; + } + return true; +} + yp2::odr::odr(int type) { m_odr = odr_createmem(type); diff --git a/src/util.hpp b/src/util.hpp index 2ab14a1..cb3c165 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -1,4 +1,4 @@ -/* $Id: util.hpp,v 1.4 2006-01-13 15:09:35 adam Exp $ +/* $Id: util.hpp,v 1.5 2006-01-16 15:51:56 adam Exp $ Copyright (c) 2005, Index Data. %LICENSE% @@ -17,6 +17,8 @@ namespace yp2 { bool pqf(ODR odr, Z_APDU *apdu, const std::string &q); Z_ReferenceId **get_referenceId(Z_APDU *apdu); Z_APDU *create_APDU(ODR odr, int type, Z_APDU *in_apdu); + bool set_databases_from_zurl(ODR odr, std::string zurl, + int *db_num, char ***db_strings); }; class odr : public boost::noncopyable { -- 1.7.10.4