X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Ffilter_virt_db.cpp;h=c9474c894dfdd75882ff627a2977dfb516bb6c18;hb=fa059d055896a26b113e0e486531b71e7ec63c14;hp=a7d0ae8c2dcabb8236efdfbdd5b6bbc455ed9b4e;hpb=0d323a3f455c0a0a669fa33e77b670d1af23fec5;p=metaproxy-moved-to-github.git diff --git a/src/filter_virt_db.cpp b/src/filter_virt_db.cpp index a7d0ae8..c9474c8 100644 --- a/src/filter_virt_db.cpp +++ b/src/filter_virt_db.cpp @@ -1,5 +1,5 @@ /* This file is part of Metaproxy. - Copyright (C) 2005-2009 Index Data + Copyright (C) 2005-2011 Index Data Metaproxy is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -18,20 +18,22 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include "config.hpp" -#include "filter.hpp" -#include "package.hpp" +#include "filter_virt_db.hpp" +#include #include #include #include -#include "util.hpp" -#include "filter_virt_db.hpp" +#include +#include "torus.hpp" #include #include #include #include +#include +#include #include #include @@ -52,17 +54,21 @@ namespace metaproxy_1 { }; struct VirtualDB::Map { Map(std::string database, std::list targets, std::string route); + Map(std::string database, std::string target, std::string route); Map(); bool match(const std::string db) const; std::string m_dbpattern; std::list m_targets; std::string m_route; + public: + std::string query_encoding; }; struct VirtualDB::Backend { mp::Session m_backend_session; std::list m_frontend_databases; std::list m_targets; std::string m_route; + std::string query_encoding; bool m_named_result_sets; int m_number_of_sets; }; @@ -79,6 +85,7 @@ namespace metaproxy_1 { void search(Package &package, Z_APDU *apdu); void present(Package &package, Z_APDU *apdu); void scan(Package &package, Z_APDU *apdu); + int relay_apdu(Package &package, Z_APDU *apdu); void close(Package &package); typedef std::map::iterator Sets_it; @@ -107,6 +114,7 @@ namespace metaproxy_1 { FrontendPtr get_frontend(Package &package); void release_frontend(Package &package); + void refresh_torus(); private: std::listm_maps; typedef std::map::iterator Sets_it; @@ -114,6 +122,7 @@ namespace metaproxy_1 { boost::condition m_cond_session_ready; std::map m_clients; bool pass_vhosts; + mp::Torus torus; }; } } @@ -174,12 +183,20 @@ yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::create_backend_from_databases // see if we have a route conflict. if (!first_route && b->m_route != map_it->m_route) { - // we have a conflict.. + // we have a conflict.. routing must be same for all error_code = YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP; BackendPtr ptr; return ptr; } b->m_route = map_it->m_route; + if (!first_route && b->query_encoding != map_it->query_encoding) + { + // we have a conflict. query encoding must be same for all + error_code = YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP; + BackendPtr ptr; + return ptr; + } + b->query_encoding = map_it->query_encoding; first_route = false; } return b; @@ -212,6 +229,8 @@ yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::init_backend( req->implementationId = org_init->implementationId; req->implementationName = org_init->implementationName; req->implementationVersion = org_init->implementationVersion; + *req->preferredMessageSize = *org_init->preferredMessageSize; + *req->maximumRecordSize = *org_init->maximumRecordSize; ODR_MASK_SET(req->options, Z_Options_search); ODR_MASK_SET(req->options, Z_Options_present); @@ -228,21 +247,21 @@ yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::init_backend( Z_GDU *gdu = init_package.response().get(); // we hope to get an init response - if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which == - Z_APDU_initResponse) + error_code = YAZ_BIB1_DATABASE_UNAVAILABLE; + if (gdu && gdu->which == Z_GDU_Z3950 + && gdu->u.z3950->which == Z_APDU_initResponse + && *gdu->u.z3950->u.initResponse->result) { Z_InitResponse *res = gdu->u.z3950->u.initResponse; if (ODR_MASK_GET(res->options, Z_Options_namedResultSets)) { b->m_named_result_sets = true; } - if (*res->result) + if (*res->result && !init_package.session().is_closed()) { m_backend_list.push_back(b); return b; - } - error_code = YAZ_BIB1_DATABASE_UNAVAILABLE; mp::util::get_init_diagnostics(res, error_code, addinfo); } if (!init_package.session().is_closed()) @@ -268,7 +287,6 @@ void yf::VirtualDB::Frontend::search(mp::Package &package, Z_APDU *apdu_req) for (i = 0; inum_databaseNames; i++) databases.push_back(req->databaseNames[i]); - BackendPtr b; // null for now Sets_it sets_it = m_sets.find(req->resultSetName); if (sets_it != m_sets.end()) { @@ -287,37 +305,22 @@ void yf::VirtualDB::Frontend::search(mp::Package &package, Z_APDU *apdu_req) return; } sets_it->second.m_backend->m_number_of_sets--; - - // pick up any existing backend with a database match - std::list::const_iterator map_it; - map_it = m_backend_list.begin(); - for (; map_it != m_backend_list.end(); map_it++) - { - BackendPtr tmp = *map_it; - if (tmp->m_frontend_databases == databases) - break; - } - if (map_it != m_backend_list.end()) - b = *map_it; } - else + // pick up any existing database with named result sets .. + // or one which has no result sets.. yet. + BackendPtr b; // null for now + std::list::const_iterator map_it; + map_it = m_backend_list.begin(); + for (; map_it != m_backend_list.end(); map_it++) { - // new result set. - - // pick up any existing database with named result sets .. - // or one which has no result sets.. yet. - std::list::const_iterator map_it; - map_it = m_backend_list.begin(); - for (; map_it != m_backend_list.end(); map_it++) - { - BackendPtr tmp = *map_it; - if (tmp->m_frontend_databases == databases && - (tmp->m_named_result_sets || - tmp->m_number_of_sets == 0)) - break; - } - if (map_it != m_backend_list.end()) + BackendPtr tmp = *map_it; + if (tmp->m_frontend_databases == databases && + (tmp->m_named_result_sets || + tmp->m_number_of_sets == 0)) + { b = *map_it; + break; + } } if (!b) // no backend yet. Must create a new one { @@ -361,6 +364,20 @@ void yf::VirtualDB::Frontend::search(mp::Package &package, Z_APDU *apdu_req) &req->databaseNames); } + if (b->query_encoding.length() && + (req->query->which == Z_Query_type_1 || + req->query->which == Z_Query_type_101)) + { + yaz_iconv_t cd = yaz_iconv_open(b->query_encoding.c_str(), "UTF-8"); + if (cd) + { + int r = yaz_query_charset_convert_rpnquery_check( + req->query->u.type_1, odr, cd); + yaz_iconv_close(cd); + if (r) + yaz_log(YLOG_LOG, "query could not be converted"); + } + } *req->replaceIndicator = 1; search_package.request() = yazpp_1::GDU(apdu_req); @@ -450,6 +467,72 @@ void yf::VirtualDB::Rep::release_frontend(mp::Package &package) } } +void yf::VirtualDB::Rep::refresh_torus(void) +{ + xmlDoc *doc = torus.get_doc(); + if (!doc) + return; + + xmlNode *ptr1 = xmlDocGetRootElement(doc); + if (!ptr1) + return ; + + for (ptr1 = ptr1->children; ptr1; ptr1 = ptr1->next) + { + if (ptr1->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) ptr1->name, "record")) + { + xmlNode *ptr2 = ptr1; + for (ptr2 = ptr2->children; ptr2; ptr2 = ptr2->next) + { + if (ptr2->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) ptr2->name, "layer")) + { + std::string database; + std::string target; + std::string route; + std::string solr; + std::string query_encoding; + xmlNode *ptr3 = ptr2; + for (ptr3 = ptr3->children; ptr3; ptr3 = ptr3->next) + { + if (ptr3->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) ptr3->name, "id")) + { + database = mp::xml::get_text(ptr3); + } + else if (!strcmp((const char *) ptr3->name, "zurl")) + { + target = mp::xml::get_text(ptr3); + } + else if (!strcmp((const char *) ptr3->name, "sru")) + { + solr = mp::xml::get_text(ptr3); + } + else if (!strcmp((const char *) ptr3->name, + "queryEncoding")) + { + query_encoding = mp::xml::get_text(ptr3); + } + } + if (solr.length() == 0 && + database.length() && target.length()) + { + VirtualDB::Map vmap( + mp::util::database_name_normalize(database), + target, route); + vmap.query_encoding = query_encoding; + m_maps.push_back(vmap); + } + } + } + } + } +} + yf::VirtualDB::Set::Set(BackendPtr b, std::string setname) : m_backend(b), m_setname(setname) { @@ -471,6 +554,14 @@ yf::VirtualDB::Map::Map(std::string database, { } +yf::VirtualDB::Map::Map(std::string database, + std::string target, std::string route) + : m_dbpattern(database), m_route(route) +{ + m_targets.push_back(target); +} + + yf::VirtualDB::Map::Map() { } @@ -617,6 +708,27 @@ void yf::VirtualDB::Frontend::present(mp::Package &package, Z_APDU *apdu_req) delete id; } +int yf::VirtualDB::Frontend::relay_apdu(mp::Package &package, Z_APDU *apdu_req) +{ + int no = 0; + std::list::const_iterator map_it; + map_it = m_backend_list.begin(); + for (; map_it != m_backend_list.end(); map_it++) + { + BackendPtr b = *map_it; + + Package relay_package(b->m_backend_session, package.origin()); + relay_package.copy_filter(package); + + relay_package.request() = yazpp_1::GDU(apdu_req); + + relay_package.move(b->m_route); + package.response() = relay_package.response(); + no++; + } + return no; +} + void yf::VirtualDB::Frontend::scan(mp::Package &package, Z_APDU *apdu_req) { Z_ScanRequest *req = apdu_req->u.scanRequest; @@ -669,6 +781,17 @@ void yf::VirtualDB::Frontend::scan(mp::Package &package, Z_APDU *apdu_req) &req->num_databaseNames, &req->databaseNames); } + + if (b->query_encoding.length()) + { + yaz_iconv_t cd = yaz_iconv_open(b->query_encoding.c_str(), "UTF-8"); + if (cd) + { + yaz_query_charset_convert_apt(req->termListAndStartPoint, odr, cd); + yaz_iconv_close(cd); + } + } + scan_package.request() = yazpp_1::GDU(apdu_req); scan_package.move(b->m_route); @@ -684,8 +807,8 @@ void yf::VirtualDB::Frontend::scan(mp::Package &package, Z_APDU *apdu_req) void yf::VirtualDB::add_map_db2targets(std::string db, - std::list targets, - std::string route) + std::list targets, + std::string route) { m_p->m_maps.push_back( VirtualDB::Map(mp::util::database_name_normalize(db), targets, route)); @@ -693,13 +816,12 @@ void yf::VirtualDB::add_map_db2targets(std::string db, void yf::VirtualDB::add_map_db2target(std::string db, - std::string target, - std::string route) -{ - std::list targets; - targets.push_back(target); + std::string target, + std::string route) - add_map_db2targets(db, targets, route); +{ + m_p->m_maps.push_back( + VirtualDB::Map(mp::util::database_name_normalize(db), target, route)); } void yf::VirtualDB::process(mp::Package &package) const @@ -752,6 +874,9 @@ void yf::VirtualDB::process(mp::Package &package) const else break; + *resp->preferredMessageSize = *req->preferredMessageSize; + *resp->maximumRecordSize = *req->maximumRecordSize; + package.response() = apdu; f->m_is_virtual = true; } @@ -786,7 +911,15 @@ void yf::VirtualDB::process(mp::Package &package) const } else if (apdu->which == Z_APDU_close) { - package.session().close(); + if (f->relay_apdu(package, apdu) == 0) + { + mp::odr odr; + + package.response() = odr.create_close( + apdu, Z_Close_finished, "virt_db"); + + package.session().close(); + } } else { @@ -802,7 +935,6 @@ void yf::VirtualDB::process(mp::Package &package) const m_p->release_frontend(package); } - void mp::filter::VirtualDB::configure(const xmlNode * ptr, bool test_only) { for (ptr = ptr->children; ptr; ptr = ptr->next) @@ -815,6 +947,7 @@ void mp::filter::VirtualDB::configure(const xmlNode * ptr, bool test_only) } else if (!strcmp((const char *) ptr->name, "virtual")) { + std::string query_encoding; std::string database; std::list targets; xmlNode *v_node = ptr->children; @@ -827,6 +960,8 @@ void mp::filter::VirtualDB::configure(const xmlNode * ptr, bool test_only) database = mp::xml::get_text(v_node); else if (mp::xml::is_element_mp(v_node, "target")) targets.push_back(mp::xml::get_text(v_node)); + else if (mp::xml::is_element_mp(v_node, "query-encoding")) + targets.push_back(mp::xml::get_text(v_node)); else throw mp::filter::FilterException ("Bad element " @@ -835,7 +970,27 @@ void mp::filter::VirtualDB::configure(const xmlNode * ptr, bool test_only) ); } std::string route = mp::xml::get_route(ptr); - add_map_db2targets(database, targets, route); + + VirtualDB::Map vmap(mp::util::database_name_normalize(database), + targets, route); + vmap.query_encoding = query_encoding; + m_p->m_maps.push_back(vmap); + } + else if (!strcmp((const char *) ptr->name, "torus")) + { + std::string url; + const struct _xmlAttr *attr; + for (attr = ptr->properties; attr; attr = attr->next) + { + if (!strcmp((const char *) attr->name, "url")) + url = mp::xml::get_text(attr->children); + else + throw mp::filter::FilterException( + "Bad attribute " + std::string((const char *) + attr->name)); + } + m_p->torus.read_searchables(url); + m_p->refresh_torus(); } else {