1 /* This file is part of Metaproxy.
2 Copyright (C) 2005-2011 Index Data
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
21 #include "filter_virt_db.hpp"
22 #include <metaproxy/package.hpp>
24 #include <boost/thread/mutex.hpp>
25 #include <boost/thread/condition.hpp>
26 #include <boost/shared_ptr.hpp>
28 #include <metaproxy/util.hpp>
32 #include <yaz/otherinfo.h>
33 #include <yaz/diagbib1.h>
34 #include <yaz/match_glob.h>
40 namespace mp = metaproxy_1;
41 namespace yf = mp::filter;
43 namespace metaproxy_1 {
46 struct VirtualDB::Set {
47 Set(BackendPtr b, std::string setname);
52 std::string m_setname;
54 struct VirtualDB::Map {
55 Map(std::string database, std::list<std::string> targets, std::string route);
57 bool match(const std::string db) const;
58 std::string m_dbpattern;
59 std::list<std::string> m_targets;
62 struct VirtualDB::Backend {
63 mp::Session m_backend_session;
64 std::list<std::string> m_frontend_databases;
65 std::list<std::string> m_targets;
67 bool m_named_result_sets;
70 struct VirtualDB::Frontend {
73 mp::Session m_session;
76 yazpp_1::GDU m_init_gdu;
77 std::list<BackendPtr> m_backend_list;
78 std::map<std::string,VirtualDB::Set> m_sets;
80 void search(Package &package, Z_APDU *apdu);
81 void present(Package &package, Z_APDU *apdu);
82 void scan(Package &package, Z_APDU *apdu);
83 int relay_apdu(Package &package, Z_APDU *apdu);
85 void close(Package &package);
86 typedef std::map<std::string,VirtualDB::Set>::iterator Sets_it;
88 void fixup_package(Package &p, BackendPtr b);
89 void fixup_npr_record(ODR odr, Z_NamePlusRecord *npr,
91 void fixup_npr_records(ODR odr, Z_Records *records,
94 BackendPtr lookup_backend_from_databases(
95 std::list<std::string> databases);
96 BackendPtr create_backend_from_databases(
97 std::list<std::string> databases,
99 std::string &failing_database);
101 BackendPtr init_backend(std::list<std::string> database,
103 int &error_code, std::string &addinfo);
106 class VirtualDB::Rep {
107 friend class VirtualDB;
108 friend struct Frontend;
110 FrontendPtr get_frontend(Package &package);
111 void release_frontend(Package &package);
113 std::list<VirtualDB::Map>m_maps;
114 typedef std::map<std::string,VirtualDB::Set>::iterator Sets_it;
115 boost::mutex m_mutex;
116 boost::condition m_cond_session_ready;
117 std::map<mp::Session, FrontendPtr> m_clients;
124 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::lookup_backend_from_databases(
125 std::list<std::string> databases)
127 std::list<BackendPtr>::const_iterator map_it;
128 map_it = m_backend_list.begin();
129 for (; map_it != m_backend_list.end(); map_it++)
130 if ((*map_it)->m_frontend_databases == databases)
136 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::create_backend_from_databases(
137 std::list<std::string> databases, int &error_code, std::string &addinfo)
139 BackendPtr b(new Backend);
140 std::list<std::string>::const_iterator db_it = databases.begin();
142 b->m_number_of_sets = 0;
143 b->m_frontend_databases = databases;
144 b->m_named_result_sets = false;
146 bool first_route = true;
148 std::map<std::string,bool> targets_dedup;
149 for (; db_it != databases.end(); db_it++)
151 std::list<VirtualDB::Map>::const_iterator map_it;
152 map_it = m_p->m_maps.begin();
153 while (map_it != m_p->m_maps.end())
155 if (map_it->match(*db_it))
160 if (map_it == m_p->m_maps.end()) // database not found
162 error_code = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
167 std::list<std::string>::const_iterator t_it =
168 map_it->m_targets.begin();
169 for (; t_it != map_it->m_targets.end(); t_it++) {
170 if (!targets_dedup[*t_it])
172 targets_dedup[*t_it] = true;
173 b->m_targets.push_back(*t_it);
177 // see if we have a route conflict.
178 if (!first_route && b->m_route != map_it->m_route)
180 // we have a conflict..
181 error_code = YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP;
185 b->m_route = map_it->m_route;
191 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::init_backend(
192 std::list<std::string> databases, mp::Package &package,
193 int &error_code, std::string &addinfo)
195 BackendPtr b = create_backend_from_databases(databases, error_code,
199 Package init_package(b->m_backend_session, package.origin());
200 init_package.copy_filter(package);
204 Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest);
206 mp::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo, odr,
208 Z_InitRequest *req = init_apdu->u.initRequest;
210 // copy stuff from Frontend Init Request
211 Z_GDU *org_gdu = m_init_gdu.get();
212 Z_InitRequest *org_init = org_gdu->u.z3950->u.initRequest;
214 req->idAuthentication = org_init->idAuthentication;
215 req->implementationId = org_init->implementationId;
216 req->implementationName = org_init->implementationName;
217 req->implementationVersion = org_init->implementationVersion;
218 *req->preferredMessageSize = *org_init->preferredMessageSize;
219 *req->maximumRecordSize = *org_init->maximumRecordSize;
221 ODR_MASK_SET(req->options, Z_Options_search);
222 ODR_MASK_SET(req->options, Z_Options_present);
223 ODR_MASK_SET(req->options, Z_Options_namedResultSets);
224 ODR_MASK_SET(req->options, Z_Options_scan);
226 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
227 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
228 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3);
230 init_package.request() = init_apdu;
232 init_package.move(b->m_route); // sending init
234 Z_GDU *gdu = init_package.response().get();
235 // we hope to get an init response
236 error_code = YAZ_BIB1_DATABASE_UNAVAILABLE;
237 if (gdu && gdu->which == Z_GDU_Z3950
238 && gdu->u.z3950->which == Z_APDU_initResponse
239 && *gdu->u.z3950->u.initResponse->result)
241 Z_InitResponse *res = gdu->u.z3950->u.initResponse;
242 if (ODR_MASK_GET(res->options, Z_Options_namedResultSets))
244 b->m_named_result_sets = true;
246 if (*res->result && !init_package.session().is_closed())
248 m_backend_list.push_back(b);
251 mp::util::get_init_diagnostics(res, error_code, addinfo);
253 if (!init_package.session().is_closed())
255 Package close_package(b->m_backend_session, package.origin());
256 close_package.copy_filter(package);
257 close_package.session().close();
258 close_package.move(b->m_route); // closing it
264 void yf::VirtualDB::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
266 Z_SearchRequest *req = apdu_req->u.searchRequest;
268 std::string resultSetId = req->resultSetName;
271 std::list<std::string> databases;
273 for (i = 0; i<req->num_databaseNames; i++)
274 databases.push_back(req->databaseNames[i]);
276 Sets_it sets_it = m_sets.find(req->resultSetName);
277 if (sets_it != m_sets.end())
279 // result set already exist
280 // if replace indicator is off: we return diagnostic if
281 // result set already exist.
282 if (*req->replaceIndicator == 0)
285 odr.create_searchResponse(
287 YAZ_BIB1_RESULT_SET_EXISTS_AND_REPLACE_INDICATOR_OFF,
289 package.response() = apdu;
293 sets_it->second.m_backend->m_number_of_sets--;
295 // pick up any existing database with named result sets ..
296 // or one which has no result sets.. yet.
297 BackendPtr b; // null for now
298 std::list<BackendPtr>::const_iterator map_it;
299 map_it = m_backend_list.begin();
300 for (; map_it != m_backend_list.end(); map_it++)
302 BackendPtr tmp = *map_it;
303 if (tmp->m_frontend_databases == databases &&
304 (tmp->m_named_result_sets ||
305 tmp->m_number_of_sets == 0))
311 if (!b) // no backend yet. Must create a new one
315 b = init_backend(databases, package, error_code, addinfo);
318 // did not get a backend (unavailable somehow?)
321 odr.create_searchResponse(
322 apdu_req, error_code, addinfo.c_str());
323 package.response() = apdu;
327 m_sets.erase(req->resultSetName);
328 // sending search to backend
329 Package search_package(b->m_backend_session, package.origin());
331 search_package.copy_filter(package);
333 std::string backend_setname;
334 if (b->m_named_result_sets)
336 backend_setname = std::string(req->resultSetName);
340 backend_setname = "default";
341 req->resultSetName = odr_strdup(odr, backend_setname.c_str());
344 // pick first targets spec and move the databases from it ..
345 std::list<std::string>::const_iterator t_it = b->m_targets.begin();
346 if (t_it != b->m_targets.end())
348 mp::util::set_databases_from_zurl(odr, *t_it,
349 &req->num_databaseNames,
350 &req->databaseNames);
353 *req->replaceIndicator = 1;
355 search_package.request() = yazpp_1::GDU(apdu_req);
357 search_package.move(b->m_route);
359 if (search_package.session().is_closed())
361 package.response() = search_package.response();
362 package.session().close();
365 b->m_number_of_sets++;
367 m_sets[resultSetId] = VirtualDB::Set(b, backend_setname);
368 fixup_package(search_package, b);
369 package.response() = search_package.response();
372 yf::VirtualDB::Frontend::Frontend(Rep *rep)
375 m_is_virtual = false;
378 void yf::VirtualDB::Frontend::close(mp::Package &package)
380 std::list<BackendPtr>::const_iterator b_it;
382 for (b_it = m_backend_list.begin(); b_it != m_backend_list.end(); b_it++)
384 (*b_it)->m_backend_session.close();
385 Package close_package((*b_it)->m_backend_session, package.origin());
386 close_package.copy_filter(package);
387 close_package.move((*b_it)->m_route);
389 m_backend_list.clear();
392 yf::VirtualDB::Frontend::~Frontend()
396 yf::VirtualDB::FrontendPtr yf::VirtualDB::Rep::get_frontend(mp::Package &package)
398 boost::mutex::scoped_lock lock(m_mutex);
400 std::map<mp::Session,yf::VirtualDB::FrontendPtr>::iterator it;
404 it = m_clients.find(package.session());
405 if (it == m_clients.end())
408 if (!it->second->m_in_use)
410 it->second->m_in_use = true;
413 m_cond_session_ready.wait(lock);
415 FrontendPtr f(new Frontend(this));
416 m_clients[package.session()] = f;
421 void yf::VirtualDB::Rep::release_frontend(mp::Package &package)
423 boost::mutex::scoped_lock lock(m_mutex);
424 std::map<mp::Session,yf::VirtualDB::FrontendPtr>::iterator it;
426 it = m_clients.find(package.session());
427 if (it != m_clients.end())
429 if (package.session().is_closed())
431 it->second->close(package);
436 it->second->m_in_use = false;
438 m_cond_session_ready.notify_all();
442 void yf::VirtualDB::refresh_torus(void)
444 xmlDoc *doc = m_p->torus.get_doc();
446 if (doc && (ptr1 = xmlDocGetRootElement(doc)))
448 for (ptr1 = ptr1->children; ptr1; ptr1 = ptr1->next)
450 if (ptr1->type != XML_ELEMENT_NODE)
452 if (!strcmp((const char *) ptr1->name, "record"))
454 xmlNode *ptr2 = ptr1;
455 for (ptr2 = ptr2->children; ptr2; ptr2 = ptr2->next)
457 if (ptr2->type != XML_ELEMENT_NODE)
459 if (!strcmp((const char *) ptr2->name, "layer"))
461 std::string database;
465 xmlNode *ptr3 = ptr2;
466 for (ptr3 = ptr3->children; ptr3; ptr3 = ptr3->next)
468 if (ptr3->type != XML_ELEMENT_NODE)
470 if (!strcmp((const char *) ptr3->name, "id"))
472 database = mp::xml::get_text(ptr3);
474 else if (!strcmp((const char *) ptr3->name, "zurl"))
476 target = mp::xml::get_text(ptr3);
478 else if (!strcmp((const char *) ptr3->name, "sru"))
480 solr = mp::xml::get_text(ptr3);
483 if (solr.length() == 0 &&
484 database.length() && target.length())
486 add_map_db2target(database, target, route);
495 yf::VirtualDB::Set::Set(BackendPtr b, std::string setname)
496 : m_backend(b), m_setname(setname)
501 yf::VirtualDB::Set::Set()
506 yf::VirtualDB::Set::~Set()
510 yf::VirtualDB::Map::Map(std::string database,
511 std::list<std::string> targets, std::string route)
512 : m_dbpattern(database), m_targets(targets), m_route(route)
516 yf::VirtualDB::Map::Map()
520 bool yf::VirtualDB::Map::match(const std::string db) const
522 std::string norm_db = mp::util::database_name_normalize(db);
523 if (yaz_match_glob(m_dbpattern.c_str(), norm_db.c_str()))
528 yf::VirtualDB::VirtualDB() : m_p(new VirtualDB::Rep)
530 m_p->pass_vhosts = false;
533 yf::VirtualDB::~VirtualDB() {
536 void yf::VirtualDB::Frontend::fixup_npr_record(ODR odr, Z_NamePlusRecord *npr,
539 if (npr->databaseName)
541 std::string b_database = std::string(npr->databaseName);
543 // consider each of the frontend databases..
544 std::list<std::string>::const_iterator db_it;
545 for (db_it = b->m_frontend_databases.begin();
546 db_it != b->m_frontend_databases.end(); db_it++)
548 // see which target it corresponds to.. (if any)
549 std::list<VirtualDB::Map>::const_iterator map_it =
551 while (map_it != m_p->m_maps.end())
553 if (map_it->match(*db_it))
557 if (map_it != m_p->m_maps.end())
559 std::list<std::string>::const_iterator t
560 = map_it->m_targets.begin();
561 while (t != map_it->m_targets.end())
563 if (*t == b_database)
565 npr->databaseName = odr_strdup(odr, (*db_it).c_str());
573 db_it = b->m_frontend_databases.begin();
574 if (db_it != b->m_frontend_databases.end())
576 std::string database = *db_it;
577 npr->databaseName = odr_strdup(odr, database.c_str());
582 void yf::VirtualDB::Frontend::fixup_npr_records(ODR odr, Z_Records *records,
585 if (records && records->which == Z_Records_DBOSD)
587 Z_NamePlusRecordList *nprlist = records->u.databaseOrSurDiagnostics;
589 for (i = 0; i < nprlist->num_records; i++)
591 fixup_npr_record(odr, nprlist->records[i], b);
596 void yf::VirtualDB::Frontend::fixup_package(mp::Package &p, BackendPtr b)
598 Z_GDU *gdu = p.response().get();
601 if (gdu && gdu->which == Z_GDU_Z3950)
603 Z_APDU *apdu = gdu->u.z3950;
604 if (apdu->which == Z_APDU_presentResponse)
606 fixup_npr_records(odr, apdu->u.presentResponse->records, b);
609 else if (apdu->which == Z_APDU_searchResponse)
611 fixup_npr_records(odr, apdu->u.searchResponse->records, b);
617 void yf::VirtualDB::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
619 Z_PresentRequest *req = apdu_req->u.presentRequest;
620 std::string resultSetId = req->resultSetId;
623 Sets_it sets_it = m_sets.find(resultSetId);
624 if (sets_it == m_sets.end())
627 odr.create_presentResponse(
629 YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
630 resultSetId.c_str());
631 package.response() = apdu;
635 new mp::Session(sets_it->second.m_backend->m_backend_session);
637 // sending present to backend
638 Package present_package(*id, package.origin());
639 present_package.copy_filter(package);
641 req->resultSetId = odr_strdup(odr, sets_it->second.m_setname.c_str());
643 present_package.request() = yazpp_1::GDU(apdu_req);
645 present_package.move(sets_it->second.m_backend->m_route);
647 fixup_package(present_package, sets_it->second.m_backend);
649 if (present_package.session().is_closed())
651 package.response() = present_package.response();
652 package.session().close();
657 package.response() = present_package.response();
662 int yf::VirtualDB::Frontend::relay_apdu(mp::Package &package, Z_APDU *apdu_req)
665 std::list<BackendPtr>::const_iterator map_it;
666 map_it = m_backend_list.begin();
667 for (; map_it != m_backend_list.end(); map_it++)
669 BackendPtr b = *map_it;
671 Package relay_package(b->m_backend_session, package.origin());
672 relay_package.copy_filter(package);
674 relay_package.request() = yazpp_1::GDU(apdu_req);
676 relay_package.move(b->m_route);
677 package.response() = relay_package.response();
683 void yf::VirtualDB::Frontend::scan(mp::Package &package, Z_APDU *apdu_req)
685 Z_ScanRequest *req = apdu_req->u.scanRequest;
689 std::list<std::string> databases;
691 for (i = 0; i<req->num_databaseNames; i++)
692 databases.push_back(req->databaseNames[i]);
695 // pick up any existing backend with a database match
696 std::list<BackendPtr>::const_iterator map_it;
697 map_it = m_backend_list.begin();
698 for (; map_it != m_backend_list.end(); map_it++)
700 BackendPtr tmp = *map_it;
701 if (tmp->m_frontend_databases == databases)
704 if (map_it != m_backend_list.end())
706 if (!b) // no backend yet. Must create a new one
710 b = init_backend(databases, package, error_code, addinfo);
713 // did not get a backend (unavailable somehow?)
715 odr.create_scanResponse(
716 apdu_req, error_code, addinfo.c_str());
717 package.response() = apdu;
722 // sending scan to backend
723 Package scan_package(b->m_backend_session, package.origin());
725 scan_package.copy_filter(package);
727 // pick first targets spec and move the databases from it ..
728 std::list<std::string>::const_iterator t_it = b->m_targets.begin();
729 if (t_it != b->m_targets.end())
731 mp::util::set_databases_from_zurl(odr, *t_it,
732 &req->num_databaseNames,
733 &req->databaseNames);
735 scan_package.request() = yazpp_1::GDU(apdu_req);
737 scan_package.move(b->m_route);
739 if (scan_package.session().is_closed())
741 package.response() = scan_package.response();
742 package.session().close();
745 package.response() = scan_package.response();
749 void yf::VirtualDB::add_map_db2targets(std::string db,
750 std::list<std::string> targets,
753 m_p->m_maps.push_back(
754 VirtualDB::Map(mp::util::database_name_normalize(db), targets, route));
758 void yf::VirtualDB::add_map_db2target(std::string db,
762 std::list<std::string> targets;
763 targets.push_back(target);
765 add_map_db2targets(db, targets, route);
768 void yf::VirtualDB::process(mp::Package &package) const
770 FrontendPtr f = m_p->get_frontend(package);
772 Z_GDU *gdu = package.request().get();
774 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
775 Z_APDU_initRequest && !f->m_is_virtual)
777 Z_InitRequest *req = gdu->u.z3950->u.initRequest;
779 std::list<std::string> vhosts;
780 mp::util::get_vhost_otherinfo(req->otherInfo, vhosts);
782 if (vhosts.size() > 0 && m_p->pass_vhosts)
791 Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
792 Z_InitResponse *resp = apdu->u.initResponse;
795 static const int masks[] = {
798 Z_Options_namedResultSets,
802 for (i = 0; masks[i] != -1; i++)
803 if (ODR_MASK_GET(req->options, masks[i]))
804 ODR_MASK_SET(resp->options, masks[i]);
806 static const int versions[] = {
812 for (i = 0; versions[i] != -1; i++)
813 if (ODR_MASK_GET(req->protocolVersion, versions[i]))
814 ODR_MASK_SET(resp->protocolVersion, versions[i]);
818 *resp->preferredMessageSize = *req->preferredMessageSize;
819 *resp->maximumRecordSize = *req->maximumRecordSize;
821 package.response() = apdu;
822 f->m_is_virtual = true;
825 else if (!f->m_is_virtual)
827 else if (gdu && gdu->which == Z_GDU_Z3950)
829 Z_APDU *apdu = gdu->u.z3950;
830 if (apdu->which == Z_APDU_initRequest)
834 package.response() = odr.create_close(
836 Z_Close_protocolError,
839 package.session().close();
841 else if (apdu->which == Z_APDU_searchRequest)
843 f->search(package, apdu);
845 else if (apdu->which == Z_APDU_presentRequest)
847 f->present(package, apdu);
849 else if (apdu->which == Z_APDU_scanRequest)
851 f->scan(package, apdu);
853 else if (apdu->which == Z_APDU_close)
855 if (f->relay_apdu(package, apdu) == 0)
859 package.response() = odr.create_close(
860 apdu, Z_Close_finished, "virt_db");
862 package.session().close();
869 package.response() = odr.create_close(
870 apdu, Z_Close_protocolError,
871 "unsupported APDU in filter_virt_db");
873 package.session().close();
876 m_p->release_frontend(package);
879 void mp::filter::VirtualDB::configure(const xmlNode * ptr, bool test_only)
881 for (ptr = ptr->children; ptr; ptr = ptr->next)
883 if (ptr->type != XML_ELEMENT_NODE)
885 if (!strcmp((const char *) ptr->name, "pass-vhosts"))
887 m_p->pass_vhosts = mp::xml::get_bool(ptr->children, false);
889 else if (!strcmp((const char *) ptr->name, "virtual"))
891 std::string database;
892 std::list<std::string> targets;
893 xmlNode *v_node = ptr->children;
894 for (; v_node; v_node = v_node->next)
896 if (v_node->type != XML_ELEMENT_NODE)
899 if (mp::xml::is_element_mp(v_node, "database"))
900 database = mp::xml::get_text(v_node);
901 else if (mp::xml::is_element_mp(v_node, "target"))
902 targets.push_back(mp::xml::get_text(v_node));
904 throw mp::filter::FilterException
906 + std::string((const char *) v_node->name)
907 + " in virtual section"
910 std::string route = mp::xml::get_route(ptr);
911 add_map_db2targets(database, targets, route);
913 else if (!strcmp((const char *) ptr->name, "torus"))
916 const struct _xmlAttr *attr;
917 for (attr = ptr->properties; attr; attr = attr->next)
919 if (!strcmp((const char *) attr->name, "url"))
920 url = mp::xml::get_text(attr->children);
922 throw mp::filter::FilterException(
923 "Bad attribute " + std::string((const char *)
926 m_p->torus.read_searchables(url);
931 throw mp::filter::FilterException
933 + std::string((const char *) ptr->name)
934 + " in virt_db filter");
939 static mp::filter::Base* filter_creator()
941 return new mp::filter::VirtualDB;
945 struct metaproxy_1_filter_struct metaproxy_1_filter_virt_db = {
956 * c-file-style: "Stroustrup"
957 * indent-tabs-mode: nil
959 * vim: shiftwidth=4 tabstop=8 expandtab