1 /* This file is part of Metaproxy.
2 Copyright (C) 2005-2011 Index Data
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
21 #include "filter_virt_db.hpp"
22 #include <metaproxy/package.hpp>
24 #include <boost/thread/mutex.hpp>
25 #include <boost/thread/condition.hpp>
26 #include <boost/shared_ptr.hpp>
28 #include <metaproxy/util.hpp>
32 #include <yaz/otherinfo.h>
33 #include <yaz/diagbib1.h>
34 #include <yaz/match_glob.h>
35 #include <yaz/query-charset.h>
41 namespace mp = metaproxy_1;
42 namespace yf = mp::filter;
44 namespace metaproxy_1 {
47 struct VirtualDB::Set {
48 Set(BackendPtr b, std::string setname);
53 std::string m_setname;
55 struct VirtualDB::Map {
56 Map(std::string database, std::list<std::string> targets, std::string route);
57 Map(std::string database, std::string target, std::string route);
59 bool match(const std::string db) const;
60 std::string m_dbpattern;
61 std::list<std::string> m_targets;
64 std::string query_encoding;
66 struct VirtualDB::Backend {
67 mp::Session m_backend_session;
68 std::list<std::string> m_frontend_databases;
69 std::list<std::string> m_targets;
71 std::string query_encoding;
72 bool m_named_result_sets;
75 struct VirtualDB::Frontend {
78 mp::Session m_session;
81 yazpp_1::GDU m_init_gdu;
82 std::list<BackendPtr> m_backend_list;
83 std::map<std::string,VirtualDB::Set> m_sets;
85 void search(Package &package, Z_APDU *apdu);
86 void present(Package &package, Z_APDU *apdu);
87 void scan(Package &package, Z_APDU *apdu);
88 int relay_apdu(Package &package, Z_APDU *apdu);
90 void close(Package &package);
91 typedef std::map<std::string,VirtualDB::Set>::iterator Sets_it;
93 void fixup_package(Package &p, BackendPtr b);
94 void fixup_npr_record(ODR odr, Z_NamePlusRecord *npr,
96 void fixup_npr_records(ODR odr, Z_Records *records,
99 BackendPtr lookup_backend_from_databases(
100 std::list<std::string> databases);
101 BackendPtr create_backend_from_databases(
102 std::list<std::string> databases,
104 std::string &failing_database);
106 BackendPtr init_backend(std::list<std::string> database,
108 int &error_code, std::string &addinfo);
111 class VirtualDB::Rep {
112 friend class VirtualDB;
113 friend struct Frontend;
115 FrontendPtr get_frontend(Package &package);
116 void release_frontend(Package &package);
117 void refresh_torus();
119 std::list<VirtualDB::Map>m_maps;
120 typedef std::map<std::string,VirtualDB::Set>::iterator Sets_it;
121 boost::mutex m_mutex;
122 boost::condition m_cond_session_ready;
123 std::map<mp::Session, FrontendPtr> m_clients;
129 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::lookup_backend_from_databases(
130 std::list<std::string> databases)
132 std::list<BackendPtr>::const_iterator map_it;
133 map_it = m_backend_list.begin();
134 for (; map_it != m_backend_list.end(); map_it++)
135 if ((*map_it)->m_frontend_databases == databases)
141 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::create_backend_from_databases(
142 std::list<std::string> databases, int &error_code, std::string &addinfo)
144 BackendPtr b(new Backend);
145 std::list<std::string>::const_iterator db_it = databases.begin();
147 b->m_number_of_sets = 0;
148 b->m_frontend_databases = databases;
149 b->m_named_result_sets = false;
151 bool first_route = true;
153 std::map<std::string,bool> targets_dedup;
154 for (; db_it != databases.end(); db_it++)
156 std::list<VirtualDB::Map>::const_iterator map_it;
157 map_it = m_p->m_maps.begin();
158 while (map_it != m_p->m_maps.end())
160 if (map_it->match(*db_it))
165 if (map_it == m_p->m_maps.end()) // database not found
167 error_code = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
172 std::list<std::string>::const_iterator t_it =
173 map_it->m_targets.begin();
174 for (; t_it != map_it->m_targets.end(); t_it++) {
175 if (!targets_dedup[*t_it])
177 targets_dedup[*t_it] = true;
178 b->m_targets.push_back(*t_it);
182 // see if we have a route conflict.
183 if (!first_route && b->m_route != map_it->m_route)
185 // we have a conflict.. routing must be same for all
186 error_code = YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP;
190 b->m_route = map_it->m_route;
191 if (!first_route && b->query_encoding != map_it->query_encoding)
193 // we have a conflict. query encoding must be same for all
194 error_code = YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP;
198 b->query_encoding = map_it->query_encoding;
204 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::init_backend(
205 std::list<std::string> databases, mp::Package &package,
206 int &error_code, std::string &addinfo)
208 BackendPtr b = create_backend_from_databases(databases, error_code,
212 Package init_package(b->m_backend_session, package.origin());
213 init_package.copy_filter(package);
217 Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest);
219 mp::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo, odr,
221 Z_InitRequest *req = init_apdu->u.initRequest;
223 // copy stuff from Frontend Init Request
224 Z_GDU *org_gdu = m_init_gdu.get();
225 Z_InitRequest *org_init = org_gdu->u.z3950->u.initRequest;
227 req->idAuthentication = org_init->idAuthentication;
228 req->implementationId = org_init->implementationId;
229 req->implementationName = org_init->implementationName;
230 req->implementationVersion = org_init->implementationVersion;
231 *req->preferredMessageSize = *org_init->preferredMessageSize;
232 *req->maximumRecordSize = *org_init->maximumRecordSize;
234 ODR_MASK_SET(req->options, Z_Options_search);
235 ODR_MASK_SET(req->options, Z_Options_present);
236 ODR_MASK_SET(req->options, Z_Options_namedResultSets);
237 ODR_MASK_SET(req->options, Z_Options_scan);
239 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
240 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
241 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3);
243 init_package.request() = init_apdu;
245 init_package.move(b->m_route); // sending init
247 Z_GDU *gdu = init_package.response().get();
248 // we hope to get an init response
249 error_code = YAZ_BIB1_DATABASE_UNAVAILABLE;
250 if (gdu && gdu->which == Z_GDU_Z3950
251 && gdu->u.z3950->which == Z_APDU_initResponse
252 && *gdu->u.z3950->u.initResponse->result)
254 Z_InitResponse *res = gdu->u.z3950->u.initResponse;
255 if (ODR_MASK_GET(res->options, Z_Options_namedResultSets))
257 b->m_named_result_sets = true;
259 if (*res->result && !init_package.session().is_closed())
261 m_backend_list.push_back(b);
264 mp::util::get_init_diagnostics(res, error_code, addinfo);
266 if (!init_package.session().is_closed())
268 Package close_package(b->m_backend_session, package.origin());
269 close_package.copy_filter(package);
270 close_package.session().close();
271 close_package.move(b->m_route); // closing it
277 void yf::VirtualDB::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
279 Z_SearchRequest *req = apdu_req->u.searchRequest;
281 std::string resultSetId = req->resultSetName;
284 std::list<std::string> databases;
286 for (i = 0; i<req->num_databaseNames; i++)
287 databases.push_back(req->databaseNames[i]);
289 Sets_it sets_it = m_sets.find(req->resultSetName);
290 if (sets_it != m_sets.end())
292 // result set already exist
293 // if replace indicator is off: we return diagnostic if
294 // result set already exist.
295 if (*req->replaceIndicator == 0)
298 odr.create_searchResponse(
300 YAZ_BIB1_RESULT_SET_EXISTS_AND_REPLACE_INDICATOR_OFF,
302 package.response() = apdu;
306 sets_it->second.m_backend->m_number_of_sets--;
308 // pick up any existing database with named result sets ..
309 // or one which has no result sets.. yet.
310 BackendPtr b; // null for now
311 std::list<BackendPtr>::const_iterator map_it;
312 map_it = m_backend_list.begin();
313 for (; map_it != m_backend_list.end(); map_it++)
315 BackendPtr tmp = *map_it;
316 if (tmp->m_frontend_databases == databases &&
317 (tmp->m_named_result_sets ||
318 tmp->m_number_of_sets == 0))
324 if (!b) // no backend yet. Must create a new one
328 b = init_backend(databases, package, error_code, addinfo);
331 // did not get a backend (unavailable somehow?)
334 odr.create_searchResponse(
335 apdu_req, error_code, addinfo.c_str());
336 package.response() = apdu;
340 m_sets.erase(req->resultSetName);
341 // sending search to backend
342 Package search_package(b->m_backend_session, package.origin());
344 search_package.copy_filter(package);
346 std::string backend_setname;
347 if (b->m_named_result_sets)
349 backend_setname = std::string(req->resultSetName);
353 backend_setname = "default";
354 req->resultSetName = odr_strdup(odr, backend_setname.c_str());
357 // pick first targets spec and move the databases from it ..
358 std::list<std::string>::const_iterator t_it = b->m_targets.begin();
359 if (t_it != b->m_targets.end())
361 mp::util::set_databases_from_zurl(odr, *t_it,
362 &req->num_databaseNames,
363 &req->databaseNames);
366 if (b->query_encoding.length() &&
367 (req->query->which == Z_Query_type_1 ||
368 req->query->which == Z_Query_type_101))
370 yaz_iconv_t cd = yaz_iconv_open(b->query_encoding.c_str(), "UTF-8");
373 int r = yaz_query_charset_convert_rpnquery_check(
374 req->query->u.type_1, odr, cd);
377 yaz_log(YLOG_LOG, "query could not be converted");
380 *req->replaceIndicator = 1;
382 search_package.request() = yazpp_1::GDU(apdu_req);
384 search_package.move(b->m_route);
386 if (search_package.session().is_closed())
388 package.response() = search_package.response();
389 package.session().close();
392 b->m_number_of_sets++;
394 m_sets[resultSetId] = VirtualDB::Set(b, backend_setname);
395 fixup_package(search_package, b);
396 package.response() = search_package.response();
399 yf::VirtualDB::Frontend::Frontend(Rep *rep)
402 m_is_virtual = false;
405 void yf::VirtualDB::Frontend::close(mp::Package &package)
407 std::list<BackendPtr>::const_iterator b_it;
409 for (b_it = m_backend_list.begin(); b_it != m_backend_list.end(); b_it++)
411 (*b_it)->m_backend_session.close();
412 Package close_package((*b_it)->m_backend_session, package.origin());
413 close_package.copy_filter(package);
414 close_package.move((*b_it)->m_route);
416 m_backend_list.clear();
419 yf::VirtualDB::Frontend::~Frontend()
423 yf::VirtualDB::FrontendPtr yf::VirtualDB::Rep::get_frontend(mp::Package &package)
425 boost::mutex::scoped_lock lock(m_mutex);
427 std::map<mp::Session,yf::VirtualDB::FrontendPtr>::iterator it;
431 it = m_clients.find(package.session());
432 if (it == m_clients.end())
435 if (!it->second->m_in_use)
437 it->second->m_in_use = true;
440 m_cond_session_ready.wait(lock);
442 FrontendPtr f(new Frontend(this));
443 m_clients[package.session()] = f;
448 void yf::VirtualDB::Rep::release_frontend(mp::Package &package)
450 boost::mutex::scoped_lock lock(m_mutex);
451 std::map<mp::Session,yf::VirtualDB::FrontendPtr>::iterator it;
453 it = m_clients.find(package.session());
454 if (it != m_clients.end())
456 if (package.session().is_closed())
458 it->second->close(package);
463 it->second->m_in_use = false;
465 m_cond_session_ready.notify_all();
470 yf::VirtualDB::Set::Set(BackendPtr b, std::string setname)
471 : m_backend(b), m_setname(setname)
476 yf::VirtualDB::Set::Set()
481 yf::VirtualDB::Set::~Set()
485 yf::VirtualDB::Map::Map(std::string database,
486 std::list<std::string> targets, std::string route)
487 : m_dbpattern(database), m_targets(targets), m_route(route)
491 yf::VirtualDB::Map::Map(std::string database,
492 std::string target, std::string route)
493 : m_dbpattern(database), m_route(route)
495 m_targets.push_back(target);
499 yf::VirtualDB::Map::Map()
503 bool yf::VirtualDB::Map::match(const std::string db) const
505 std::string norm_db = mp::util::database_name_normalize(db);
506 if (yaz_match_glob(m_dbpattern.c_str(), norm_db.c_str()))
511 yf::VirtualDB::VirtualDB() : m_p(new VirtualDB::Rep)
513 m_p->pass_vhosts = false;
516 yf::VirtualDB::~VirtualDB() {
519 void yf::VirtualDB::Frontend::fixup_npr_record(ODR odr, Z_NamePlusRecord *npr,
522 if (npr->databaseName)
524 std::string b_database = std::string(npr->databaseName);
526 // consider each of the frontend databases..
527 std::list<std::string>::const_iterator db_it;
528 for (db_it = b->m_frontend_databases.begin();
529 db_it != b->m_frontend_databases.end(); db_it++)
531 // see which target it corresponds to.. (if any)
532 std::list<VirtualDB::Map>::const_iterator map_it =
534 while (map_it != m_p->m_maps.end())
536 if (map_it->match(*db_it))
540 if (map_it != m_p->m_maps.end())
542 std::list<std::string>::const_iterator t
543 = map_it->m_targets.begin();
544 while (t != map_it->m_targets.end())
546 if (*t == b_database)
548 npr->databaseName = odr_strdup(odr, (*db_it).c_str());
556 db_it = b->m_frontend_databases.begin();
557 if (db_it != b->m_frontend_databases.end())
559 std::string database = *db_it;
560 npr->databaseName = odr_strdup(odr, database.c_str());
565 void yf::VirtualDB::Frontend::fixup_npr_records(ODR odr, Z_Records *records,
568 if (records && records->which == Z_Records_DBOSD)
570 Z_NamePlusRecordList *nprlist = records->u.databaseOrSurDiagnostics;
572 for (i = 0; i < nprlist->num_records; i++)
574 fixup_npr_record(odr, nprlist->records[i], b);
579 void yf::VirtualDB::Frontend::fixup_package(mp::Package &p, BackendPtr b)
581 Z_GDU *gdu = p.response().get();
584 if (gdu && gdu->which == Z_GDU_Z3950)
586 Z_APDU *apdu = gdu->u.z3950;
587 if (apdu->which == Z_APDU_presentResponse)
589 fixup_npr_records(odr, apdu->u.presentResponse->records, b);
592 else if (apdu->which == Z_APDU_searchResponse)
594 fixup_npr_records(odr, apdu->u.searchResponse->records, b);
600 void yf::VirtualDB::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
602 Z_PresentRequest *req = apdu_req->u.presentRequest;
603 std::string resultSetId = req->resultSetId;
606 Sets_it sets_it = m_sets.find(resultSetId);
607 if (sets_it == m_sets.end())
610 odr.create_presentResponse(
612 YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
613 resultSetId.c_str());
614 package.response() = apdu;
618 new mp::Session(sets_it->second.m_backend->m_backend_session);
620 // sending present to backend
621 Package present_package(*id, package.origin());
622 present_package.copy_filter(package);
624 req->resultSetId = odr_strdup(odr, sets_it->second.m_setname.c_str());
626 present_package.request() = yazpp_1::GDU(apdu_req);
628 present_package.move(sets_it->second.m_backend->m_route);
630 fixup_package(present_package, sets_it->second.m_backend);
632 if (present_package.session().is_closed())
634 package.response() = present_package.response();
635 package.session().close();
640 package.response() = present_package.response();
645 int yf::VirtualDB::Frontend::relay_apdu(mp::Package &package, Z_APDU *apdu_req)
648 std::list<BackendPtr>::const_iterator map_it;
649 map_it = m_backend_list.begin();
650 for (; map_it != m_backend_list.end(); map_it++)
652 BackendPtr b = *map_it;
654 Package relay_package(b->m_backend_session, package.origin());
655 relay_package.copy_filter(package);
657 relay_package.request() = yazpp_1::GDU(apdu_req);
659 relay_package.move(b->m_route);
660 package.response() = relay_package.response();
666 void yf::VirtualDB::Frontend::scan(mp::Package &package, Z_APDU *apdu_req)
668 Z_ScanRequest *req = apdu_req->u.scanRequest;
672 std::list<std::string> databases;
674 for (i = 0; i<req->num_databaseNames; i++)
675 databases.push_back(req->databaseNames[i]);
678 // pick up any existing backend with a database match
679 std::list<BackendPtr>::const_iterator map_it;
680 map_it = m_backend_list.begin();
681 for (; map_it != m_backend_list.end(); map_it++)
683 BackendPtr tmp = *map_it;
684 if (tmp->m_frontend_databases == databases)
687 if (map_it != m_backend_list.end())
689 if (!b) // no backend yet. Must create a new one
693 b = init_backend(databases, package, error_code, addinfo);
696 // did not get a backend (unavailable somehow?)
698 odr.create_scanResponse(
699 apdu_req, error_code, addinfo.c_str());
700 package.response() = apdu;
705 // sending scan to backend
706 Package scan_package(b->m_backend_session, package.origin());
708 scan_package.copy_filter(package);
710 // pick first targets spec and move the databases from it ..
711 std::list<std::string>::const_iterator t_it = b->m_targets.begin();
712 if (t_it != b->m_targets.end())
714 mp::util::set_databases_from_zurl(odr, *t_it,
715 &req->num_databaseNames,
716 &req->databaseNames);
719 if (b->query_encoding.length())
721 yaz_iconv_t cd = yaz_iconv_open(b->query_encoding.c_str(), "UTF-8");
724 yaz_query_charset_convert_apt(req->termListAndStartPoint, odr, cd);
729 scan_package.request() = yazpp_1::GDU(apdu_req);
731 scan_package.move(b->m_route);
733 if (scan_package.session().is_closed())
735 package.response() = scan_package.response();
736 package.session().close();
739 package.response() = scan_package.response();
743 void yf::VirtualDB::add_map_db2targets(std::string db,
744 std::list<std::string> targets,
747 m_p->m_maps.push_back(
748 VirtualDB::Map(mp::util::database_name_normalize(db), targets, route));
752 void yf::VirtualDB::add_map_db2target(std::string db,
757 m_p->m_maps.push_back(
758 VirtualDB::Map(mp::util::database_name_normalize(db), target, route));
761 void yf::VirtualDB::process(mp::Package &package) const
763 FrontendPtr f = m_p->get_frontend(package);
765 Z_GDU *gdu = package.request().get();
767 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
768 Z_APDU_initRequest && !f->m_is_virtual)
770 Z_InitRequest *req = gdu->u.z3950->u.initRequest;
772 std::list<std::string> vhosts;
773 mp::util::get_vhost_otherinfo(req->otherInfo, vhosts);
775 if (vhosts.size() > 0 && m_p->pass_vhosts)
784 Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
785 Z_InitResponse *resp = apdu->u.initResponse;
788 static const int masks[] = {
791 Z_Options_namedResultSets,
795 for (i = 0; masks[i] != -1; i++)
796 if (ODR_MASK_GET(req->options, masks[i]))
797 ODR_MASK_SET(resp->options, masks[i]);
799 static const int versions[] = {
805 for (i = 0; versions[i] != -1; i++)
806 if (ODR_MASK_GET(req->protocolVersion, versions[i]))
807 ODR_MASK_SET(resp->protocolVersion, versions[i]);
811 *resp->preferredMessageSize = *req->preferredMessageSize;
812 *resp->maximumRecordSize = *req->maximumRecordSize;
814 package.response() = apdu;
815 f->m_is_virtual = true;
818 else if (!f->m_is_virtual)
820 else if (gdu && gdu->which == Z_GDU_Z3950)
822 Z_APDU *apdu = gdu->u.z3950;
823 if (apdu->which == Z_APDU_initRequest)
827 package.response() = odr.create_close(
829 Z_Close_protocolError,
832 package.session().close();
834 else if (apdu->which == Z_APDU_searchRequest)
836 f->search(package, apdu);
838 else if (apdu->which == Z_APDU_presentRequest)
840 f->present(package, apdu);
842 else if (apdu->which == Z_APDU_scanRequest)
844 f->scan(package, apdu);
846 else if (apdu->which == Z_APDU_close)
848 if (f->relay_apdu(package, apdu) == 0)
852 package.response() = odr.create_close(
853 apdu, Z_Close_finished, "virt_db");
855 package.session().close();
862 package.response() = odr.create_close(
863 apdu, Z_Close_protocolError,
864 "unsupported APDU in filter_virt_db");
866 package.session().close();
869 m_p->release_frontend(package);
872 void mp::filter::VirtualDB::configure(const xmlNode * ptr, bool test_only)
874 for (ptr = ptr->children; ptr; ptr = ptr->next)
876 if (ptr->type != XML_ELEMENT_NODE)
878 if (!strcmp((const char *) ptr->name, "pass-vhosts"))
880 m_p->pass_vhosts = mp::xml::get_bool(ptr, false);
882 else if (!strcmp((const char *) ptr->name, "virtual"))
884 std::string query_encoding;
885 std::string database;
886 std::list<std::string> targets;
887 xmlNode *v_node = ptr->children;
888 for (; v_node; v_node = v_node->next)
890 if (v_node->type != XML_ELEMENT_NODE)
893 if (mp::xml::is_element_mp(v_node, "database"))
894 database = mp::xml::get_text(v_node);
895 else if (mp::xml::is_element_mp(v_node, "target"))
896 targets.push_back(mp::xml::get_text(v_node));
897 else if (mp::xml::is_element_mp(v_node, "query-encoding"))
898 targets.push_back(mp::xml::get_text(v_node));
900 throw mp::filter::FilterException
902 + std::string((const char *) v_node->name)
903 + " in virtual section"
906 std::string route = mp::xml::get_route(ptr);
908 VirtualDB::Map vmap(mp::util::database_name_normalize(database),
910 vmap.query_encoding = query_encoding;
911 m_p->m_maps.push_back(vmap);
915 throw mp::filter::FilterException
917 + std::string((const char *) ptr->name)
918 + " in virt_db filter");
923 static mp::filter::Base* filter_creator()
925 return new mp::filter::VirtualDB;
929 struct metaproxy_1_filter_struct metaproxy_1_filter_virt_db = {
940 * c-file-style: "Stroustrup"
941 * indent-tabs-mode: nil
943 * vim: shiftwidth=4 tabstop=8 expandtab