1 /* This file is part of Metaproxy.
2 Copyright (C) 2005-2011 Index Data
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 #include "filter_zoom.hpp"
22 #include <metaproxy/package.hpp>
23 #include <metaproxy/util.hpp>
26 #include <libxslt/xsltutils.h>
27 #include <libxslt/transform.h>
29 #include <boost/thread/mutex.hpp>
30 #include <boost/thread/condition.hpp>
33 #include <yaz/oid_db.h>
34 #include <yaz/diagbib1.h>
37 #include <yaz/querytowrbuf.h>
39 namespace mp = metaproxy_1;
40 namespace yf = mp::filter;
42 namespace metaproxy_1 {
44 struct Zoom::Searchable : boost::noncopyable {
47 std::string query_encoding;
49 std::string request_syntax;
50 std::string element_set;
51 std::string record_encoding;
52 std::string transform_xsl_fname;
55 CCL_bibset ccl_bibset;
59 class Zoom::Backend : boost::noncopyable {
61 friend class Frontend;
63 ZOOM_connection m_connection;
64 ZOOM_resultset m_resultset;
65 std::string m_frontend_database;
67 xsltStylesheetPtr xsp;
69 Backend(SearchablePtr sptr);
71 void connect(std::string zurl, int *error, const char **addinfo);
72 void search_pqf(const char *pqf, Odr_int *hits,
73 int *error, const char **addinfo);
74 void present(Odr_int start, Odr_int number, ZOOM_record *recs,
75 int *error, const char **addinfo);
76 void set_option(const char *name, const char *value);
77 int get_error(const char **addinfo);
79 class Zoom::Frontend : boost::noncopyable {
84 yazpp_1::GDU m_init_gdu;
86 void handle_package(mp::Package &package);
87 void handle_search(mp::Package &package);
88 void handle_present(mp::Package &package);
89 BackendPtr get_backend_from_databases(std::string &database,
91 const char **addinfo);
92 Z_Records *get_records(Odr_int start,
93 Odr_int number_to_present,
96 Odr_int *number_of_records_returned,
97 ODR odr, BackendPtr b,
98 Odr_oid *preferredRecordSyntax,
99 const char *element_set_name);
101 Frontend(Impl *impl);
105 friend class Frontend;
109 void process(metaproxy_1::Package & package);
110 void configure(const xmlNode * ptr, bool test_only);
112 FrontendPtr get_frontend(mp::Package &package);
113 void release_frontend(mp::Package &package);
114 void parse_torus(const xmlNode *ptr);
116 std::list<Zoom::SearchablePtr>m_searchables;
118 std::map<mp::Session, FrontendPtr> m_clients;
119 boost::mutex m_mutex;
120 boost::condition m_cond_session_ready;
126 // define Pimpl wrapper forwarding to Impl
128 yf::Zoom::Zoom() : m_p(new Impl)
133 { // must have a destructor because of boost::scoped_ptr
136 void yf::Zoom::configure(const xmlNode *xmlnode, bool test_only)
138 m_p->configure(xmlnode, test_only);
141 void yf::Zoom::process(mp::Package &package) const
143 m_p->process(package);
147 // define Implementation stuff
149 yf::Zoom::Backend::Backend(SearchablePtr ptr) : sptr(ptr)
151 m_connection = ZOOM_connection_create(0);
156 yf::Zoom::Backend::~Backend()
159 xsltFreeStylesheet(xsp);
160 ZOOM_connection_destroy(m_connection);
161 ZOOM_resultset_destroy(m_resultset);
164 void yf::Zoom::Backend::connect(std::string zurl,
165 int *error, const char **addinfo)
167 ZOOM_connection_connect(m_connection, zurl.c_str(), 0);
168 *error = ZOOM_connection_error(m_connection, 0, addinfo);
171 void yf::Zoom::Backend::search_pqf(const char *pqf, Odr_int *hits,
172 int *error, const char **addinfo)
174 m_resultset = ZOOM_connection_search_pqf(m_connection, pqf);
175 *error = ZOOM_connection_error(m_connection, 0, addinfo);
177 *hits = ZOOM_resultset_size(m_resultset);
182 void yf::Zoom::Backend::present(Odr_int start, Odr_int number,
184 int *error, const char **addinfo)
186 ZOOM_resultset_records(m_resultset, recs, start, number);
187 *error = ZOOM_connection_error(m_connection, 0, addinfo);
190 void yf::Zoom::Backend::set_option(const char *name, const char *value)
192 ZOOM_connection_option_set(m_connection, name, value);
194 ZOOM_resultset_option_set(m_resultset, name, value);
197 int yf::Zoom::Backend::get_error(const char **addinfo)
199 return ZOOM_connection_error(m_connection, 0, addinfo);
202 yf::Zoom::Searchable::Searchable()
205 use_turbomarc = true;
206 ccl_bibset = ccl_qual_mk();
209 yf::Zoom::Searchable::~Searchable()
211 ccl_qual_rm(&ccl_bibset);
214 yf::Zoom::Frontend::Frontend(Impl *impl) :
215 m_p(impl), m_is_virtual(false), m_in_use(true)
219 yf::Zoom::Frontend::~Frontend()
223 yf::Zoom::FrontendPtr yf::Zoom::Impl::get_frontend(mp::Package &package)
225 boost::mutex::scoped_lock lock(m_mutex);
227 std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
231 it = m_clients.find(package.session());
232 if (it == m_clients.end())
235 if (!it->second->m_in_use)
237 it->second->m_in_use = true;
240 m_cond_session_ready.wait(lock);
242 FrontendPtr f(new Frontend(this));
243 m_clients[package.session()] = f;
248 void yf::Zoom::Impl::release_frontend(mp::Package &package)
250 boost::mutex::scoped_lock lock(m_mutex);
251 std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
253 it = m_clients.find(package.session());
254 if (it != m_clients.end())
256 if (package.session().is_closed())
262 it->second->m_in_use = false;
264 m_cond_session_ready.notify_all();
268 yf::Zoom::Impl::Impl()
272 yf::Zoom::Impl::~Impl()
276 void yf::Zoom::Impl::parse_torus(const xmlNode *ptr1)
280 for (ptr1 = ptr1->children; ptr1; ptr1 = ptr1->next)
282 if (ptr1->type != XML_ELEMENT_NODE)
284 if (!strcmp((const char *) ptr1->name, "record"))
286 const xmlNode *ptr2 = ptr1;
287 for (ptr2 = ptr2->children; ptr2; ptr2 = ptr2->next)
289 if (ptr2->type != XML_ELEMENT_NODE)
291 if (!strcmp((const char *) ptr2->name, "layer"))
293 Zoom::SearchablePtr s(new Searchable);
295 const xmlNode *ptr3 = ptr2;
296 for (ptr3 = ptr3->children; ptr3; ptr3 = ptr3->next)
298 if (ptr3->type != XML_ELEMENT_NODE)
300 if (!strcmp((const char *) ptr3->name, "id"))
302 s->database = mp::xml::get_text(ptr3);
304 else if (!strcmp((const char *) ptr3->name, "zurl"))
306 s->target = mp::xml::get_text(ptr3);
308 else if (!strcmp((const char *) ptr3->name, "sru"))
310 s->sru = mp::xml::get_text(ptr3);
312 else if (!strcmp((const char *) ptr3->name,
315 s->query_encoding = mp::xml::get_text(ptr3);
317 else if (!strcmp((const char *) ptr3->name,
320 s->piggyback = mp::xml::get_bool(ptr3, true);
322 else if (!strcmp((const char *) ptr3->name,
325 s->request_syntax = mp::xml::get_text(ptr3);
327 else if (!strcmp((const char *) ptr3->name,
330 s->element_set = mp::xml::get_text(ptr3);
332 else if (!strcmp((const char *) ptr3->name,
335 s->record_encoding = mp::xml::get_text(ptr3);
337 else if (!strcmp((const char *) ptr3->name,
340 s->transform_xsl_fname = mp::xml::get_text(ptr3);
342 else if (!strcmp((const char *) ptr3->name,
345 ; // useTurboMarc is ignored
347 else if (!strncmp((const char *) ptr3->name,
350 std::string value = mp::xml::get_text(ptr3);
351 ccl_qual_fitem(s->ccl_bibset, value.c_str(),
352 (const char *) ptr3->name + 7);
355 if (s->database.length() && s->target.length())
357 yaz_log(YLOG_LOG, "add db=%s target=%s turbomarc=%s",
358 s->database.c_str(), s->target.c_str(),
359 s->use_turbomarc ? "1" : "0");
360 m_searchables.push_back(s);
368 void yf::Zoom::Impl::configure(const xmlNode *ptr, bool test_only)
370 for (ptr = ptr->children; ptr; ptr = ptr->next)
372 if (ptr->type != XML_ELEMENT_NODE)
374 if (!strcmp((const char *) ptr->name, "records"))
378 else if (!strcmp((const char *) ptr->name, "torus"))
381 const struct _xmlAttr *attr;
382 for (attr = ptr->properties; attr; attr = attr->next)
384 if (!strcmp((const char *) attr->name, "url"))
385 url = mp::xml::get_text(attr->children);
387 throw mp::filter::FilterException(
388 "Bad attribute " + std::string((const char *)
391 torus.read_searchables(url);
392 xmlDoc *doc = torus.get_doc();
395 xmlNode *ptr = xmlDocGetRootElement(doc);
401 throw mp::filter::FilterException
403 + std::string((const char *) ptr->name)
404 + " in zoom filter");
409 yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
410 std::string &database, int *error, const char **addinfo)
412 std::list<BackendPtr>::const_iterator map_it;
413 if (m_backend && m_backend->m_frontend_database == database)
416 std::list<Zoom::SearchablePtr>::iterator map_s =
417 m_p->m_searchables.begin();
419 std::string c_db = mp::util::database_name_normalize(database);
421 while (map_s != m_p->m_searchables.end())
423 if (c_db.compare((*map_s)->database) == 0)
427 if (map_s == m_p->m_searchables.end())
429 *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
430 *addinfo = database.c_str();
435 xsltStylesheetPtr xsp = 0;
436 if ((*map_s)->transform_xsl_fname.length())
438 xmlDoc *xsp_doc = xmlParseFile((*map_s)->transform_xsl_fname.c_str());
441 *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
442 *addinfo = "xmlParseFile failed";
446 xsp = xsltParseStylesheetDoc(xsp_doc);
449 *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
450 *addinfo = "xsltParseStylesheetDoc failed";
457 SearchablePtr sptr = *map_s;
461 BackendPtr b(new Backend(sptr));
464 b->m_frontend_database = database;
466 if (sptr->query_encoding.length())
467 b->set_option("rpnCharset", sptr->query_encoding.c_str());
470 if (sptr->sru.length())
472 url = "http://" + sptr->target;
473 b->set_option("sru", sptr->sru.c_str());
478 b->connect(url, error, addinfo);
486 Z_Records *yf::Zoom::Frontend::get_records(Odr_int start,
487 Odr_int number_to_present,
489 const char **addinfo,
490 Odr_int *number_of_records_returned,
493 Odr_oid *preferredRecordSyntax,
494 const char *element_set_name)
496 *number_of_records_returned = 0;
497 Z_Records *records = 0;
498 bool enable_pz2_transform = false;
500 if (start < 0 || number_to_present <= 0)
503 if (number_to_present > 10000)
504 number_to_present = 10000;
506 ZOOM_record *recs = (ZOOM_record *)
507 odr_malloc(odr, number_to_present * sizeof(*recs));
509 char oid_name_str[OID_STR_MAX];
510 const char *syntax_name = 0;
512 if (preferredRecordSyntax)
514 if (!oid_oidcmp(preferredRecordSyntax, yaz_oid_recsyn_xml)
515 && !strcmp(element_set_name, "pz2"))
517 if (b->sptr->request_syntax.length())
519 syntax_name = b->sptr->request_syntax.c_str();
520 enable_pz2_transform = true;
526 yaz_oid_to_string_buf(preferredRecordSyntax, 0, oid_name_str);
530 b->set_option("preferredRecordSyntax", syntax_name);
532 if (enable_pz2_transform)
534 element_set_name = "F";
535 if (b->sptr->element_set.length())
536 element_set_name = b->sptr->element_set.c_str();
539 b->set_option("elementSetName", element_set_name);
541 b->present(start, number_to_present, recs, error, addinfo);
546 for (i = 0; i < number_to_present; i++)
551 { // only return records if no error and at least one record
552 char *odr_database = odr_strdup(odr,
553 b->m_frontend_database.c_str());
554 Z_NamePlusRecordList *npl = (Z_NamePlusRecordList *)
555 odr_malloc(odr, sizeof(*npl));
556 *number_of_records_returned = i;
557 npl->num_records = i;
558 npl->records = (Z_NamePlusRecord **)
559 odr_malloc(odr, i * sizeof(*npl->records));
560 for (i = 0; i < number_to_present; i++)
562 Z_NamePlusRecord *npr = 0;
564 int sur_error = ZOOM_record_error(recs[i], 0 /* msg */,
565 &addinfo, 0 /* diagset */);
569 npr = zget_surrogateDiagRec(odr, odr_database, sur_error,
572 else if (enable_pz2_transform)
574 char rec_type_str[100];
576 strcpy(rec_type_str, b->sptr->use_turbomarc ?
579 // prevent buffer overflow ...
580 if (b->sptr->record_encoding.length() > 0 &&
581 b->sptr->record_encoding.length() <
582 (sizeof(rec_type_str)-20))
584 strcat(rec_type_str, "; charset=");
585 strcat(rec_type_str, b->sptr->record_encoding.c_str());
589 const char *rec_buf = ZOOM_record_get(recs[i], rec_type_str,
591 if (rec_buf && b->xsp)
593 xmlDoc *rec_doc = xmlParseMemory(rec_buf, rec_len);
597 rec_res = xsltApplyStylesheet(b->xsp, rec_doc, 0);
600 xsltSaveResultToString((xmlChar **) &rec_buf, &rec_len,
607 npr = (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
608 npr->databaseName = odr_database;
609 npr->which = Z_NamePlusRecord_databaseRecord;
610 npr->u.databaseRecord =
611 z_ext_record_xml(odr, rec_buf, rec_len);
615 npr = zget_surrogateDiagRec(
617 YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
624 (Z_External *) ZOOM_record_get(recs[i], "ext", 0);
627 npr = (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
628 npr->databaseName = odr_database;
629 npr->which = Z_NamePlusRecord_databaseRecord;
630 npr->u.databaseRecord = ext;
634 npr = zget_surrogateDiagRec(
636 YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
637 "ZOOM_record, type ext");
640 npl->records[i] = npr;
642 records = (Z_Records*) odr_malloc(odr, sizeof(*records));
643 records->which = Z_Records_DBOSD;
644 records->u.databaseOrSurDiagnostics = npl;
650 void yf::Zoom::Frontend::handle_search(mp::Package &package)
652 Z_GDU *gdu = package.request().get();
653 Z_APDU *apdu_req = gdu->u.z3950;
654 Z_APDU *apdu_res = 0;
656 Z_SearchRequest *sr = apdu_req->u.searchRequest;
657 if (sr->num_databaseNames != 1)
659 apdu_res = odr.create_searchResponse(
660 apdu_req, YAZ_BIB1_TOO_MANY_DATABASES_SPECIFIED, 0);
661 package.response() = apdu_res;
666 const char *addinfo = 0;
667 std::string db(sr->databaseNames[0]);
668 BackendPtr b = get_backend_from_databases(db, &error, &addinfo);
672 odr.create_searchResponse(
673 apdu_req, error, addinfo);
674 package.response() = apdu_res;
678 b->set_option("setname", "default");
681 Z_Query *query = sr->query;
685 if (query->which == Z_Query_type_1 || query->which == Z_Query_type_101)
688 pqf_wrbuf = wrbuf_alloc();
689 yaz_rpnquery_to_wrbuf(pqf_wrbuf, query->u.type_1);
691 else if (query->which == Z_Query_type_2)
694 ccl_wrbuf = wrbuf_alloc();
695 wrbuf_write(ccl_wrbuf, (const char *) query->u.type_2->buf,
696 query->u.type_2->len);
698 else if (query->which == Z_Query_type_104 &&
699 query->u.type_104->which == Z_External_CQL)
702 const char *cql = query->u.type_104->u.cql;
703 CQL_parser cp = cql_parser_create();
704 int r = cql_parser_string(cp, cql);
707 cql_parser_destroy(cp);
709 odr.create_searchResponse(apdu_req,
710 YAZ_BIB1_MALFORMED_QUERY,
712 package.response() = apdu_res;
715 struct cql_node *cn = cql_parser_result(cp);
718 r = cql_to_ccl_buf(cn, ccl_buf, sizeof(ccl_buf));
719 yaz_log(YLOG_LOG, "cql_to_ccl_buf returned %d", r);
722 ccl_wrbuf = wrbuf_alloc();
723 wrbuf_puts(ccl_wrbuf, ccl_buf);
725 cql_parser_destroy(cp);
729 odr.create_searchResponse(apdu_req,
730 YAZ_BIB1_MALFORMED_QUERY,
731 "CQL to CCL conversion error");
732 package.response() = apdu_res;
739 odr.create_searchResponse(apdu_req, YAZ_BIB1_QUERY_TYPE_UNSUPP, 0);
740 package.response() = apdu_res;
747 assert(pqf_wrbuf == 0);
749 struct ccl_rpn_node *cn;
750 cn = ccl_find_str(b->sptr->ccl_bibset, wrbuf_cstr(ccl_wrbuf),
752 wrbuf_destroy(ccl_wrbuf);
755 char *addinfo = odr_strdup(odr, ccl_err_msg(cerror));
758 odr.create_searchResponse(apdu_req,
759 YAZ_BIB1_MALFORMED_QUERY,
761 package.response() = apdu_res;
764 pqf_wrbuf = wrbuf_alloc();
765 ccl_pquery(pqf_wrbuf, cn);
770 b->search_pqf(wrbuf_cstr(pqf_wrbuf), &hits, &error, &addinfo);
772 wrbuf_destroy(pqf_wrbuf);
774 const char *element_set_name = 0;
775 Odr_int number_to_present = 0;
777 mp::util::piggyback_sr(sr, hits, number_to_present, &element_set_name);
779 Odr_int number_of_records_returned = 0;
780 Z_Records *records = get_records(
781 0, number_to_present, &error, &addinfo,
782 &number_of_records_returned, odr, b, sr->preferredRecordSyntax,
784 apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
787 apdu_res->u.searchResponse->records = records;
788 apdu_res->u.searchResponse->numberOfRecordsReturned =
789 odr_intdup(odr, number_of_records_returned);
791 apdu_res->u.searchResponse->resultCount = odr_intdup(odr, hits);
792 package.response() = apdu_res;
795 void yf::Zoom::Frontend::handle_present(mp::Package &package)
797 Z_GDU *gdu = package.request().get();
798 Z_APDU *apdu_req = gdu->u.z3950;
799 Z_APDU *apdu_res = 0;
800 Z_PresentRequest *pr = apdu_req->u.presentRequest;
805 package.response() = odr.create_presentResponse(
806 apdu_req, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST, 0);
809 const char *element_set_name = 0;
810 Z_RecordComposition *comp = pr->recordComposition;
811 if (comp && comp->which != Z_RecordComp_simple)
813 package.response() = odr.create_presentResponse(
815 YAZ_BIB1_PRESENT_COMP_SPEC_PARAMETER_UNSUPP, 0);
818 if (comp && comp->u.simple->which == Z_ElementSetNames_generic)
819 element_set_name = comp->u.simple->u.generic;
820 Odr_int number_of_records_returned = 0;
822 const char *addinfo = 0;
823 Z_Records *records = get_records(
824 *pr->resultSetStartPoint - 1, *pr->numberOfRecordsRequested,
825 &error, &addinfo, &number_of_records_returned, odr, m_backend,
826 pr->preferredRecordSyntax, element_set_name);
828 apdu_res = odr.create_presentResponse(apdu_req, error, addinfo);
831 apdu_res->u.presentResponse->records = records;
832 apdu_res->u.presentResponse->numberOfRecordsReturned =
833 odr_intdup(odr, number_of_records_returned);
835 package.response() = apdu_res;
838 void yf::Zoom::Frontend::handle_package(mp::Package &package)
840 Z_GDU *gdu = package.request().get();
843 else if (gdu->which == Z_GDU_Z3950)
845 Z_APDU *apdu_req = gdu->u.z3950;
846 if (apdu_req->which == Z_APDU_initRequest)
849 package.response() = odr.create_close(
851 Z_Close_protocolError,
854 else if (apdu_req->which == Z_APDU_searchRequest)
856 handle_search(package);
858 else if (apdu_req->which == Z_APDU_presentRequest)
860 handle_present(package);
865 package.response() = odr.create_close(
867 Z_Close_protocolError,
868 "zoom filter cannot handle this APDU");
869 package.session().close();
874 package.session().close();
878 void yf::Zoom::Impl::process(mp::Package &package)
880 FrontendPtr f = get_frontend(package);
881 Z_GDU *gdu = package.request().get();
885 f->handle_package(package);
887 else if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
890 Z_InitRequest *req = gdu->u.z3950->u.initRequest;
894 Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
895 Z_InitResponse *resp = apdu->u.initResponse;
898 static const int masks[] = {
903 for (i = 0; masks[i] != -1; i++)
904 if (ODR_MASK_GET(req->options, masks[i]))
905 ODR_MASK_SET(resp->options, masks[i]);
907 static const int versions[] = {
913 for (i = 0; versions[i] != -1; i++)
914 if (ODR_MASK_GET(req->protocolVersion, versions[i]))
915 ODR_MASK_SET(resp->protocolVersion, versions[i]);
919 *resp->preferredMessageSize = *req->preferredMessageSize;
920 *resp->maximumRecordSize = *req->maximumRecordSize;
922 package.response() = apdu;
923 f->m_is_virtual = true;
928 release_frontend(package);
932 static mp::filter::Base* filter_creator()
934 return new mp::filter::Zoom;
938 struct metaproxy_1_filter_struct metaproxy_1_filter_zoom = {
949 * c-file-style: "Stroustrup"
950 * indent-tabs-mode: nil
952 * vim: shiftwidth=4 tabstop=8 expandtab