1 /* This file is part of Metaproxy.
2 Copyright (C) 2005-2013 Index Data
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23 #include <metaproxy/filter.hpp>
24 #include <metaproxy/package.hpp>
26 #include <boost/thread/thread.hpp>
27 #include <boost/thread/mutex.hpp>
28 #include <boost/thread/condition.hpp>
29 #include <boost/shared_ptr.hpp>
31 #include <metaproxy/util.hpp>
32 #include "filter_multi.hpp"
35 #include <yaz/otherinfo.h>
36 #include <yaz/diagbib1.h>
37 #include <yaz/match_glob.h>
44 namespace mp = metaproxy_1;
45 namespace yf = mp::filter;
47 namespace metaproxy_1 {
49 enum multi_merge_type {
53 struct Multi::BackendSet {
56 bool operator < (const BackendSet &k) const;
57 bool operator == (const BackendSet &k) const;
59 struct Multi::ScanTermInfo {
60 std::string m_norm_term;
61 std::string m_display_term;
63 bool operator < (const ScanTermInfo &) const;
64 bool operator == (const ScanTermInfo &) const;
65 Z_Entry *get_entry(ODR odr);
67 struct Multi::FrontendSet {
71 int m_pos; // position for backend (1=first, 2=second,..
72 int m_start; // present request start
73 PresentJob(BackendPtr ptr, int pos) :
74 m_backend(ptr), m_pos(pos), m_start(0) {};
76 FrontendSet(std::string setname);
80 void round_robin(int pos, int number, std::list<PresentJob> &job);
81 void serve_order(int pos, int number, std::list<PresentJob> &job);
83 std::list<BackendSet> m_backend_sets;
84 std::string m_setname;
86 struct Multi::Backend {
88 std::string m_backend_database;
92 void operator() (void); // thread operation
94 struct Multi::Frontend {
99 std::list<BackendPtr> m_backend_list;
100 std::map<std::string,Multi::FrontendSet> m_sets;
102 void multi_move(std::list<BackendPtr> &blist);
103 void init(Package &package, Z_GDU *gdu);
104 void close(Package &package);
105 void search(Package &package, Z_APDU *apdu);
106 void present(Package &package, Z_APDU *apdu);
107 void scan(Package &package, Z_APDU *apdu);
108 void relay_apdu(Package &package, Z_APDU *apdu);
109 void record_diagnostics(Z_Records *records,
110 Z_DiagRecs * &z_diag,
116 std::string m_target_pattern;
120 Map(std::string pattern, std::string route, std::string auth) :
121 m_target_pattern(pattern), m_route(route), m_auth(auth) {};
122 bool match(const std::string target, std::string *ret,
123 std::string *auth) const {
124 if (yaz_match_glob(m_target_pattern.c_str(), target.c_str()))
135 friend struct Frontend;
138 FrontendPtr get_frontend(Package &package);
139 void release_frontend(Package &package);
141 std::list<Multi::Map> m_route_patterns;
142 boost::mutex m_mutex;
143 boost::condition m_cond_session_ready;
144 std::map<mp::Session, FrontendPtr> m_clients;
145 bool m_hide_unavailable;
147 multi_merge_type m_merge_type;
152 yf::Multi::Rep::Rep()
154 m_hide_unavailable = false;
155 m_hide_errors = false;
156 m_merge_type = round_robin;
159 bool yf::Multi::BackendSet::operator < (const BackendSet &k) const
161 return m_count < k.m_count;
164 yf::Multi::Frontend::Frontend(Rep *rep)
170 yf::Multi::Frontend::~Frontend()
174 yf::Multi::FrontendPtr yf::Multi::Rep::get_frontend(mp::Package &package)
176 boost::mutex::scoped_lock lock(m_mutex);
178 std::map<mp::Session,yf::Multi::FrontendPtr>::iterator it;
182 it = m_clients.find(package.session());
183 if (it == m_clients.end())
186 if (!it->second->m_in_use)
188 it->second->m_in_use = true;
191 m_cond_session_ready.wait(lock);
193 FrontendPtr f(new Frontend(this));
194 m_clients[package.session()] = f;
199 void yf::Multi::Rep::release_frontend(mp::Package &package)
201 boost::mutex::scoped_lock lock(m_mutex);
202 std::map<mp::Session,yf::Multi::FrontendPtr>::iterator it;
204 it = m_clients.find(package.session());
205 if (it != m_clients.end())
207 if (package.session().is_closed())
209 it->second->close(package);
214 it->second->m_in_use = false;
216 m_cond_session_ready.notify_all();
220 yf::Multi::FrontendSet::FrontendSet(std::string setname)
226 yf::Multi::FrontendSet::FrontendSet()
231 yf::Multi::FrontendSet::~FrontendSet()
235 yf::Multi::Multi() : m_p(new Multi::Rep)
239 yf::Multi::~Multi() {
243 void yf::Multi::Backend::operator() (void)
245 m_package->move(m_route);
249 void yf::Multi::Frontend::close(mp::Package &package)
251 std::list<BackendPtr>::const_iterator bit;
252 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
256 b->m_package->copy_filter(package);
257 b->m_package->request() = (Z_GDU *) 0;
258 b->m_package->session().close();
259 b->m_package->move(b->m_route);
263 void yf::Multi::Frontend::multi_move(std::list<BackendPtr> &blist)
265 std::list<BackendPtr>::const_iterator bit;
266 boost::thread_group g;
267 for (bit = blist.begin(); bit != blist.end(); bit++)
269 g.add_thread(new boost::thread(**bit));
274 void yf::Multi::FrontendSet::serve_order(int start, int number,
275 std::list<PresentJob> &jobs)
278 for (i = 0; i < number; i++)
280 std::list<BackendSet>::const_iterator bsit;
282 int offset = start + i - 1;
283 for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end();
286 if (offset >= voffset && offset < voffset + bsit->m_count)
288 PresentJob job(bsit->m_backend, offset - voffset + 1);
292 voffset += bsit->m_count;
297 void yf::Multi::FrontendSet::round_robin(int start, int number,
298 std::list<PresentJob> &jobs)
301 std::list<BackendSet>::const_iterator bsit;
302 for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); bsit++)
309 // optimization step!
315 // find min count for each set which is > omin
316 for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); bsit++)
318 if (bsit->m_count > omin)
320 if (no_left == 0 || bsit->m_count < min)
325 if (no_left == 0) // if nothing greater than omin, bail out.
327 int skip = no_left * min;
328 if (p + skip > start) // step gets us "into" present range?
330 // Yes. skip until start.. Rounding off is deliberate!
331 min = (start-p) / no_left;
334 // update positions in each set..
335 std::list<int>::iterator psit = pos.begin();
336 for (psit = pos.begin(); psit != pos.end(); psit++)
340 // skip on each set.. before "present range"..
343 std::list<int>::iterator psit = pos.begin();
344 for (psit = pos.begin(); psit != pos.end(); psit++)
347 omin = min; // update so we consider next class (with higher count)
355 std::list<int>::iterator psit = pos.begin();
356 bsit = m_backend_sets.begin();
358 for (; bsit != m_backend_sets.end(); psit++,bsit++)
360 if (fetched >= number)
365 if (*psit <= bsit->m_count)
369 PresentJob job(bsit->m_backend, *psit);
381 void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu)
383 Z_InitRequest *req = gdu->u.z3950->u.initRequest;
385 std::list<std::string> targets;
387 mp::util::get_vhost_otherinfo(req->otherInfo, targets);
389 if (targets.size() < 1)
395 std::list<std::string>::const_iterator t_it = targets.begin();
396 for (; t_it != targets.end(); t_it++)
399 Backend *b = new Backend;
402 std::list<Multi::Map>::const_iterator it =
403 m_p->m_route_patterns.begin();
404 while (it != m_p->m_route_patterns.end()) {
405 if (it->match(*t_it, &b->m_route, &b->m_auth))
409 // b->m_route = m_p->m_target_route[*t_it];
411 b->m_package = PackagePtr(new Package(s, package.origin()));
413 m_backend_list.push_back(BackendPtr(b));
417 // create init request
418 std::list<BackendPtr>::iterator bit;
419 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
423 Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest);
425 std::list<std::string>vhost_one;
426 vhost_one.push_back(b->m_vhost);
427 mp::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo,
431 Z_InitRequest *breq = init_apdu->u.initRequest;
433 if (b->m_auth.length())
435 breq->idAuthentication =
436 (Z_IdAuthentication *)
437 odr_malloc(odr, sizeof(*breq->idAuthentication));
438 breq->idAuthentication->which = Z_IdAuthentication_open;
439 breq->idAuthentication->u.open = odr_strdup(odr, b->m_auth.c_str());
442 breq->idAuthentication = req->idAuthentication;
444 *breq->preferredMessageSize = *req->preferredMessageSize;
445 *breq->maximumRecordSize = *req->maximumRecordSize;
447 ODR_MASK_SET(breq->options, Z_Options_search);
448 ODR_MASK_SET(breq->options, Z_Options_present);
449 ODR_MASK_SET(breq->options, Z_Options_namedResultSets);
450 ODR_MASK_SET(breq->options, Z_Options_scan);
452 ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_1);
453 ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_2);
454 ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_3);
456 b->m_package->request() = init_apdu;
458 b->m_package->copy_filter(package);
460 multi_move(m_backend_list);
462 // create the frontend init response based on each backend init response
465 Z_APDU *f_apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
466 Z_InitResponse *f_resp = f_apdu->u.initResponse;
468 ODR_MASK_SET(f_resp->options, Z_Options_search);
469 ODR_MASK_SET(f_resp->options, Z_Options_present);
470 ODR_MASK_SET(f_resp->options, Z_Options_namedResultSets);
471 ODR_MASK_SET(f_resp->options, Z_Options_scan);
473 ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_1);
474 ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_2);
475 ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_3);
478 int no_succeeded = 0;
480 Odr_int preferredMessageSize = *req->preferredMessageSize;
481 Odr_int maximumRecordSize = *req->maximumRecordSize;
482 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); )
484 PackagePtr p = (*bit)->m_package;
486 if (p->session().is_closed())
488 // failed. Remove from list and increment number of failed
490 bit = m_backend_list.erase(bit);
493 Z_GDU *gdu = p->response().get();
494 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
498 Z_APDU *b_apdu = gdu->u.z3950;
499 Z_InitResponse *b_resp = b_apdu->u.initResponse;
501 // common options for all backends
502 for (i = 0; i <= Z_Options_stringSchema; i++)
504 if (!ODR_MASK_GET(b_resp->options, i))
505 ODR_MASK_CLEAR(f_resp->options, i);
507 // common protocol version
508 for (i = 0; i <= Z_ProtocolVersion_3; i++)
509 if (!ODR_MASK_GET(b_resp->protocolVersion, i))
510 ODR_MASK_CLEAR(f_resp->protocolVersion, i);
514 if (preferredMessageSize > *b_resp->preferredMessageSize)
515 preferredMessageSize = *b_resp->preferredMessageSize;
516 if (maximumRecordSize > *b_resp->maximumRecordSize)
517 maximumRecordSize = *b_resp->maximumRecordSize;
521 if (!f_resp->userInformationField
522 && b_resp->userInformationField)
523 f_resp->userInformationField = b_resp->userInformationField;
531 *f_resp->preferredMessageSize = preferredMessageSize;
532 *f_resp->maximumRecordSize = maximumRecordSize;
534 if (m_p->m_hide_unavailable)
536 if (no_succeeded == 0)
539 package.session().close();
547 package.session().close();
550 package.response() = f_apdu;
553 void yf::Multi::Frontend::record_diagnostics(Z_Records *records,
554 Z_DiagRecs * &z_diag,
558 // see we get any errors (AKA diagnstics)
561 if (records->which == Z_Records_NSD)
565 z_diag = (Z_DiagRecs *)
566 odr_malloc(odr, sizeof(*z_diag));
567 z_diag->num_diagRecs = 0;
568 z_diag->diagRecs = (Z_DiagRec**)
569 odr_malloc(odr, sizeof(*z_diag->diagRecs));
573 Z_DiagRec **n = (Z_DiagRec **)
575 (1 + z_diag->num_diagRecs) * sizeof(*n));
576 memcpy(n, z_diag->diagRecs, z_diag->num_diagRecs
578 z_diag->diagRecs = n;
580 Z_DiagRec *nr = (Z_DiagRec *) odr_malloc(odr, sizeof(*nr));
581 nr->which = Z_DiagRec_defaultFormat;
582 nr->u.defaultFormat =
583 records->u.nonSurrogateDiagnostic;
584 z_diag->diagRecs[z_diag->num_diagRecs++] = nr;
586 else if (records->which == Z_Records_multipleNSD)
588 Z_DiagRecs * dr =records->u.multipleNonSurDiagnostics;
592 z_diag = (Z_DiagRecs *) odr_malloc(odr, sizeof(*z_diag));
593 z_diag->num_diagRecs = 0;
594 z_diag->diagRecs = 0;
596 Z_DiagRec **n = (Z_DiagRec **)
598 (dr->num_diagRecs + z_diag->num_diagRecs) *
600 if (z_diag->num_diagRecs)
601 memcpy(n, z_diag->diagRecs, z_diag->num_diagRecs * sizeof(*n));
602 memcpy(n + z_diag->num_diagRecs,
603 dr->diagRecs, dr->num_diagRecs * sizeof(*n));
604 z_diag->diagRecs = n;
605 z_diag->num_diagRecs += dr->num_diagRecs;
608 no_successful++; // probably piggyback
611 no_successful++; // no records and no diagnostics
614 void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
616 // create search request
617 Z_SearchRequest *req = apdu_req->u.searchRequest;
619 std::list<BackendPtr>::const_iterator bit;
620 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
622 PackagePtr p = (*bit)->m_package;
623 yazpp_1::GDU gdu1(apdu_req);
625 Z_SearchRequest *req1 = gdu1.get()->u.z3950->u.searchRequest;
627 // they are altered now - to disable piggyback
628 *req1->smallSetUpperBound = 0;
629 *req1->largeSetLowerBound = 1;
630 *req1->mediumSetPresentNumber = 0;
632 if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
633 &req1->num_databaseNames,
634 &req1->databaseNames))
636 req1->num_databaseNames = req->num_databaseNames;
637 req1->databaseNames = req->databaseNames;
640 p->copy_filter(package);
642 multi_move(m_backend_list);
644 // look at each response
645 FrontendSet resultSet(std::string(req->resultSetName));
648 Odr_int result_set_size = 0;
649 Z_DiagRecs *z_diag = 0;
650 int no_successful = 0;
653 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
655 PackagePtr p = (*bit)->m_package;
657 // save closing package for at least one target
658 if (p->session().is_closed())
661 Z_GDU *gdu = p->response().get();
662 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
663 Z_APDU_searchResponse)
665 Z_APDU *b_apdu = gdu->u.z3950;
666 Z_SearchResponse *b_resp = b_apdu->u.searchResponse;
668 record_diagnostics(b_resp->records, z_diag, odr, no_successful);
670 BackendSet backendSet;
671 backendSet.m_backend = *bit;
672 backendSet.m_count = *b_resp->resultCount;
673 result_set_size += *b_resp->resultCount;
674 resultSet.m_backend_sets.push_back(backendSet);
678 Z_APDU *f_apdu = odr.create_searchResponse(apdu_req, 0, 0);
679 Z_SearchResponse *f_resp = f_apdu->u.searchResponse;
681 yaz_log(YLOG_DEBUG, "no_successful=%d is_closed=%s hide_errors=%s",
683 close_p ? "true" : "false",
684 m_p->m_hide_errors ? "true" : "false");
685 *f_resp->resultCount = result_set_size;
686 if (close_p && (no_successful == 0 || !m_p->m_hide_errors))
688 package.session().close();
689 package.response() = close_p->response();
692 if (z_diag && (no_successful == 0 || !m_p->m_hide_errors))
694 f_resp->records = (Z_Records *)
695 odr_malloc(odr, sizeof(*f_resp->records));
696 if (z_diag->num_diagRecs > 1)
698 f_resp->records->which = Z_Records_multipleNSD;
699 f_resp->records->u.multipleNonSurDiagnostics = z_diag;
703 f_resp->records->which = Z_Records_NSD;
704 f_resp->records->u.nonSurrogateDiagnostic =
705 z_diag->diagRecs[0]->u.defaultFormat;
709 m_sets[resultSet.m_setname] = resultSet;
712 mp::util::piggyback(*req->smallSetUpperBound,
713 *req->largeSetLowerBound,
714 *req->mediumSetPresentNumber,
718 Package pp(package.session(), package.origin());
719 if (z_diag == 0 && number > 0)
721 pp.copy_filter(package);
722 Z_APDU *p_apdu = zget_APDU(odr, Z_APDU_presentRequest);
723 Z_PresentRequest *p_req = p_apdu->u.presentRequest;
724 p_req->preferredRecordSyntax = req->preferredRecordSyntax;
725 p_req->resultSetId = req->resultSetName;
726 *p_req->resultSetStartPoint = 1;
727 *p_req->numberOfRecordsRequested = number;
728 pp.request() = p_apdu;
731 if (pp.session().is_closed())
732 package.session().close();
734 Z_GDU *gdu = pp.response().get();
735 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
736 Z_APDU_presentResponse)
738 Z_PresentResponse *p_res = gdu->u.z3950->u.presentResponse;
739 f_resp->records = p_res->records;
740 *f_resp->numberOfRecordsReturned =
741 *p_res->numberOfRecordsReturned;
742 *f_resp->nextResultSetPosition =
743 *p_res->nextResultSetPosition;
747 package.response() = pp.response();
751 package.response() = f_apdu; // in this scope because of p
754 void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
756 // create present request
757 Z_PresentRequest *req = apdu_req->u.presentRequest;
760 it = m_sets.find(std::string(req->resultSetId));
761 if (it == m_sets.end())
765 odr.create_presentResponse(
767 YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
769 package.response() = apdu;
772 std::list<Multi::FrontendSet::PresentJob> jobs;
773 int start = *req->resultSetStartPoint;
774 int number = *req->numberOfRecordsRequested;
776 if (m_p->m_merge_type == round_robin)
777 it->second.round_robin(start, number, jobs);
778 else if (m_p->m_merge_type == serve_order)
779 it->second.serve_order(start, number, jobs);
783 std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
784 for (jit = jobs.begin(); jit != jobs.end(); jit++)
786 yaz_log(YLOG_DEBUG, "job pos=%d", jit->m_pos);
790 std::list<BackendPtr> present_backend_list;
792 std::list<BackendSet>::const_iterator bsit;
793 bsit = it->second.m_backend_sets.begin();
794 for (; bsit != it->second.m_backend_sets.end(); bsit++)
799 std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
800 for (jit = jobs.begin(); jit != jobs.end(); jit++)
802 if (jit->m_backend == bsit->m_backend)
804 if (start == -1 || jit->m_pos < start)
806 if (end == -1 || jit->m_pos > end)
813 std::list<Multi::FrontendSet::PresentJob>::iterator jit;
814 for (jit = jobs.begin(); jit != jobs.end(); jit++)
816 if (jit->m_backend == bsit->m_backend)
818 if (jit->m_pos >= start && jit->m_pos <= end)
819 jit->m_start = start;
823 PackagePtr p = bsit->m_backend->m_package;
825 *req->resultSetStartPoint = start;
826 *req->numberOfRecordsRequested = end - start + 1;
828 p->request() = apdu_req;
829 p->copy_filter(package);
831 present_backend_list.push_back(bsit->m_backend);
834 multi_move(present_backend_list);
836 // look at each response
837 Z_DiagRecs *z_diag = 0;
838 int no_successful = 0;
842 std::list<BackendPtr>::const_iterator pbit = present_backend_list.begin();
843 for (; pbit != present_backend_list.end(); pbit++)
845 PackagePtr p = (*pbit)->m_package;
847 // save closing package for at least one target
848 if (p->session().is_closed())
851 Z_GDU *gdu = p->response().get();
852 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
853 Z_APDU_presentResponse)
855 Z_APDU *b_apdu = gdu->u.z3950;
856 Z_PresentResponse *b_resp = b_apdu->u.presentResponse;
858 record_diagnostics(b_resp->records, z_diag, odr, no_successful);
862 Z_APDU *f_apdu = odr.create_presentResponse(apdu_req, 0, 0);
863 Z_PresentResponse *f_resp = f_apdu->u.presentResponse;
865 if (close_p && (no_successful == 0 || !m_p->m_hide_errors))
867 package.session().close();
868 package.response() = close_p->response();
871 if (z_diag && (no_successful == 0 || !m_p->m_hide_errors))
873 f_resp->records = (Z_Records *)
874 odr_malloc(odr, sizeof(*f_resp->records));
875 if (z_diag->num_diagRecs > 1)
877 f_resp->records->which = Z_Records_multipleNSD;
878 f_resp->records->u.multipleNonSurDiagnostics = z_diag;
882 f_resp->records->which = Z_Records_NSD;
883 f_resp->records->u.nonSurrogateDiagnostic =
884 z_diag->diagRecs[0]->u.defaultFormat;
887 else if (number < 0 || (size_t) number > jobs.size())
890 odr.create_presentResponse(
892 YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
897 f_resp->records = (Z_Records *) odr_malloc(odr, sizeof(Z_Records));
898 Z_Records * records = f_resp->records;
899 records->which = Z_Records_DBOSD;
900 records->u.databaseOrSurDiagnostics =
901 (Z_NamePlusRecordList *)
902 odr_malloc(odr, sizeof(Z_NamePlusRecordList));
903 Z_NamePlusRecordList *nprl = records->u.databaseOrSurDiagnostics;
904 nprl->num_records = jobs.size();
905 nprl->records = (Z_NamePlusRecord**)
906 odr_malloc(odr, sizeof(Z_NamePlusRecord *) * nprl->num_records);
908 std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
909 for (jit = jobs.begin(); jit != jobs.end(); jit++)
911 PackagePtr p = jit->m_backend->m_package;
913 Z_GDU *gdu = p->response().get();
914 Z_APDU *b_apdu = gdu->u.z3950;
915 int inside_pos = jit->m_pos - jit->m_start;
916 Z_Records *records = b_apdu->u.presentResponse->records;
918 if (records && records->which == Z_Records_DBOSD
920 records->u.databaseOrSurDiagnostics->num_records)
922 nprl->records[i] = (Z_NamePlusRecord*)
923 odr_malloc(odr, sizeof(Z_NamePlusRecord));
924 *nprl->records[i] = *records->
925 u.databaseOrSurDiagnostics->records[inside_pos];
926 nprl->records[i]->databaseName =
927 odr_strdup(odr, jit->m_backend->m_vhost.c_str());
931 nprl->num_records = i; // usually same as jobs.size();
932 *f_resp->nextResultSetPosition = start + i;
933 *f_resp->numberOfRecordsReturned = i;
935 package.response() = f_apdu;
938 bool yf::Multi::ScanTermInfo::operator < (const ScanTermInfo &k) const
940 return m_norm_term < k.m_norm_term;
943 bool yf::Multi::ScanTermInfo::operator == (const ScanTermInfo &k) const
945 return m_norm_term == k.m_norm_term;
948 Z_Entry *yf::Multi::ScanTermInfo::get_entry(ODR odr)
950 Z_Entry *e = (Z_Entry *)odr_malloc(odr, sizeof(*e));
951 e->which = Z_Entry_termInfo;
953 t = e->u.termInfo = (Z_TermInfo *) odr_malloc(odr, sizeof(*t));
954 t->suggestedAttributes = 0;
956 t->alternativeTerm = 0;
958 t->otherTermInfo = 0;
959 t->globalOccurrences = odr_intdup(odr, m_count);
961 odr_malloc(odr, sizeof(*t->term));
962 t->term->which = Z_Term_general;
964 t->term->u.general = o = (Odr_oct *)odr_malloc(odr, sizeof(Odr_oct));
966 o->len = o->size = m_norm_term.size();
967 o->buf = (unsigned char *) odr_malloc(odr, o->len);
968 memcpy(o->buf, m_norm_term.c_str(), o->len);
972 void yf::Multi::Frontend::relay_apdu(mp::Package &package, Z_APDU *apdu_req)
974 std::list<BackendPtr>::const_iterator bit;
975 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
977 PackagePtr p = (*bit)->m_package;
980 p->request() = apdu_req;
981 p->copy_filter(package);
983 multi_move(m_backend_list);
984 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
986 PackagePtr p = (*bit)->m_package;
988 if (p->session().is_closed()) // if any backend closes, close frontend
989 package.session().close();
991 package.response() = p->response();
996 void yf::Multi::Frontend::scan(mp::Package &package, Z_APDU *apdu_req)
998 Z_ScanRequest *req = apdu_req->u.scanRequest;
1000 std::list<BackendPtr>::const_iterator bit;
1001 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
1003 PackagePtr p = (*bit)->m_package;
1004 yazpp_1::GDU gdu1(apdu_req);
1006 Z_ScanRequest *req1 = gdu1.get()->u.z3950->u.scanRequest;
1008 if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
1009 &req1->num_databaseNames,
1010 &req1->databaseNames))
1012 req1->num_databaseNames = req->num_databaseNames;
1013 req1->databaseNames = req->databaseNames;
1015 p->request() = gdu1;
1016 p->copy_filter(package);
1018 multi_move(m_backend_list);
1020 ScanTermInfoList entries_before;
1021 ScanTermInfoList entries_after;
1025 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
1027 PackagePtr p = (*bit)->m_package;
1029 if (p->session().is_closed()) // if any backend closes, close frontend
1030 package.session().close();
1032 Z_GDU *gdu = p->response().get();
1033 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
1034 Z_APDU_scanResponse)
1036 Z_ScanResponse *res = gdu->u.z3950->u.scanResponse;
1038 if (res->entries && res->entries->nonsurrogateDiagnostics)
1042 Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 1, 0);
1043 Z_ScanResponse *f_res = f_apdu->u.scanResponse;
1045 f_res->entries->nonsurrogateDiagnostics =
1046 res->entries->nonsurrogateDiagnostics;
1047 f_res->entries->num_nonsurrogateDiagnostics =
1048 res->entries->num_nonsurrogateDiagnostics;
1050 package.response() = f_apdu;
1054 if (res->entries && res->entries->entries)
1056 Z_Entry **entries = res->entries->entries;
1057 int num_entries = res->entries->num_entries;
1059 if (req->preferredPositionInResponse)
1060 position = *req->preferredPositionInResponse;
1061 if (res->positionOfTerm)
1062 position = *res->positionOfTerm;
1066 for (i = 0; i<position-1 && i<num_entries; i++)
1068 Z_Entry *ent = entries[i];
1070 if (ent->which == Z_Entry_termInfo)
1074 Odr_int *occur = ent->u.termInfo->globalOccurrences;
1075 my.m_count = occur ? *occur : 0;
1077 if (ent->u.termInfo->term->which == Z_Term_general)
1079 my.m_norm_term = std::string(
1081 ent->u.termInfo->term->u.general->buf,
1082 ent->u.termInfo->term->u.general->len);
1084 if (my.m_norm_term.length())
1086 ScanTermInfoList::iterator it =
1087 entries_before.begin();
1088 while (it != entries_before.end() && my <*it)
1090 if (it != entries_before.end() && my == *it)
1092 it->m_count += my.m_count;
1096 entries_before.insert(it, my);
1107 for ( ; i<num_entries; i++)
1109 Z_Entry *ent = entries[i];
1111 if (ent->which == Z_Entry_termInfo)
1115 Odr_int *occur = ent->u.termInfo->globalOccurrences;
1116 my.m_count = occur ? *occur : 0;
1118 if (ent->u.termInfo->term->which == Z_Term_general)
1120 my.m_norm_term = std::string(
1122 ent->u.termInfo->term->u.general->buf,
1123 ent->u.termInfo->term->u.general->len);
1125 if (my.m_norm_term.length())
1127 ScanTermInfoList::iterator it =
1128 entries_after.begin();
1129 while (it != entries_after.end() && *it < my)
1131 if (it != entries_after.end() && my == *it)
1133 it->m_count += my.m_count;
1137 entries_after.insert(it, my);
1148 // if any target does not return scan response - return that
1149 package.response() = p->response();
1156 std::cout << "BEFORE\n";
1157 ScanTermInfoList::iterator it = entries_before.begin();
1158 for(; it != entries_before.end(); it++)
1160 std::cout << " " << it->m_norm_term << " " << it->m_count << "\n";
1163 std::cout << "AFTER\n";
1164 it = entries_after.begin();
1165 for(; it != entries_after.end(); it++)
1167 std::cout << " " << it->m_norm_term << " " << it->m_count << "\n";
1174 Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 1, "not implemented");
1175 package.response() = f_apdu;
1180 Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 0, 0);
1181 Z_ScanResponse *resp = f_apdu->u.scanResponse;
1183 int number_returned = *req->numberOfTermsRequested;
1184 int position_returned = *req->preferredPositionInResponse;
1186 resp->entries->num_entries = number_returned;
1187 resp->entries->entries = (Z_Entry**)
1188 odr_malloc(odr, sizeof(Z_Entry*) * number_returned);
1191 int lbefore = entries_before.size();
1192 if (lbefore < position_returned-1)
1193 position_returned = lbefore+1;
1195 ScanTermInfoList::iterator it = entries_before.begin();
1196 for (i = 0; i<position_returned-1 && it != entries_before.end(); i++, it++)
1198 resp->entries->entries[position_returned-2-i] = it->get_entry(odr);
1201 it = entries_after.begin();
1203 if (position_returned <= 0)
1206 i = position_returned-1;
1207 for (; i<number_returned && it != entries_after.end(); i++, it++)
1209 resp->entries->entries[i] = it->get_entry(odr);
1212 number_returned = i;
1214 resp->positionOfTerm = odr_intdup(odr, position_returned);
1215 resp->numberOfEntriesReturned = odr_intdup(odr, number_returned);
1216 resp->entries->num_entries = number_returned;
1218 package.response() = f_apdu;
1223 void yf::Multi::process(mp::Package &package) const
1225 FrontendPtr f = m_p->get_frontend(package);
1227 Z_GDU *gdu = package.request().get();
1229 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
1230 Z_APDU_initRequest && !f->m_is_multi)
1232 f->init(package, gdu);
1234 else if (!f->m_is_multi)
1236 else if (gdu && gdu->which == Z_GDU_Z3950)
1238 Z_APDU *apdu = gdu->u.z3950;
1239 if (apdu->which == Z_APDU_initRequest)
1243 package.response() = odr.create_close(
1245 Z_Close_protocolError,
1248 package.session().close();
1250 else if (apdu->which == Z_APDU_searchRequest)
1252 f->search(package, apdu);
1254 else if (apdu->which == Z_APDU_presentRequest)
1256 f->present(package, apdu);
1258 else if (apdu->which == Z_APDU_scanRequest)
1260 f->scan(package, apdu);
1262 else if (apdu->which == Z_APDU_close)
1264 f->relay_apdu(package, apdu);
1270 package.response() = odr.create_close(
1271 apdu, Z_Close_protocolError,
1272 "unsupported APDU in filter multi");
1274 package.session().close();
1277 m_p->release_frontend(package);
1280 void mp::filter::Multi::configure(const xmlNode * ptr, bool test_only,
1283 for (ptr = ptr->children; ptr; ptr = ptr->next)
1285 if (ptr->type != XML_ELEMENT_NODE)
1287 if (!strcmp((const char *) ptr->name, "target"))
1290 std::string route = mp::xml::get_route(ptr, auth);
1291 std::string target = mp::xml::get_text(ptr);
1292 if (target.length() == 0)
1294 m_p->m_route_patterns.push_back(Multi::Map(target, route, auth));
1296 else if (!strcmp((const char *) ptr->name, "hideunavailable"))
1298 m_p->m_hide_unavailable = true;
1300 else if (!strcmp((const char *) ptr->name, "hideerrors"))
1302 m_p->m_hide_errors = true;
1304 else if (!strcmp((const char *) ptr->name, "mergetype"))
1306 std::string mergetype = mp::xml::get_text(ptr);
1307 if (mergetype == "roundrobin")
1308 m_p->m_merge_type = round_robin;
1309 else if (mergetype == "serveorder")
1310 m_p->m_merge_type = serve_order;
1312 throw mp::filter::FilterException
1313 ("Bad mergetype " + mergetype + " in multi filter");
1318 throw mp::filter::FilterException
1320 + std::string((const char *) ptr->name)
1321 + " in multi filter");
1326 static mp::filter::Base* filter_creator()
1328 return new mp::filter::Multi;
1332 struct metaproxy_1_filter_struct metaproxy_1_filter_multi = {
1343 * c-file-style: "Stroustrup"
1344 * indent-tabs-mode: nil
1346 * vim: shiftwidth=4 tabstop=8 expandtab