1 /* This file is part of Metaproxy.
2 Copyright (C) 2005-2013 Index Data
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 #include "filter_sort.hpp"
21 #include <metaproxy/package.hpp>
22 #include <metaproxy/util.hpp>
24 #include <yaz/diagbib1.h>
25 #include <yaz/copy_types.h>
27 #include <yaz/oid_std.h>
28 #include <libxml/xpath.h>
29 #include <libxml/xpathInternals.h>
31 #include <boost/thread/mutex.hpp>
32 #include <boost/thread/condition.hpp>
36 namespace mp = metaproxy_1;
37 namespace yf = mp::filter;
39 namespace metaproxy_1 {
42 friend class Frontend;
46 void process(metaproxy_1::Package & package);
47 void configure(const xmlNode * ptr, bool test_only,
51 std::string m_xpath_expr;
52 std::string m_namespaces;
56 boost::condition m_cond_session_ready;
57 std::map<mp::Session, FrontendPtr> m_clients;
58 FrontendPtr get_frontend(mp::Package &package);
59 void release_frontend(mp::Package &package);
62 friend class RecordList;
63 Z_NamePlusRecord *npr;
65 void get_xpath(xmlDoc *doc, const char *namespaces,
66 const char *expr, bool debug);
67 bool register_namespaces(xmlXPathContextPtr xpathCtx,
70 Record(Z_NamePlusRecord *n, const char *namespaces,
71 const char *expr, bool debug);
73 bool operator < (const Record &rhs);
75 class Sort::RecordList : boost::noncopyable {
77 std::list<Record> npr_list;
79 std::string namespaces;
80 std::string xpath_expr;
83 bool cmp(Odr_oid *syntax);
84 void add(Z_NamePlusRecord *s);
86 Z_NamePlusRecord *get(int i, bool ascending);
88 RecordList(Odr_oid *, std::string namespaces,
89 std::string xpath_expr, bool debug);
92 class Sort::ResultSet : boost::noncopyable {
93 friend class Frontend;
95 std::list<RecordListPtr> record_lists;
97 class Sort::Frontend : boost::noncopyable {
102 std::map<std::string, ResultSetPtr> m_sets;
103 typedef std::map<std::string, ResultSetPtr>::iterator Sets_it;
104 void handle_package(mp::Package &package);
105 void handle_search(mp::Package &package, Z_APDU *apdu_req);
106 void handle_present(mp::Package &package, Z_APDU *apdu_req);
108 void handle_records(mp::Package &package,
114 Z_RecordComposition *comp,
115 const char *resultSetId);
117 Frontend(Impl *impl);
123 static void print_xpath_nodes(xmlNodeSetPtr nodes, FILE* output)
130 size = nodes ? nodes->nodeNr : 0;
132 fprintf(output, "Result (%d nodes):\n", size);
133 for (i = 0; i < size; ++i) {
134 assert(nodes->nodeTab[i]);
136 if (nodes->nodeTab[i]->type == XML_NAMESPACE_DECL)
138 xmlNsPtr ns = (xmlNsPtr)nodes->nodeTab[i];
139 cur = (xmlNodePtr)ns->next;
141 fprintf(output, "= namespace \"%s\"=\"%s\" for node %s:%s\n",
142 ns->prefix, ns->href, cur->ns->href, cur->name);
144 fprintf(output, "= namespace \"%s\"=\"%s\" for node %s\n",
145 ns->prefix, ns->href, cur->name);
147 else if (nodes->nodeTab[i]->type == XML_ELEMENT_NODE)
149 cur = nodes->nodeTab[i];
151 fprintf(output, "= element node \"%s:%s\"\n",
152 cur->ns->href, cur->name);
154 fprintf(output, "= element node \"%s\"\n", cur->name);
158 cur = nodes->nodeTab[i];
159 fprintf(output, "= node \"%s\": type %d\n", cur->name, cur->type);
164 bool yf::Sort::Record::register_namespaces(xmlXPathContextPtr xpathCtx,
175 nsListDup = xmlStrdup((const xmlChar *) nsList);
190 next = (xmlChar *) xmlStrchr(next, '=');
200 next = (xmlChar*)xmlStrchr(next, ' ');
204 /* do register namespace */
205 if (xmlXPathRegisterNs(xpathCtx, prefix, href) != 0)
218 void yf::Sort::Record::get_xpath(xmlDoc *doc, const char *namespaces,
219 const char *expr, bool debug)
221 xmlXPathContextPtr xpathCtx = xmlXPathNewContext(doc);
224 register_namespaces(xpathCtx, namespaces);
225 xmlXPathObjectPtr xpathObj =
226 xmlXPathEvalExpression((const xmlChar *) expr, xpathCtx);
229 xmlNodeSetPtr nodes = xpathObj->nodesetval;
231 print_xpath_nodes(nodes, yaz_log_file());
235 for (i = 0; i < nodes->nodeNr; i++)
238 xmlNode *ptr = nodes->nodeTab[i];
239 if (ptr->type == XML_ELEMENT_NODE ||
240 ptr->type == XML_ATTRIBUTE_NODE)
242 content = mp::xml::get_text(ptr->children);
244 else if (ptr->type == XML_TEXT_NODE)
246 content = mp::xml::get_text(ptr);
248 if (content.length())
255 xmlXPathFreeObject(xpathObj);
257 xmlXPathFreeContext(xpathCtx);
261 yf::Sort::Record::Record(Z_NamePlusRecord *n,
262 const char *namespaces,
266 if (npr->which == Z_NamePlusRecord_databaseRecord)
268 Z_External *ext = npr->u.databaseRecord;
270 if (ext->which == Z_External_octet &&
271 !oid_oidcmp(ext->direct_reference, yaz_oid_recsyn_xml))
273 xmlDoc *doc = xmlParseMemory(
274 (const char *) ext->u.octet_aligned->buf,
275 ext->u.octet_aligned->len);
278 get_xpath(doc, namespaces, expr, debug);
285 yf::Sort::Record::~Record()
289 bool yf::Sort::Record::operator < (const Record &rhs)
291 if (strcmp(this->score.c_str(), rhs.score.c_str()) < 0)
296 yf::Sort::RecordList::RecordList(Odr_oid *syntax,
297 std::string a_namespaces,
298 std::string a_xpath_expr,
300 : namespaces(a_namespaces), xpath_expr(a_xpath_expr), debug(a_debug)
304 this->syntax = odr_oiddup(m_odr, syntax);
309 yf::Sort::RecordList::~RecordList()
314 bool yf::Sort::RecordList::cmp(Odr_oid *syntax)
316 if ((!this->syntax && !syntax)
318 (this->syntax && syntax && !oid_oidcmp(this->syntax, syntax)))
323 int yf::Sort::RecordList::size()
325 return npr_list.size();
328 void yf::Sort::RecordList::add(Z_NamePlusRecord *s)
331 Z_NamePlusRecord *npr = yaz_clone_z_NamePlusRecord(s, oi->mem);
332 Record record(npr, namespaces.c_str(), xpath_expr.c_str(), debug);
333 npr_list.push_back(record);
336 Z_NamePlusRecord *yf::Sort::RecordList::get(int pos, bool ascending)
338 std::list<Record>::const_iterator it = npr_list.begin();
341 i = npr_list.size() - pos - 1;
342 for (; it != npr_list.end(); it++, --i)
350 void yf::Sort::RecordList::sort()
355 yf::Sort::Sort() : m_p(new Impl)
360 { // must have a destructor because of boost::scoped_ptr
363 void yf::Sort::configure(const xmlNode *xmlnode, bool test_only,
366 m_p->configure(xmlnode, test_only, path);
369 void yf::Sort::process(mp::Package &package) const
371 m_p->process(package);
375 yf::Sort::Frontend::Frontend(Impl *impl) :
376 m_p(impl), m_is_virtual(false), m_in_use(true)
380 yf::Sort::Frontend::~Frontend()
385 yf::Sort::Impl::Impl() : m_prefetch(20), m_ascending(true), m_debug(false)
389 yf::Sort::Impl::~Impl()
393 yf::Sort::FrontendPtr yf::Sort::Impl::get_frontend(mp::Package &package)
395 boost::mutex::scoped_lock lock(m_mutex);
397 std::map<mp::Session,yf::Sort::FrontendPtr>::iterator it;
401 it = m_clients.find(package.session());
402 if (it == m_clients.end())
405 if (!it->second->m_in_use)
407 it->second->m_in_use = true;
410 m_cond_session_ready.wait(lock);
412 FrontendPtr f(new Frontend(this));
413 m_clients[package.session()] = f;
418 void yf::Sort::Impl::release_frontend(mp::Package &package)
420 boost::mutex::scoped_lock lock(m_mutex);
421 std::map<mp::Session,yf::Sort::FrontendPtr>::iterator it;
423 it = m_clients.find(package.session());
424 if (it != m_clients.end())
426 if (package.session().is_closed())
432 it->second->m_in_use = false;
434 m_cond_session_ready.notify_all();
438 void yf::Sort::Impl::configure(const xmlNode *ptr, bool test_only,
441 for (ptr = ptr->children; ptr; ptr = ptr->next)
443 if (ptr->type != XML_ELEMENT_NODE)
445 if (!strcmp((const char *) ptr->name, "sort"))
447 const struct _xmlAttr *attr;
448 for (attr = ptr->properties; attr; attr = attr->next)
450 if (!strcmp((const char *) attr->name, "prefetch"))
452 m_prefetch = mp::xml::get_int(attr->children, -1);
455 throw mp::filter::FilterException(
456 "Bad or missing value for attribute " +
457 std::string((const char *) attr->name));
460 else if (!strcmp((const char *) attr->name, "xpath"))
462 m_xpath_expr = mp::xml::get_text(attr->children);
464 else if (!strcmp((const char *) attr->name, "namespaces"))
466 m_namespaces = mp::xml::get_text(attr->children);
468 else if (!strcmp((const char *) attr->name, "ascending"))
470 m_ascending = mp::xml::get_bool(attr->children, true);
472 else if (!strcmp((const char *) attr->name, "debug"))
474 m_debug = mp::xml::get_bool(attr->children, false);
477 throw mp::filter::FilterException(
479 std::string((const char *) attr->name));
484 throw mp::filter::FilterException
486 + std::string((const char *) ptr->name)
487 + " in sort filter");
490 if (m_xpath_expr.length() == 0)
492 throw mp::filter::FilterException
493 ("Missing xpath attribute for config element in sort filter");
498 void yf::Sort::Frontend::handle_records(mp::Package &package,
504 Z_RecordComposition *comp,
505 const char *resultSetId)
507 if (records && records->which == Z_Records_DBOSD && start_pos == 1)
509 std::list<RecordListPtr>::const_iterator it = s->record_lists.begin();
511 for (; it != s->record_lists.end(); it++)
512 if ((*it)->cmp(syntax))
515 Z_NamePlusRecordList *nprl = records->u.databaseOrSurDiagnostics;
516 int i; // i is number of records fetched in last response
519 RecordListPtr rlp(new RecordList(syntax,
520 m_p->m_namespaces.c_str(),
521 m_p->m_xpath_expr.c_str(),
523 for (i = 0; i < nprl->num_records; i++, pos++)
524 rlp->add(nprl->records[i]);
526 int end_pos = m_p->m_prefetch;
527 if (end_pos > s->hit_count)
528 end_pos = s->hit_count;
529 while (i && pos <= end_pos)
534 Package present_package(package.session(), package.origin());
535 present_package.copy_filter(package);
537 Z_APDU *p_apdu = zget_APDU(odr, Z_APDU_presentRequest);
538 Z_PresentRequest *p_req = p_apdu->u.presentRequest;
540 *p_req->resultSetStartPoint = pos;
541 *p_req->numberOfRecordsRequested = end_pos - pos + 1;
542 p_req->preferredRecordSyntax = syntax;
543 p_req->resultSetId = odr_strdup(odr, resultSetId);
544 p_req->recordComposition = comp;
546 present_package.request() = p_apdu;
547 present_package.move();
549 Z_GDU *gdu_res = present_package.response().get();
550 if (gdu_res && gdu_res->which == Z_GDU_Z3950 &&
551 gdu_res->u.z3950->which == Z_APDU_presentResponse)
553 Z_PresentResponse *res = gdu_res->u.z3950->u.presentResponse;
554 Z_Records *records = res->records;
555 if (records && records->which == Z_Records_DBOSD)
557 Z_NamePlusRecordList *nprl =
558 records->u.databaseOrSurDiagnostics;
559 for (i = 0; i < nprl->num_records; i++, pos++)
560 rlp->add(nprl->records[i]);
564 s->record_lists.push_back(rlp);
567 for (i = 0; i < nprl->num_records; i++)
568 nprl->records[i] = rlp->get(i, m_p->m_ascending);
572 void yf::Sort::Frontend::handle_search(mp::Package &package, Z_APDU *apdu_req)
574 Z_SearchRequest *req = apdu_req->u.searchRequest;
575 std::string resultSetId = req->resultSetName;
576 Package b_package(package.session(), package.origin());
580 if (req->preferredRecordSyntax)
581 syntax = odr_oiddup(odr, req->preferredRecordSyntax);
583 b_package.copy_filter(package);
584 Sets_it sets_it = m_sets.find(req->resultSetName);
585 if (sets_it != m_sets.end())
587 // result set already exist
588 // if replace indicator is off: we return diagnostic if
589 // result set already exist.
590 if (*req->replaceIndicator == 0)
593 odr.create_searchResponse(
595 YAZ_BIB1_RESULT_SET_EXISTS_AND_REPLACE_INDICATOR_OFF,
597 package.response() = apdu;
600 m_sets.erase(resultSetId);
602 ResultSetPtr s(new ResultSet);
603 m_sets[resultSetId] = s;
605 Z_GDU *gdu_res = package.response().get();
606 if (gdu_res && gdu_res->which == Z_GDU_Z3950 && gdu_res->u.z3950->which ==
607 Z_APDU_searchResponse)
609 Z_SearchResponse *res = gdu_res->u.z3950->u.searchResponse;
610 Z_RecordComposition *record_comp =
611 mp::util::piggyback_to_RecordComposition(odr,
612 *res->resultCount, req);
613 s->hit_count = *res->resultCount;
614 handle_records(b_package, apdu_req, res->records, 1, s,
615 syntax, record_comp, resultSetId.c_str());
616 package.response() = gdu_res;
620 void yf::Sort::Frontend::handle_present(mp::Package &package, Z_APDU *apdu_req)
622 Z_PresentRequest *req = apdu_req->u.presentRequest;
623 std::string resultSetId = req->resultSetId;
624 Package b_package(package.session(), package.origin());
627 Odr_int start = *req->resultSetStartPoint;
629 if (req->preferredRecordSyntax)
630 syntax = odr_oiddup(odr, req->preferredRecordSyntax);
632 b_package.copy_filter(package);
633 Sets_it sets_it = m_sets.find(resultSetId);
634 if (sets_it == m_sets.end())
637 odr.create_presentResponse(
639 YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
640 resultSetId.c_str());
641 package.response() = apdu;
644 ResultSetPtr rset = sets_it->second;
645 std::list<RecordListPtr>::const_iterator it = rset->record_lists.begin();
646 for (; it != rset->record_lists.end(); it++)
647 if ((*it)->cmp(req->preferredRecordSyntax))
649 if (*req->resultSetStartPoint - 1 + *req->numberOfRecordsRequested
653 Z_APDU *p_apdu = zget_APDU(odr, Z_APDU_presentResponse);
654 Z_PresentResponse *p_res = p_apdu->u.presentResponse;
656 *p_res->nextResultSetPosition = *req->resultSetStartPoint +
657 *req->numberOfRecordsRequested;
658 *p_res->numberOfRecordsReturned =
659 *req->numberOfRecordsRequested;
660 p_res->records = (Z_Records *)
661 odr_malloc(odr, sizeof(*p_res->records));
662 p_res->records->which = Z_Records_DBOSD;
663 Z_NamePlusRecordList *nprl = (Z_NamePlusRecordList *)
664 odr_malloc(odr, sizeof(*nprl));
665 p_res->records->u.databaseOrSurDiagnostics = nprl;
666 nprl->num_records = *req->numberOfRecordsRequested;
667 nprl->records = (Z_NamePlusRecord **)
668 odr_malloc(odr, nprl->num_records * sizeof(*nprl->records));
669 for (i = 0; i < nprl->num_records; i++)
671 int pos = i + *req->resultSetStartPoint - 1;
672 nprl->records[i] = (*it)->get(pos, m_p->m_ascending);
674 package.response() = p_apdu;
680 Z_GDU *gdu_res = package.response().get();
681 if (gdu_res && gdu_res->which == Z_GDU_Z3950 && gdu_res->u.z3950->which ==
682 Z_APDU_presentResponse)
684 Z_PresentResponse *res = gdu_res->u.z3950->u.presentResponse;
685 handle_records(b_package, apdu_req, res->records,
686 start, rset, syntax, req->recordComposition,
687 resultSetId.c_str());
688 package.response() = gdu_res;
692 void yf::Sort::Frontend::handle_package(mp::Package &package)
694 Z_GDU *gdu = package.request().get();
695 if (gdu && gdu->which == Z_GDU_Z3950)
697 Z_APDU *apdu_req = gdu->u.z3950;
698 switch (apdu_req->which)
700 case Z_APDU_searchRequest:
701 handle_search(package, apdu_req);
703 case Z_APDU_presentRequest:
704 handle_present(package, apdu_req);
711 void yf::Sort::Impl::process(mp::Package &package)
713 FrontendPtr f = get_frontend(package);
714 Z_GDU *gdu = package.request().get();
718 f->handle_package(package);
720 else if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
724 f->m_is_virtual = true;
729 release_frontend(package);
733 static mp::filter::Base* filter_creator()
735 return new mp::filter::Sort;
739 struct metaproxy_1_filter_struct metaproxy_1_filter_sort = {
750 * c-file-style: "Stroustrup"
751 * indent-tabs-mode: nil
753 * vim: shiftwidth=4 tabstop=8 expandtab