1 /* This file is part of Metaproxy.
2 Copyright (C) 2005-2012 Index Data
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 #include "filter_sort.hpp"
21 #include <metaproxy/package.hpp>
22 #include <metaproxy/util.hpp>
24 #include <yaz/diagbib1.h>
25 #include <yaz/copy_types.h>
27 #include <yaz/oid_std.h>
28 #include <libxml/xpath.h>
29 #include <libxml/xpathInternals.h>
31 #include <boost/thread/mutex.hpp>
32 #include <boost/thread/condition.hpp>
36 namespace mp = metaproxy_1;
37 namespace yf = mp::filter;
39 namespace metaproxy_1 {
42 friend class Frontend;
46 void process(metaproxy_1::Package & package);
47 void configure(const xmlNode * ptr, bool test_only,
51 std::string m_xpath_expr;
52 std::string m_namespaces;
56 boost::condition m_cond_session_ready;
57 std::map<mp::Session, FrontendPtr> m_clients;
58 FrontendPtr get_frontend(mp::Package &package);
59 void release_frontend(mp::Package &package);
62 friend class RecordList;
63 Z_NamePlusRecord *npr;
65 void get_xpath(xmlDoc *doc, const char *namespaces,
66 const char *expr, bool debug);
67 bool register_namespaces(xmlXPathContextPtr xpathCtx,
70 Record(Z_NamePlusRecord *n, const char *namespaces,
71 const char *expr, bool debug);
73 bool operator < (const Record &rhs);
75 class Sort::RecordList : boost::noncopyable {
77 std::list<Record> npr_list;
79 std::string namespaces;
80 std::string xpath_expr;
83 bool cmp(Odr_oid *syntax);
84 void add(Z_NamePlusRecord *s);
86 Z_NamePlusRecord *get(int i, bool ascending);
88 RecordList(Odr_oid *, std::string namespaces,
89 std::string xpath_expr, bool debug);
92 class Sort::ResultSet : boost::noncopyable {
93 friend class Frontend;
95 std::list<RecordListPtr> record_lists;
97 class Sort::Frontend : boost::noncopyable {
102 std::map<std::string, ResultSetPtr> m_sets;
103 typedef std::map<std::string, ResultSetPtr>::iterator Sets_it;
104 void handle_package(mp::Package &package);
105 void handle_search(mp::Package &package, Z_APDU *apdu_req);
106 void handle_present(mp::Package &package, Z_APDU *apdu_req);
108 void handle_records(mp::Package &package,
114 const char *resultSetId);
116 Frontend(Impl *impl);
122 static void print_xpath_nodes(xmlNodeSetPtr nodes, FILE* output)
129 size = nodes ? nodes->nodeNr : 0;
131 fprintf(output, "Result (%d nodes):\n", size);
132 for (i = 0; i < size; ++i) {
133 assert(nodes->nodeTab[i]);
135 if (nodes->nodeTab[i]->type == XML_NAMESPACE_DECL)
137 xmlNsPtr ns = (xmlNsPtr)nodes->nodeTab[i];
138 cur = (xmlNodePtr)ns->next;
140 fprintf(output, "= namespace \"%s\"=\"%s\" for node %s:%s\n",
141 ns->prefix, ns->href, cur->ns->href, cur->name);
143 fprintf(output, "= namespace \"%s\"=\"%s\" for node %s\n",
144 ns->prefix, ns->href, cur->name);
146 else if (nodes->nodeTab[i]->type == XML_ELEMENT_NODE)
148 cur = nodes->nodeTab[i];
150 fprintf(output, "= element node \"%s:%s\"\n",
151 cur->ns->href, cur->name);
153 fprintf(output, "= element node \"%s\"\n", cur->name);
157 cur = nodes->nodeTab[i];
158 fprintf(output, "= node \"%s\": type %d\n", cur->name, cur->type);
163 bool yf::Sort::Record::register_namespaces(xmlXPathContextPtr xpathCtx,
174 nsListDup = xmlStrdup((const xmlChar *) nsList);
189 next = (xmlChar *) xmlStrchr(next, '=');
199 next = (xmlChar*)xmlStrchr(next, ' ');
203 /* do register namespace */
204 if (xmlXPathRegisterNs(xpathCtx, prefix, href) != 0)
217 void yf::Sort::Record::get_xpath(xmlDoc *doc, const char *namespaces,
218 const char *expr, bool debug)
220 xmlXPathContextPtr xpathCtx = xmlXPathNewContext(doc);
223 register_namespaces(xpathCtx, namespaces);
224 xmlXPathObjectPtr xpathObj =
225 xmlXPathEvalExpression((const xmlChar *) expr, xpathCtx);
228 xmlNodeSetPtr nodes = xpathObj->nodesetval;
230 print_xpath_nodes(nodes, yaz_log_file());
234 for (i = 0; i < nodes->nodeNr; i++)
237 xmlNode *ptr = nodes->nodeTab[i];
238 if (ptr->type == XML_ELEMENT_NODE ||
239 ptr->type == XML_ATTRIBUTE_NODE)
241 content = mp::xml::get_text(ptr->children);
243 else if (ptr->type == XML_TEXT_NODE)
245 content = mp::xml::get_text(ptr);
247 if (content.length())
254 xmlXPathFreeObject(xpathObj);
256 xmlXPathFreeContext(xpathCtx);
260 yf::Sort::Record::Record(Z_NamePlusRecord *n,
261 const char *namespaces,
265 if (npr->which == Z_NamePlusRecord_databaseRecord)
267 Z_External *ext = npr->u.databaseRecord;
269 if (ext->which == Z_External_octet &&
270 !oid_oidcmp(ext->direct_reference, yaz_oid_recsyn_xml))
272 xmlDoc *doc = xmlParseMemory(
273 (const char *) ext->u.octet_aligned->buf,
274 ext->u.octet_aligned->len);
277 get_xpath(doc, namespaces, expr, debug);
284 yf::Sort::Record::~Record()
288 bool yf::Sort::Record::operator < (const Record &rhs)
290 if (strcmp(this->score.c_str(), rhs.score.c_str()) < 0)
295 yf::Sort::RecordList::RecordList(Odr_oid *syntax,
296 std::string a_namespaces,
297 std::string a_xpath_expr,
299 : namespaces(a_namespaces), xpath_expr(a_xpath_expr), debug(a_debug)
303 this->syntax = odr_oiddup(m_odr, syntax);
308 yf::Sort::RecordList::~RecordList()
313 bool yf::Sort::RecordList::cmp(Odr_oid *syntax)
315 if ((!this->syntax && !syntax)
317 (this->syntax && syntax && !oid_oidcmp(this->syntax, syntax)))
322 int yf::Sort::RecordList::size()
324 return npr_list.size();
327 void yf::Sort::RecordList::add(Z_NamePlusRecord *s)
330 Z_NamePlusRecord *npr = yaz_clone_z_NamePlusRecord(s, oi->mem);
331 Record record(npr, namespaces.c_str(), xpath_expr.c_str(), debug);
332 npr_list.push_back(record);
335 Z_NamePlusRecord *yf::Sort::RecordList::get(int pos, bool ascending)
337 std::list<Record>::const_iterator it = npr_list.begin();
340 i = npr_list.size() - pos - 1;
341 for (; it != npr_list.end(); it++, --i)
349 void yf::Sort::RecordList::sort()
354 yf::Sort::Sort() : m_p(new Impl)
359 { // must have a destructor because of boost::scoped_ptr
362 void yf::Sort::configure(const xmlNode *xmlnode, bool test_only,
365 m_p->configure(xmlnode, test_only, path);
368 void yf::Sort::process(mp::Package &package) const
370 m_p->process(package);
374 yf::Sort::Frontend::Frontend(Impl *impl) :
375 m_p(impl), m_is_virtual(false), m_in_use(true)
379 yf::Sort::Frontend::~Frontend()
384 yf::Sort::Impl::Impl() : m_prefetch(20), m_ascending(true), m_debug(false)
388 yf::Sort::Impl::~Impl()
392 yf::Sort::FrontendPtr yf::Sort::Impl::get_frontend(mp::Package &package)
394 boost::mutex::scoped_lock lock(m_mutex);
396 std::map<mp::Session,yf::Sort::FrontendPtr>::iterator it;
400 it = m_clients.find(package.session());
401 if (it == m_clients.end())
404 if (!it->second->m_in_use)
406 it->second->m_in_use = true;
409 m_cond_session_ready.wait(lock);
411 FrontendPtr f(new Frontend(this));
412 m_clients[package.session()] = f;
417 void yf::Sort::Impl::release_frontend(mp::Package &package)
419 boost::mutex::scoped_lock lock(m_mutex);
420 std::map<mp::Session,yf::Sort::FrontendPtr>::iterator it;
422 it = m_clients.find(package.session());
423 if (it != m_clients.end())
425 if (package.session().is_closed())
431 it->second->m_in_use = false;
433 m_cond_session_ready.notify_all();
437 void yf::Sort::Impl::configure(const xmlNode *ptr, bool test_only,
440 for (ptr = ptr->children; ptr; ptr = ptr->next)
442 if (ptr->type != XML_ELEMENT_NODE)
444 if (!strcmp((const char *) ptr->name, "sort"))
446 const struct _xmlAttr *attr;
447 for (attr = ptr->properties; attr; attr = attr->next)
449 if (!strcmp((const char *) attr->name, "prefetch"))
451 m_prefetch = mp::xml::get_int(attr->children, -1);
454 throw mp::filter::FilterException(
455 "Bad or missing value for attribute " +
456 std::string((const char *) attr->name));
459 else if (!strcmp((const char *) attr->name, "xpath"))
461 m_xpath_expr = mp::xml::get_text(attr->children);
463 else if (!strcmp((const char *) attr->name, "namespaces"))
465 m_namespaces = mp::xml::get_text(attr->children);
467 else if (!strcmp((const char *) attr->name, "ascending"))
469 m_ascending = mp::xml::get_bool(attr->children, true);
471 else if (!strcmp((const char *) attr->name, "debug"))
473 m_debug = mp::xml::get_bool(attr->children, false);
476 throw mp::filter::FilterException(
478 std::string((const char *) attr->name));
483 throw mp::filter::FilterException
485 + std::string((const char *) ptr->name)
486 + " in sort filter");
489 if (m_xpath_expr.length() == 0)
491 throw mp::filter::FilterException
492 ("Missing xpath attribute for config element in sort filter");
497 void yf::Sort::Frontend::handle_records(mp::Package &package,
503 const char *resultSetId)
505 if (records && records->which == Z_Records_DBOSD && start_pos == 1)
507 std::list<RecordListPtr>::const_iterator it = s->record_lists.begin();
509 for (; it != s->record_lists.end(); it++)
510 if ((*it)->cmp(syntax))
513 Z_NamePlusRecordList *nprl = records->u.databaseOrSurDiagnostics;
514 int i; // i is number of records fetched in last response
517 RecordListPtr rlp(new RecordList(syntax,
518 m_p->m_namespaces.c_str(),
519 m_p->m_xpath_expr.c_str(),
521 for (i = 0; i < nprl->num_records; i++, pos++)
522 rlp->add(nprl->records[i]);
524 int end_pos = m_p->m_prefetch;
525 if (end_pos > s->hit_count)
526 end_pos = s->hit_count;
527 while (i && pos <= end_pos)
532 Package present_package(package.session(), package.origin());
533 present_package.copy_filter(package);
535 Z_APDU *p_apdu = zget_APDU(odr, Z_APDU_presentRequest);
536 Z_PresentRequest *p_req = p_apdu->u.presentRequest;
538 *p_req->resultSetStartPoint = pos;
539 *p_req->numberOfRecordsRequested = end_pos - pos + 1;
540 p_req->preferredRecordSyntax = syntax;
541 p_req->resultSetId = odr_strdup(odr, resultSetId);
543 present_package.request() = p_apdu;
544 present_package.move();
546 Z_GDU *gdu_res = present_package.response().get();
547 if (gdu_res && gdu_res->which == Z_GDU_Z3950 &&
548 gdu_res->u.z3950->which == Z_APDU_presentResponse)
550 Z_PresentResponse *res = gdu_res->u.z3950->u.presentResponse;
551 Z_Records *records = res->records;
552 if (records && records->which == Z_Records_DBOSD)
554 Z_NamePlusRecordList *nprl =
555 records->u.databaseOrSurDiagnostics;
556 for (i = 0; i < nprl->num_records; i++, pos++)
557 rlp->add(nprl->records[i]);
561 s->record_lists.push_back(rlp);
564 for (i = 0; i < nprl->num_records; i++)
565 nprl->records[i] = rlp->get(i, m_p->m_ascending);
569 void yf::Sort::Frontend::handle_search(mp::Package &package, Z_APDU *apdu_req)
571 Z_SearchRequest *req = apdu_req->u.searchRequest;
572 std::string resultSetId = req->resultSetName;
573 Package b_package(package.session(), package.origin());
577 if (req->preferredRecordSyntax)
578 syntax = odr_oiddup(odr, req->preferredRecordSyntax);
580 b_package.copy_filter(package);
581 Sets_it sets_it = m_sets.find(req->resultSetName);
582 if (sets_it != m_sets.end())
584 // result set already exist
585 // if replace indicator is off: we return diagnostic if
586 // result set already exist.
587 if (*req->replaceIndicator == 0)
590 odr.create_searchResponse(
592 YAZ_BIB1_RESULT_SET_EXISTS_AND_REPLACE_INDICATOR_OFF,
594 package.response() = apdu;
597 m_sets.erase(resultSetId);
599 ResultSetPtr s(new ResultSet);
600 m_sets[resultSetId] = s;
602 Z_GDU *gdu_res = package.response().get();
603 if (gdu_res && gdu_res->which == Z_GDU_Z3950 && gdu_res->u.z3950->which ==
604 Z_APDU_searchResponse)
606 Z_SearchResponse *res = gdu_res->u.z3950->u.searchResponse;
607 s->hit_count = *res->resultCount;
608 handle_records(b_package, apdu_req, res->records, 1, s,
609 syntax, resultSetId.c_str());
610 package.response() = gdu_res;
614 void yf::Sort::Frontend::handle_present(mp::Package &package, Z_APDU *apdu_req)
616 Z_PresentRequest *req = apdu_req->u.presentRequest;
617 std::string resultSetId = req->resultSetId;
618 Package b_package(package.session(), package.origin());
621 Odr_int start = *req->resultSetStartPoint;
623 if (req->preferredRecordSyntax)
624 syntax = odr_oiddup(odr, req->preferredRecordSyntax);
626 b_package.copy_filter(package);
627 Sets_it sets_it = m_sets.find(resultSetId);
628 if (sets_it == m_sets.end())
631 odr.create_presentResponse(
633 YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
634 resultSetId.c_str());
635 package.response() = apdu;
638 ResultSetPtr rset = sets_it->second;
639 std::list<RecordListPtr>::const_iterator it = rset->record_lists.begin();
640 for (; it != rset->record_lists.end(); it++)
641 if ((*it)->cmp(req->preferredRecordSyntax))
643 if (*req->resultSetStartPoint - 1 + *req->numberOfRecordsRequested
647 Z_APDU *p_apdu = zget_APDU(odr, Z_APDU_presentResponse);
648 Z_PresentResponse *p_res = p_apdu->u.presentResponse;
650 *p_res->nextResultSetPosition = *req->resultSetStartPoint +
651 *req->numberOfRecordsRequested;
652 *p_res->numberOfRecordsReturned =
653 *req->numberOfRecordsRequested;
654 p_res->records = (Z_Records *)
655 odr_malloc(odr, sizeof(*p_res->records));
656 p_res->records->which = Z_Records_DBOSD;
657 Z_NamePlusRecordList *nprl = (Z_NamePlusRecordList *)
658 odr_malloc(odr, sizeof(*nprl));
659 p_res->records->u.databaseOrSurDiagnostics = nprl;
660 nprl->num_records = *req->numberOfRecordsRequested;
661 nprl->records = (Z_NamePlusRecord **)
662 odr_malloc(odr, nprl->num_records * sizeof(*nprl->records));
663 for (i = 0; i < nprl->num_records; i++)
665 int pos = i + *req->resultSetStartPoint - 1;
666 nprl->records[i] = (*it)->get(pos, m_p->m_ascending);
668 package.response() = p_apdu;
674 Z_GDU *gdu_res = package.response().get();
675 if (gdu_res && gdu_res->which == Z_GDU_Z3950 && gdu_res->u.z3950->which ==
676 Z_APDU_presentResponse)
678 Z_PresentResponse *res = gdu_res->u.z3950->u.presentResponse;
679 handle_records(b_package, apdu_req, res->records,
680 start, rset, syntax, resultSetId.c_str());
681 package.response() = gdu_res;
685 void yf::Sort::Frontend::handle_package(mp::Package &package)
687 Z_GDU *gdu = package.request().get();
688 if (gdu && gdu->which == Z_GDU_Z3950)
690 Z_APDU *apdu_req = gdu->u.z3950;
691 switch (apdu_req->which)
693 case Z_APDU_searchRequest:
694 handle_search(package, apdu_req);
696 case Z_APDU_presentRequest:
697 handle_present(package, apdu_req);
704 void yf::Sort::Impl::process(mp::Package &package)
706 FrontendPtr f = get_frontend(package);
707 Z_GDU *gdu = package.request().get();
711 f->handle_package(package);
713 else if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
717 f->m_is_virtual = true;
722 release_frontend(package);
726 static mp::filter::Base* filter_creator()
728 return new mp::filter::Sort;
732 struct metaproxy_1_filter_struct metaproxy_1_filter_sort = {
743 * c-file-style: "Stroustrup"
744 * indent-tabs-mode: nil
746 * vim: shiftwidth=4 tabstop=8 expandtab