76992fa531e299df8cb3bd39ec18f10e5c700768
[metaproxy-moved-to-github.git] / src / filter_zoom.cpp
1 /* This file is part of Metaproxy.
2    Copyright (C) 2005-2011 Index Data
3
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17 */
18
19 #include "config.hpp"
20 #include "filter_zoom.hpp"
21 #include <yaz/zoom.h>
22 #include <yaz/yaz-version.h>
23 #include <yaz/srw.h>
24 #include <metaproxy/package.hpp>
25 #include <metaproxy/util.hpp>
26 #include <metaproxy/xmlutil.hpp>
27 #include "torus.hpp"
28
29 #include <libxslt/xsltutils.h>
30 #include <libxslt/transform.h>
31
32 #include <boost/thread/mutex.hpp>
33 #include <boost/thread/condition.hpp>
34 #include <yaz/ccl_xml.h>
35 #include <yaz/ccl.h>
36 #include <yaz/rpn2cql.h>
37 #include <yaz/rpn2solr.h>
38 #include <yaz/pquery.h>
39 #include <yaz/cql.h>
40 #include <yaz/oid_db.h>
41 #include <yaz/diagbib1.h>
42 #include <yaz/log.h>
43 #include <yaz/zgdu.h>
44 #include <yaz/querytowrbuf.h>
45
46 namespace mp = metaproxy_1;
47 namespace yf = mp::filter;
48
49 namespace metaproxy_1 {
50     namespace filter {
51         struct Zoom::Searchable : boost::noncopyable {
52             std::string authentication;
53             std::string cfAuth;
54             std::string cfProxy;
55             std::string cfSubDb;
56             std::string udb;
57             std::string target;
58             std::string query_encoding;
59             std::string sru;
60             std::string request_syntax;
61             std::string element_set;
62             std::string record_encoding;
63             std::string transform_xsl_fname;
64             std::string urlRecipe;
65             bool use_turbomarc;
66             bool piggyback;
67             CCL_bibset ccl_bibset;
68             Searchable(CCL_bibset base);
69             ~Searchable();
70         };
71         class Zoom::Backend : boost::noncopyable {
72             friend class Impl;
73             friend class Frontend;
74             std::string zurl;
75             ZOOM_connection m_connection;
76             ZOOM_resultset m_resultset;
77             std::string m_frontend_database;
78             SearchablePtr sptr;
79             xsltStylesheetPtr xsp;
80         public:
81             Backend(SearchablePtr sptr);
82             ~Backend();
83             void connect(std::string zurl, int *error, const char **addinfo);
84             void search_pqf(const char *pqf, Odr_int *hits,
85                             int *error, const char **addinfo);
86             void search_cql(const char *cql, Odr_int *hits,
87                             int *error, const char **addinfo);
88             void present(Odr_int start, Odr_int number, ZOOM_record *recs,
89                          int *error, const char **addinfo);
90             void set_option(const char *name, const char *value);
91             const char *get_option(const char *name);
92             void get_zoom_error(int *error, const char **addinfo);
93         };
94         class Zoom::Frontend : boost::noncopyable {
95             friend class Impl;
96             Impl *m_p;
97             bool m_is_virtual;
98             bool m_in_use;
99             yazpp_1::GDU m_init_gdu;
100             BackendPtr m_backend;
101             void handle_package(mp::Package &package);
102             void handle_search(mp::Package &package);
103             void handle_present(mp::Package &package);
104             BackendPtr get_backend_from_databases(std::string &database,
105                                                   int *error,
106                                                   char **addinfo,
107                                                   ODR odr);
108             Z_Records *get_records(Odr_int start,
109                                    Odr_int number_to_present,
110                                    int *error,
111                                    const char **addinfo,
112                                    Odr_int *number_of_records_returned,
113                                    ODR odr, BackendPtr b,
114                                    Odr_oid *preferredRecordSyntax,
115                                    const char *element_set_name);
116         public:
117             Frontend(Impl *impl);
118             ~Frontend();
119         };
120         class Zoom::Impl {
121             friend class Frontend;
122         public:
123             Impl();
124             ~Impl();
125             void process(metaproxy_1::Package & package);
126             void configure(const xmlNode * ptr, bool test_only);
127         private:
128             void configure_local_records(const xmlNode * ptr, bool test_only);
129             FrontendPtr get_frontend(mp::Package &package);
130             void release_frontend(mp::Package &package);
131             SearchablePtr parse_torus_record(const xmlNode *ptr);
132             struct cql_node *convert_cql_fields(struct cql_node *cn, ODR odr);
133             std::map<mp::Session, FrontendPtr> m_clients;            
134             boost::mutex m_mutex;
135             boost::condition m_cond_session_ready;
136             std::string torus_url;
137             std::map<std::string,std::string> fieldmap;
138             std::string xsldir;
139             CCL_bibset bibset;
140             std::string element_transform;
141             std::string element_raw;
142             std::map<std::string,SearchablePtr> s_map;
143         };
144     }
145 }
146
147 // define Pimpl wrapper forwarding to Impl
148  
149 yf::Zoom::Zoom() : m_p(new Impl)
150 {
151 }
152
153 yf::Zoom::~Zoom()
154 {  // must have a destructor because of boost::scoped_ptr
155 }
156
157 void yf::Zoom::configure(const xmlNode *xmlnode, bool test_only)
158 {
159     m_p->configure(xmlnode, test_only);
160 }
161
162 void yf::Zoom::process(mp::Package &package) const
163 {
164     m_p->process(package);
165 }
166
167
168 // define Implementation stuff
169
170 yf::Zoom::Backend::Backend(SearchablePtr ptr) : sptr(ptr)
171 {
172     m_connection = ZOOM_connection_create(0);
173     m_resultset = 0;
174     xsp = 0;
175 }
176
177 yf::Zoom::Backend::~Backend()
178 {
179     if (xsp)
180         xsltFreeStylesheet(xsp);
181     ZOOM_connection_destroy(m_connection);
182     ZOOM_resultset_destroy(m_resultset);
183 }
184
185
186 void yf::Zoom::Backend::get_zoom_error(int *error, const char **addinfo)
187 {
188     const char *msg = 0;
189     *error = ZOOM_connection_error(m_connection, &msg, addinfo);
190     if (*error)
191     {
192         if (*error >= ZOOM_ERROR_CONNECT)
193         {
194             // turn ZOOM diagnostic into a Bib-1 2: with addinfo=zoom err msg
195             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
196             if (addinfo)
197                 *addinfo = msg;
198         }
199     }
200 }
201
202 void yf::Zoom::Backend::connect(std::string zurl,
203                                 int *error, const char **addinfo)
204 {
205     ZOOM_connection_connect(m_connection, zurl.c_str(), 0);
206     get_zoom_error(error, addinfo);
207 }
208
209 void yf::Zoom::Backend::search_pqf(const char *pqf, Odr_int *hits,
210                                    int *error, const char **addinfo)
211 {
212     m_resultset = ZOOM_connection_search_pqf(m_connection, pqf);
213     get_zoom_error(error, addinfo);
214     if (*error == 0)
215         *hits = ZOOM_resultset_size(m_resultset);
216     else
217         *hits = 0;
218 }
219
220 void yf::Zoom::Backend::search_cql(const char *cql, Odr_int *hits,
221                                    int *error, const char **addinfo)
222 {
223     ZOOM_query q = ZOOM_query_create();
224
225     ZOOM_query_cql(q, cql);
226
227     m_resultset = ZOOM_connection_search(m_connection, q);
228     ZOOM_query_destroy(q);
229     get_zoom_error(error, addinfo);
230     if (*error == 0)
231         *hits = ZOOM_resultset_size(m_resultset);
232     else
233         *hits = 0;
234 }
235
236 void yf::Zoom::Backend::present(Odr_int start, Odr_int number,
237                                 ZOOM_record *recs,
238                                 int *error, const char **addinfo)
239 {
240     ZOOM_resultset_records(m_resultset, recs, start, number);
241     get_zoom_error(error, addinfo);
242 }
243
244 void yf::Zoom::Backend::set_option(const char *name, const char *value)
245 {
246     ZOOM_connection_option_set(m_connection, name, value);
247     if (m_resultset)
248         ZOOM_resultset_option_set(m_resultset, name, value);
249 }
250
251 const char *yf::Zoom::Backend::get_option(const char *name)
252 {
253     return ZOOM_connection_option_get(m_connection, name);
254 }
255
256 yf::Zoom::Searchable::Searchable(CCL_bibset base)
257 {
258     piggyback = true;
259     use_turbomarc = true;
260     ccl_bibset = ccl_qual_dup(base);
261 }
262
263 yf::Zoom::Searchable::~Searchable()
264 {
265     ccl_qual_rm(&ccl_bibset);
266 }
267
268 yf::Zoom::Frontend::Frontend(Impl *impl) : 
269     m_p(impl), m_is_virtual(false), m_in_use(true)
270 {
271 }
272
273 yf::Zoom::Frontend::~Frontend()
274 {
275 }
276
277 yf::Zoom::FrontendPtr yf::Zoom::Impl::get_frontend(mp::Package &package)
278 {
279     boost::mutex::scoped_lock lock(m_mutex);
280
281     std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
282     
283     while(true)
284     {
285         it = m_clients.find(package.session());
286         if (it == m_clients.end())
287             break;
288         
289         if (!it->second->m_in_use)
290         {
291             it->second->m_in_use = true;
292             return it->second;
293         }
294         m_cond_session_ready.wait(lock);
295     }
296     FrontendPtr f(new Frontend(this));
297     m_clients[package.session()] = f;
298     f->m_in_use = true;
299     return f;
300 }
301
302 void yf::Zoom::Impl::release_frontend(mp::Package &package)
303 {
304     boost::mutex::scoped_lock lock(m_mutex);
305     std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
306     
307     it = m_clients.find(package.session());
308     if (it != m_clients.end())
309     {
310         if (package.session().is_closed())
311         {
312             m_clients.erase(it);
313         }
314         else
315         {
316             it->second->m_in_use = false;
317         }
318         m_cond_session_ready.notify_all();
319     }
320 }
321
322 yf::Zoom::Impl::Impl() : element_transform("pz2") , element_raw("raw")
323 {
324     bibset = ccl_qual_mk();
325 }
326
327 yf::Zoom::Impl::~Impl()
328
329     ccl_qual_rm(&bibset);
330 }
331
332 yf::Zoom::SearchablePtr yf::Zoom::Impl::parse_torus_record(const xmlNode *ptr)
333 {
334     Zoom::SearchablePtr s(new Searchable(bibset));
335     
336     for (ptr = ptr->children; ptr; ptr = ptr->next)
337     {
338         if (ptr->type != XML_ELEMENT_NODE)
339             continue;
340         if (!strcmp((const char *) ptr->name, "layer"))
341             ptr = ptr->children;
342         else if (!strcmp((const char *) ptr->name,
343                          "authentication"))
344         {
345             s->authentication = mp::xml::get_text(ptr);
346         }
347         else if (!strcmp((const char *) ptr->name,
348                          "cfAuth"))
349         {
350             s->cfAuth = mp::xml::get_text(ptr);
351         } 
352         else if (!strcmp((const char *) ptr->name,
353                          "cfProxy"))
354         {
355             s->cfProxy = mp::xml::get_text(ptr);
356         }  
357         else if (!strcmp((const char *) ptr->name,
358                          "cfSubDb"))
359         {
360             s->cfSubDb = mp::xml::get_text(ptr);
361         }  
362         else if (!strcmp((const char *) ptr->name, "udb"))
363         {
364             s->udb = mp::xml::get_text(ptr);
365         }
366         else if (!strcmp((const char *) ptr->name, "zurl"))
367         {
368             s->target = mp::xml::get_text(ptr);
369         }
370         else if (!strcmp((const char *) ptr->name, "sru"))
371         {
372             s->sru = mp::xml::get_text(ptr);
373         }
374         else if (!strcmp((const char *) ptr->name,
375                          "queryEncoding"))
376         {
377             s->query_encoding = mp::xml::get_text(ptr);
378         }
379         else if (!strcmp((const char *) ptr->name,
380                          "piggyback"))
381         {
382             s->piggyback = mp::xml::get_bool(ptr, true);
383         }
384         else if (!strcmp((const char *) ptr->name,
385                          "requestSyntax"))
386         {
387             s->request_syntax = mp::xml::get_text(ptr);
388         }
389         else if (!strcmp((const char *) ptr->name,
390                          "elementSet"))
391         {
392             s->element_set = mp::xml::get_text(ptr);
393         }
394         else if (!strcmp((const char *) ptr->name,
395                          "recordEncoding"))
396         {
397             s->record_encoding = mp::xml::get_text(ptr);
398         }
399         else if (!strcmp((const char *) ptr->name,
400                          "transform"))
401         {
402             s->transform_xsl_fname = mp::xml::get_text(ptr);
403         }
404         else if (!strcmp((const char *) ptr->name,
405                          "urlRecipe"))
406         {
407             s->urlRecipe = mp::xml::get_text(ptr);
408         }
409         else if (!strcmp((const char *) ptr->name,
410                          "useTurboMarc"))
411         {
412             ; // useTurboMarc is ignored
413         }
414         else if (!strncmp((const char *) ptr->name,
415                           "cclmap_", 7))
416         {
417             std::string value = mp::xml::get_text(ptr);
418             ccl_qual_fitem(s->ccl_bibset, value.c_str(),
419                            (const char *) ptr->name + 7);
420         }
421     }
422     return s;
423 }
424
425 void yf::Zoom::Impl::configure_local_records(const xmlNode *ptr, bool test_only)
426 {
427     while (ptr && ptr->type != XML_ELEMENT_NODE)
428         ptr = ptr->next;
429     
430     if (ptr)
431     {
432         if (!strcmp((const char *) ptr->name, "records"))
433         {
434             for (ptr = ptr->children; ptr; ptr = ptr->next)
435             {
436                 if (ptr->type != XML_ELEMENT_NODE)
437                     continue;
438                 if (!strcmp((const char *) ptr->name, "record"))
439                 {
440                     SearchablePtr s = parse_torus_record(ptr);
441                     if (s)
442                     {
443                         std::string udb = s->udb;
444                         if (udb.length())
445                             s_map[s->udb] = s;
446                         else
447                         {
448                             throw mp::filter::FilterException
449                                 ("No udb for local torus record");
450                         }
451                     }
452                 }
453                 else
454                 {
455                     throw mp::filter::FilterException
456                         ("Bad element " 
457                          + std::string((const char *) ptr->name)
458                          + " in zoom filter inside element "
459                          "<torus><records>");
460                 }
461             }
462         }
463         else
464         {
465             throw mp::filter::FilterException
466                 ("Bad element " 
467                  + std::string((const char *) ptr->name)
468                  + " in zoom filter inside element <torus>");
469         }
470     }
471 }
472
473 void yf::Zoom::Impl::configure(const xmlNode *ptr, bool test_only)
474 {
475     for (ptr = ptr->children; ptr; ptr = ptr->next)
476     {
477         if (ptr->type != XML_ELEMENT_NODE)
478             continue;
479         else if (!strcmp((const char *) ptr->name, "torus"))
480         {
481             const struct _xmlAttr *attr;
482             for (attr = ptr->properties; attr; attr = attr->next)
483             {
484                 if (!strcmp((const char *) attr->name, "url"))
485                     torus_url = mp::xml::get_text(attr->children);
486                 else if (!strcmp((const char *) attr->name, "xsldir"))
487                     xsldir = mp::xml::get_text(attr->children);
488                 else if (!strcmp((const char *) attr->name, "element_transform"))
489                     element_transform = mp::xml::get_text(attr->children);
490                 else if (!strcmp((const char *) attr->name, "element_raw"))
491                     element_raw = mp::xml::get_text(attr->children);
492                 else
493                     throw mp::filter::FilterException(
494                         "Bad attribute " + std::string((const char *)
495                                                        attr->name));
496             }
497             configure_local_records(ptr->children, test_only);
498         }
499         else if (!strcmp((const char *) ptr->name, "cclmap"))
500         {
501             const char *addinfo = 0;
502             ccl_xml_config(bibset, ptr, &addinfo);
503         }
504         else if (!strcmp((const char *) ptr->name, "fieldmap"))
505         {
506             const struct _xmlAttr *attr;
507             std::string ccl_field;
508             std::string cql_field;
509             for (attr = ptr->properties; attr; attr = attr->next)
510             {
511                 if (!strcmp((const char *) attr->name, "ccl"))
512                     ccl_field = mp::xml::get_text(attr->children);
513                 else if (!strcmp((const char *) attr->name, "cql"))
514                     cql_field = mp::xml::get_text(attr->children);
515                 else
516                     throw mp::filter::FilterException(
517                         "Bad attribute " + std::string((const char *)
518                                                        attr->name));
519             }
520             if (cql_field.length())
521                 fieldmap[cql_field] = ccl_field;
522         }
523         else
524         {
525             throw mp::filter::FilterException
526                 ("Bad element " 
527                  + std::string((const char *) ptr->name)
528                  + " in zoom filter");
529         }
530     }
531 }
532
533 yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
534     std::string &database, int *error, char **addinfo, ODR odr)
535 {
536     std::list<BackendPtr>::const_iterator map_it;
537     if (m_backend && m_backend->m_frontend_database == database)
538         return m_backend;
539
540     const char *sru_proxy = 0;
541     std::string db_args;
542     std::string torus_db;
543     size_t db_arg_pos = database.find(',');
544     if (db_arg_pos != std::string::npos)
545     {
546         torus_db = database.substr(0, db_arg_pos);
547         db_args = database.substr(db_arg_pos + 1);
548     }
549     else
550         torus_db = database;
551  
552     SearchablePtr sptr;
553
554     std::map<std::string,SearchablePtr>::iterator it;
555     it = m_p->s_map.find(torus_db);
556     if (it != m_p->s_map.end())
557         sptr = it->second;
558     else
559     {
560         xmlDoc *doc = mp::get_searchable(m_p->torus_url, torus_db);
561         if (!doc)
562         {
563             *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
564             *addinfo = odr_strdup(odr, database.c_str());
565             BackendPtr b;
566             return b;
567         }
568         const xmlNode *ptr = xmlDocGetRootElement(doc);
569         if (ptr)
570         {   // presumably ptr is a records element node
571             // parse first record in document
572             for (ptr = ptr->children; ptr; ptr = ptr->next)
573             {
574                 if (ptr->type == XML_ELEMENT_NODE
575                     && !strcmp((const char *) ptr->name, "record"))
576                 {
577                     sptr = m_p->parse_torus_record(ptr);
578                     break;
579                 }
580             }
581         }
582         xmlFreeDoc(doc);
583     }
584
585     if (!sptr)
586     {
587         *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
588         *addinfo = odr_strdup(odr, database.c_str());
589         BackendPtr b;
590         return b;
591     }
592         
593     xsltStylesheetPtr xsp = 0;
594     if (sptr->transform_xsl_fname.length())
595     {
596         std::string fname;
597
598         if (m_p->xsldir.length()) 
599             fname = m_p->xsldir + "/" + sptr->transform_xsl_fname;
600         else
601             fname = sptr->transform_xsl_fname;
602         xmlDoc *xsp_doc = xmlParseFile(fname.c_str());
603         if (!xsp_doc)
604         {
605             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
606             *addinfo = (char *) odr_malloc(odr, 40 + strlen(fname.c_str()));
607             sprintf(*addinfo, "xmlParseFile failed. File %s", fname.c_str());
608             BackendPtr b;
609             return b;
610         }
611         xsp = xsltParseStylesheetDoc(xsp_doc);
612         if (!xsp)
613         {
614             *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
615             *addinfo = odr_strdup(odr, "xsltParseStylesheetDoc failed");
616             BackendPtr b;
617             xmlFreeDoc(xsp_doc);
618             return b;
619         }
620     }
621
622     m_backend.reset();
623
624     BackendPtr b(new Backend(sptr));
625
626     b->xsp = xsp;
627     b->m_frontend_database = database;
628     std::string authentication = sptr->authentication;
629         
630     b->set_option("timeout", "40");
631
632     if (sptr->query_encoding.length())
633         b->set_option("rpnCharset", sptr->query_encoding.c_str());
634
635     if (sptr->cfAuth.length())
636     {
637         // A CF target
638         b->set_option("user", sptr->cfAuth.c_str());
639         if (db_args.length() == 0)
640         {
641             if (authentication.length())
642             {
643                 // no database (auth) args specified already.. and the
644                 // Torus authentication has it.. Generate the args that CF
645                 // understands..
646                 size_t found = authentication.find('/');
647                 if (found != std::string::npos)
648                 {
649                     db_args += "user=" + mp::util::uri_encode(authentication.substr(0, found))
650                         + "&password=" + mp::util::uri_encode(authentication.substr(found+1));
651                 }
652                 else
653                     db_args += "user=" + mp::util::uri_encode(authentication);
654             }
655             if (sptr->cfProxy.length())
656             {
657                 if (db_args.length())
658                     db_args += "&";
659                 db_args += "proxy=" + mp::util::uri_encode(sptr->cfProxy);
660             }
661         }
662         if (sptr->cfSubDb.length())
663         {
664             if (db_args.length())
665                 db_args += "&";
666             db_args += "subdatabase=" + mp::util::uri_encode(sptr->cfSubDb);
667         }
668     }
669     else
670     {
671         // A non-CF target
672         if (db_args.length())
673         {
674             // user has specified backend authentication
675             const char *param_user = 0;
676             const char *param_password = 0;
677             char **names;
678             char **values;
679             int i;
680             int no_parms = yaz_uri_to_array(db_args.c_str(),
681                                             odr, &names, &values);
682             for (i = 0; i < no_parms; i++)
683             {
684                 const char *name = names[i];
685                 const char *value = values[i];
686                 if (!strcmp(name, "user"))
687                     param_user = value;
688                 else if (!strcmp(name, "password"))
689                     param_password = value;
690                 else if (!strcmp(name, "proxy"))
691                     sru_proxy = value;
692                 else
693                 {
694                     BackendPtr notfound;
695                     char *msg = (char*) odr_malloc(odr, strlen(name) + 30);
696                     *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
697                     sprintf(msg, "Bad database argument: %s", name);
698                     *addinfo = msg;
699                     return notfound;
700                 }
701             }
702             if (param_user && param_password)
703             {
704                 char *auth = (char*) odr_malloc(
705                     odr, strlen(param_user) + strlen(param_password) + 2);
706                 strcpy(auth, param_user);
707                 strcat(auth, "/");
708                 strcat(auth, param_password);
709                 b->set_option("user", auth);
710             }
711             db_args.clear(); // no arguments to be passed (non-CF)
712         }
713         else
714         {
715             // use authentication from Torus, if given
716             if (authentication.length())
717                 b->set_option("user", authentication.c_str());
718         }
719     }
720
721     if (sru_proxy)
722         b->set_option("proxy", sru_proxy);
723
724     std::string url;
725     if (sptr->sru.length())
726     {
727         url = "http://" + sptr->target;
728         b->set_option("sru", sptr->sru.c_str());
729     }
730     else
731     {
732         url = sptr->target;
733     }
734     if (db_args.length())
735         url += "," + db_args;
736     yaz_log(YLOG_LOG, "url=%s", url.c_str());
737     const char *addinfo_c = 0;
738     b->connect(url, error, &addinfo_c);
739     if (addinfo_c)
740         *addinfo = odr_strdup(odr, addinfo_c);
741     if (*error == 0)
742     {
743         m_backend = b;
744     }
745     return b;
746 }
747
748 Z_Records *yf::Zoom::Frontend::get_records(Odr_int start,
749                                            Odr_int number_to_present,
750                                            int *error,
751                                            const char **addinfo,
752                                            Odr_int *number_of_records_returned,
753                                            ODR odr,
754                                            BackendPtr b,
755                                            Odr_oid *preferredRecordSyntax,
756                                            const char *element_set_name)
757 {
758     *number_of_records_returned = 0;
759     Z_Records *records = 0;
760     bool enable_pz2_retrieval = false; // whether target profile is used
761     bool enable_pz2_transform = false; // whether XSLT is used as well
762     bool assume_marc8_charset = false;
763
764     if (start < 0 || number_to_present <= 0)
765         return records;
766     
767     if (number_to_present > 10000)
768         number_to_present = 10000;
769     
770     ZOOM_record *recs = (ZOOM_record *)
771         odr_malloc(odr, number_to_present * sizeof(*recs));
772
773     char oid_name_str[OID_STR_MAX];
774     const char *syntax_name = 0;
775     
776     if (preferredRecordSyntax &&
777         !oid_oidcmp(preferredRecordSyntax, yaz_oid_recsyn_xml)
778         && element_set_name)
779     {
780         if (!strcmp(element_set_name, m_p->element_transform.c_str()))
781         {
782             enable_pz2_retrieval = true;
783             enable_pz2_transform = true;
784         }
785         else if (!strcmp(element_set_name, m_p->element_raw.c_str()))
786         {
787             enable_pz2_retrieval = true;
788         }
789     }
790     
791     if (enable_pz2_retrieval)
792     {
793         if (b->sptr->request_syntax.length())
794         {
795             syntax_name = b->sptr->request_syntax.c_str();
796             const Odr_oid *syntax_oid = 
797                 yaz_string_to_oid(yaz_oid_std(), CLASS_RECSYN, syntax_name);
798             if (!oid_oidcmp(syntax_oid, yaz_oid_recsyn_usmarc)
799                 || !oid_oidcmp(syntax_oid, yaz_oid_recsyn_opac))
800                 assume_marc8_charset = true;
801         }
802     }
803     else if (preferredRecordSyntax)
804         syntax_name =
805             yaz_oid_to_string_buf(preferredRecordSyntax, 0, oid_name_str);
806
807     b->set_option("preferredRecordSyntax", syntax_name);
808
809     if (enable_pz2_retrieval)
810     {
811         element_set_name = 0;
812         if (b->sptr->element_set.length())
813             element_set_name = b->sptr->element_set.c_str();
814     }
815
816     b->set_option("elementSetName", element_set_name);
817
818     b->present(start, number_to_present, recs, error, addinfo);
819
820     Odr_int i = 0;
821     if (!*error)
822     {
823         for (i = 0; i < number_to_present; i++)
824             if (!recs[i])
825                 break;
826     }
827     if (i > 0)
828     {  // only return records if no error and at least one record
829         char *odr_database = odr_strdup(odr,
830                                         b->m_frontend_database.c_str());
831         Z_NamePlusRecordList *npl = (Z_NamePlusRecordList *)
832             odr_malloc(odr, sizeof(*npl));
833         *number_of_records_returned = i;
834         npl->num_records = i;
835         npl->records = (Z_NamePlusRecord **)
836             odr_malloc(odr, i * sizeof(*npl->records));
837         for (i = 0; i < number_to_present; i++)
838         {
839             Z_NamePlusRecord *npr = 0;
840             const char *addinfo;
841             int sur_error = ZOOM_record_error(recs[i], 0 /* msg */,
842                                               &addinfo, 0 /* diagset */);
843                 
844             if (sur_error)
845             {
846                 npr = zget_surrogateDiagRec(odr, odr_database, sur_error,
847                                             addinfo);
848             }
849             else if (enable_pz2_retrieval)
850             {
851                 char rec_type_str[100];
852                 const char *record_encoding = 0;
853
854                 if (b->sptr->record_encoding.length())
855                     record_encoding = b->sptr->record_encoding.c_str();
856                 else if (assume_marc8_charset)
857                     record_encoding = "marc8";
858
859                 strcpy(rec_type_str, b->sptr->use_turbomarc ? "txml" : "xml");
860                 if (record_encoding)
861                 {
862                     strcat(rec_type_str, "; charset=");
863                     strcat(rec_type_str, record_encoding);
864                 }
865                 
866                 int rec_len;
867                 xmlChar *xmlrec_buf = 0;
868                 const char *rec_buf = ZOOM_record_get(recs[i], rec_type_str,
869                                                       &rec_len);
870                 if (rec_buf && b->xsp && enable_pz2_transform)
871                 {
872                     xmlDoc *rec_doc = xmlParseMemory(rec_buf, rec_len);
873                     if (rec_doc)
874                     { 
875                         xmlDoc *rec_res;
876                         rec_res = xsltApplyStylesheet(b->xsp, rec_doc, 0);
877
878                         if (rec_res)
879                             xsltSaveResultToString(&xmlrec_buf, &rec_len,
880                                                    rec_res, b->xsp);
881                         rec_buf = (const char *) xmlrec_buf;
882                         xmlFreeDoc(rec_doc);
883                         xmlFreeDoc(rec_res);
884                     }
885                 }
886
887                 if (rec_buf)
888                 {
889                     xmlDoc *doc = xmlParseMemory(rec_buf, rec_len);
890                     std::string res = 
891                         mp::xml::url_recipe_handle(doc, b->sptr->urlRecipe);
892                     if (res.length())
893                     {
894                         xmlNode *ptr = xmlDocGetRootElement(doc);
895                         while (ptr && ptr->type != XML_ELEMENT_NODE)
896                             ptr = ptr->next;
897                         xmlNode *c = 
898                             xmlNewChild(ptr, 0, BAD_CAST "generated-url", 0);
899                         xmlNode * t = xmlNewText(BAD_CAST res.c_str());
900                         xmlAddChild(c, t);
901
902                         if (xmlrec_buf)
903                             xmlFree(xmlrec_buf);
904
905                         xmlDocDumpMemory(doc, &xmlrec_buf, &rec_len);
906                         rec_buf = (const char *) xmlrec_buf;
907                     }
908                     xmlFreeDoc(doc);
909                 }
910                 if (rec_buf)
911                 {
912                     npr = (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
913                     npr->databaseName = odr_database;
914                     npr->which = Z_NamePlusRecord_databaseRecord;
915                     npr->u.databaseRecord =
916                         z_ext_record_xml(odr, rec_buf, rec_len);
917                 }
918                 else
919                 {
920                     npr = zget_surrogateDiagRec(
921                         odr, odr_database, 
922                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
923                         rec_type_str);
924                 }
925                 if (xmlrec_buf)
926                     xmlFree(xmlrec_buf);
927             }
928             else
929             {
930                 Z_External *ext =
931                     (Z_External *) ZOOM_record_get(recs[i], "ext", 0);
932                 if (ext)
933                 {
934                     npr = (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
935                     npr->databaseName = odr_database;
936                     npr->which = Z_NamePlusRecord_databaseRecord;
937                     npr->u.databaseRecord = ext;
938                 }
939                 else
940                 {
941                     npr = zget_surrogateDiagRec(
942                         odr, odr_database, 
943                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
944                         "ZOOM_record, type ext");
945                 }
946             }
947             npl->records[i] = npr;
948         }
949         records = (Z_Records*) odr_malloc(odr, sizeof(*records));
950         records->which = Z_Records_DBOSD;
951         records->u.databaseOrSurDiagnostics = npl;
952     }
953     return records;
954 }
955     
956 struct cql_node *yf::Zoom::Impl::convert_cql_fields(struct cql_node *cn,
957                                                     ODR odr)
958 {
959     struct cql_node *r = 0;
960     if (!cn)
961         return 0;
962     switch (cn->which)
963     {
964     case CQL_NODE_ST:
965         if (cn->u.st.index)
966         {
967             std::map<std::string,std::string>::const_iterator it;
968             it = fieldmap.find(cn->u.st.index);
969             if (it == fieldmap.end())
970                 return cn;
971             if (it->second.length())
972                 cn->u.st.index = odr_strdup(odr, it->second.c_str());
973             else
974                 cn->u.st.index = 0;
975         }
976         break;
977     case CQL_NODE_BOOL:
978         r = convert_cql_fields(cn->u.boolean.left, odr);
979         if (!r)
980             r = convert_cql_fields(cn->u.boolean.right, odr);
981         break;
982     case CQL_NODE_SORT:
983         r = convert_cql_fields(cn->u.sort.search, odr);
984         break;
985     }
986     return r;
987 }
988
989 void yf::Zoom::Frontend::handle_search(mp::Package &package)
990 {
991     Z_GDU *gdu = package.request().get();
992     Z_APDU *apdu_req = gdu->u.z3950;
993     Z_APDU *apdu_res = 0;
994     mp::odr odr;
995     Z_SearchRequest *sr = apdu_req->u.searchRequest;
996     if (sr->num_databaseNames != 1)
997     {
998         apdu_res = odr.create_searchResponse(
999             apdu_req, YAZ_BIB1_TOO_MANY_DATABASES_SPECIFIED, 0);
1000         package.response() = apdu_res;
1001         return;
1002     }
1003
1004     int error = 0;
1005     char *addinfo_s = 0;
1006     std::string db(sr->databaseNames[0]);
1007     BackendPtr b = get_backend_from_databases(db, &error, &addinfo_s, odr);
1008     if (error)
1009     {
1010         apdu_res = 
1011             odr.create_searchResponse(
1012                 apdu_req, error, addinfo_s);
1013         package.response() = apdu_res;
1014         return;
1015     }
1016
1017     const char *addinfo_c = 0;
1018     b->set_option("setname", "default");
1019
1020     Odr_int hits = 0;
1021     Z_Query *query = sr->query;
1022     WRBUF ccl_wrbuf = 0;
1023     WRBUF pqf_wrbuf = 0;
1024
1025     if (query->which == Z_Query_type_1 || query->which == Z_Query_type_101)
1026     {
1027         // RPN
1028         pqf_wrbuf = wrbuf_alloc();
1029         yaz_rpnquery_to_wrbuf(pqf_wrbuf, query->u.type_1);
1030     }
1031     else if (query->which == Z_Query_type_2)
1032     {
1033         // CCL
1034         ccl_wrbuf = wrbuf_alloc();
1035         wrbuf_write(ccl_wrbuf, (const char *) query->u.type_2->buf,
1036                     query->u.type_2->len);
1037     }
1038     else if (query->which == Z_Query_type_104 &&
1039              query->u.type_104->which == Z_External_CQL)
1040     {
1041         // CQL
1042         const char *cql = query->u.type_104->u.cql;
1043         CQL_parser cp = cql_parser_create();
1044         int r = cql_parser_string(cp, cql);
1045         if (r)
1046         {
1047             cql_parser_destroy(cp);
1048             apdu_res = 
1049                 odr.create_searchResponse(apdu_req, 
1050                                           YAZ_BIB1_MALFORMED_QUERY,
1051                                           "CQL syntax error");
1052             package.response() = apdu_res;
1053             return;
1054         }
1055         struct cql_node *cn = cql_parser_result(cp);
1056         struct cql_node *cn_error = m_p->convert_cql_fields(cn, odr);
1057         if (cn_error)
1058         {
1059             // hopefully we are getting a ptr to a index+relation+term node
1060             addinfo_c = 0;
1061             if (cn_error->which == CQL_NODE_ST)
1062                 addinfo_c = cn_error->u.st.index;
1063
1064             apdu_res = 
1065                 odr.create_searchResponse(apdu_req, 
1066                                           YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1067                                           addinfo_c);
1068             package.response() = apdu_res;
1069             return;
1070         }
1071         char ccl_buf[1024];
1072
1073         r = cql_to_ccl_buf(cn, ccl_buf, sizeof(ccl_buf));
1074         if (r == 0)
1075         {
1076             ccl_wrbuf = wrbuf_alloc();
1077             wrbuf_puts(ccl_wrbuf, ccl_buf);
1078         }
1079         cql_parser_destroy(cp);
1080         if (r)
1081         {
1082             apdu_res = 
1083                 odr.create_searchResponse(apdu_req, 
1084                                           YAZ_BIB1_MALFORMED_QUERY,
1085                                           "CQL to CCL conversion error");
1086             package.response() = apdu_res;
1087             return;
1088         }
1089     }
1090     else
1091     {
1092         apdu_res = 
1093             odr.create_searchResponse(apdu_req, YAZ_BIB1_QUERY_TYPE_UNSUPP, 0);
1094         package.response() = apdu_res;
1095         return;
1096     }
1097
1098     if (ccl_wrbuf)
1099     {
1100         // CCL to PQF
1101         assert(pqf_wrbuf == 0);
1102         int cerror, cpos;
1103         struct ccl_rpn_node *cn;
1104         yaz_log(YLOG_LOG, "CCL: %s", wrbuf_cstr(ccl_wrbuf));
1105         cn = ccl_find_str(b->sptr->ccl_bibset, wrbuf_cstr(ccl_wrbuf),
1106                           &cerror, &cpos);
1107         wrbuf_destroy(ccl_wrbuf);
1108         if (!cn)
1109         {
1110             char *addinfo = odr_strdup(odr, ccl_err_msg(cerror));
1111             int z3950_diag = YAZ_BIB1_MALFORMED_QUERY;
1112
1113             switch (cerror)
1114             {
1115             case CCL_ERR_UNKNOWN_QUAL:
1116                 z3950_diag = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1117                 break;
1118             case CCL_ERR_TRUNC_NOT_LEFT: 
1119             case CCL_ERR_TRUNC_NOT_RIGHT:
1120             case CCL_ERR_TRUNC_NOT_BOTH:
1121                 z3950_diag = YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE;
1122                 break;
1123             }
1124             apdu_res = 
1125                 odr.create_searchResponse(apdu_req, z3950_diag, addinfo);
1126             package.response() = apdu_res;
1127             return;
1128         }
1129         pqf_wrbuf = wrbuf_alloc();
1130         ccl_pquery(pqf_wrbuf, cn);
1131         ccl_rpn_delete(cn);
1132     }
1133     
1134     assert(pqf_wrbuf);
1135     if (b->get_option("sru"))
1136     {
1137         int status = 0;
1138         Z_RPNQuery *zquery;
1139         zquery = p_query_rpn(odr, wrbuf_cstr(pqf_wrbuf));
1140         WRBUF wrb = wrbuf_alloc();
1141             
1142         if (!strcmp(b->get_option("sru"), "solr"))
1143         {
1144             solr_transform_t cqlt = solr_transform_create();
1145             
1146             status = solr_transform_rpn2solr_wrbuf(cqlt, wrb, zquery);
1147             
1148             solr_transform_close(cqlt);
1149         }
1150         else
1151         {
1152             cql_transform_t cqlt = cql_transform_create();
1153             
1154             status = cql_transform_rpn2cql_wrbuf(cqlt, wrb, zquery);
1155             
1156             cql_transform_close(cqlt);
1157         }
1158         if (status == 0)
1159         {
1160             yaz_log(YLOG_LOG, "search CQL: %s", wrbuf_cstr(wrb));
1161             b->search_cql(wrbuf_cstr(wrb), &hits, &error, &addinfo_c);
1162         }
1163         
1164         wrbuf_destroy(wrb);
1165         wrbuf_destroy(pqf_wrbuf);
1166         if (status)
1167         {
1168             apdu_res = 
1169                 odr.create_searchResponse(apdu_req, YAZ_BIB1_MALFORMED_QUERY,
1170                                           "can not convert from RPN to CQL/SOLR");
1171             package.response() = apdu_res;
1172             return;
1173         }
1174     }
1175     else
1176     {
1177         yaz_log(YLOG_LOG, "search PQF: %s", wrbuf_cstr(pqf_wrbuf));
1178         b->search_pqf(wrbuf_cstr(pqf_wrbuf), &hits, &error, &addinfo_c);
1179         wrbuf_destroy(pqf_wrbuf);
1180     }
1181     
1182     
1183     const char *element_set_name = 0;
1184     Odr_int number_to_present = 0;
1185     if (!error)
1186         mp::util::piggyback_sr(sr, hits, number_to_present, &element_set_name);
1187     
1188     Odr_int number_of_records_returned = 0;
1189     Z_Records *records = get_records(
1190         0, number_to_present, &error, &addinfo_c,
1191         &number_of_records_returned, odr, b, sr->preferredRecordSyntax,
1192         element_set_name);
1193     apdu_res = odr.create_searchResponse(apdu_req, error, addinfo_c);
1194     if (records)
1195     {
1196         apdu_res->u.searchResponse->records = records;
1197         apdu_res->u.searchResponse->numberOfRecordsReturned =
1198             odr_intdup(odr, number_of_records_returned);
1199     }
1200     apdu_res->u.searchResponse->resultCount = odr_intdup(odr, hits);
1201     package.response() = apdu_res;
1202 }
1203
1204 void yf::Zoom::Frontend::handle_present(mp::Package &package)
1205 {
1206     Z_GDU *gdu = package.request().get();
1207     Z_APDU *apdu_req = gdu->u.z3950;
1208     Z_APDU *apdu_res = 0;
1209     Z_PresentRequest *pr = apdu_req->u.presentRequest;
1210
1211     mp::odr odr;
1212     if (!m_backend)
1213     {
1214         package.response() = odr.create_presentResponse(
1215             apdu_req, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST, 0);
1216         return;
1217     }
1218     const char *element_set_name = 0;
1219     Z_RecordComposition *comp = pr->recordComposition;
1220     if (comp && comp->which != Z_RecordComp_simple)
1221     {
1222         package.response() = odr.create_presentResponse(
1223             apdu_req, 
1224             YAZ_BIB1_PRESENT_COMP_SPEC_PARAMETER_UNSUPP, 0);
1225         return;
1226     }
1227     if (comp && comp->u.simple->which == Z_ElementSetNames_generic)
1228         element_set_name = comp->u.simple->u.generic;
1229     Odr_int number_of_records_returned = 0;
1230     int error = 0;
1231     const char *addinfo = 0;
1232     Z_Records *records = get_records(
1233         *pr->resultSetStartPoint - 1, *pr->numberOfRecordsRequested,
1234         &error, &addinfo, &number_of_records_returned, odr, m_backend,
1235         pr->preferredRecordSyntax, element_set_name);
1236
1237     apdu_res = odr.create_presentResponse(apdu_req, error, addinfo);
1238     if (records)
1239     {
1240         apdu_res->u.presentResponse->records = records;
1241         apdu_res->u.presentResponse->numberOfRecordsReturned =
1242             odr_intdup(odr, number_of_records_returned);
1243     }
1244     package.response() = apdu_res;
1245 }
1246
1247 void yf::Zoom::Frontend::handle_package(mp::Package &package)
1248 {
1249     Z_GDU *gdu = package.request().get();
1250     if (!gdu)
1251         ;
1252     else if (gdu->which == Z_GDU_Z3950)
1253     {
1254         Z_APDU *apdu_req = gdu->u.z3950;
1255         if (apdu_req->which == Z_APDU_initRequest)
1256         {
1257             mp::odr odr;
1258             package.response() = odr.create_close(
1259                 apdu_req,
1260                 Z_Close_protocolError,
1261                 "double init");
1262         }
1263         else if (apdu_req->which == Z_APDU_searchRequest)
1264         {
1265             handle_search(package);
1266         }
1267         else if (apdu_req->which == Z_APDU_presentRequest)
1268         {
1269             handle_present(package);
1270         }
1271         else
1272         {
1273             mp::odr odr;
1274             package.response() = odr.create_close(
1275                 apdu_req,
1276                 Z_Close_protocolError,
1277                 "zoom filter cannot handle this APDU");
1278             package.session().close();
1279         }
1280     }
1281     else
1282     {
1283         package.session().close();
1284     }
1285 }
1286
1287 void yf::Zoom::Impl::process(mp::Package &package)
1288 {
1289     FrontendPtr f = get_frontend(package);
1290     Z_GDU *gdu = package.request().get();
1291
1292     if (f->m_is_virtual)
1293     {
1294         f->handle_package(package);
1295     }
1296     else if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
1297              Z_APDU_initRequest)
1298     {
1299         Z_InitRequest *req = gdu->u.z3950->u.initRequest;
1300         f->m_init_gdu = gdu;
1301         
1302         mp::odr odr;
1303         Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
1304         Z_InitResponse *resp = apdu->u.initResponse;
1305         
1306         int i;
1307         static const int masks[] = {
1308             Z_Options_search,
1309             Z_Options_present,
1310             -1 
1311         };
1312         for (i = 0; masks[i] != -1; i++)
1313             if (ODR_MASK_GET(req->options, masks[i]))
1314                 ODR_MASK_SET(resp->options, masks[i]);
1315         
1316         static const int versions[] = {
1317             Z_ProtocolVersion_1,
1318             Z_ProtocolVersion_2,
1319             Z_ProtocolVersion_3,
1320             -1
1321         };
1322         for (i = 0; versions[i] != -1; i++)
1323             if (ODR_MASK_GET(req->protocolVersion, versions[i]))
1324                 ODR_MASK_SET(resp->protocolVersion, versions[i]);
1325             else
1326                 break;
1327         
1328         *resp->preferredMessageSize = *req->preferredMessageSize;
1329         *resp->maximumRecordSize = *req->maximumRecordSize;
1330         
1331         package.response() = apdu;
1332         f->m_is_virtual = true;
1333     }
1334     else
1335         package.move();
1336
1337     release_frontend(package);
1338 }
1339
1340
1341 static mp::filter::Base* filter_creator()
1342 {
1343     return new mp::filter::Zoom;
1344 }
1345
1346 extern "C" {
1347     struct metaproxy_1_filter_struct metaproxy_1_filter_zoom = {
1348         0,
1349         "zoom",
1350         filter_creator
1351     };
1352 }
1353
1354
1355 /*
1356  * Local variables:
1357  * c-basic-offset: 4
1358  * c-file-style: "Stroustrup"
1359  * indent-tabs-mode: nil
1360  * End:
1361  * vim: shiftwidth=4 tabstop=8 expandtab
1362  */
1363