zoom: use yaz_filepath_resolve for XSLT files.
[metaproxy-moved-to-github.git] / src / filter_zoom.cpp
1 /* This file is part of Metaproxy.
2    Copyright (C) 2005-2011 Index Data
3
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17 */
18
19 #include "config.hpp"
20 #include "filter_zoom.hpp"
21 #include <yaz/zoom.h>
22 #include <yaz/yaz-version.h>
23 #include <yaz/tpath.h>
24 #include <yaz/srw.h>
25 #include <metaproxy/package.hpp>
26 #include <metaproxy/util.hpp>
27 #include <metaproxy/xmlutil.hpp>
28 #include "torus.hpp"
29
30 #include <libxslt/xsltutils.h>
31 #include <libxslt/transform.h>
32
33 #include <boost/thread/mutex.hpp>
34 #include <boost/thread/condition.hpp>
35 #include <yaz/ccl_xml.h>
36 #include <yaz/ccl.h>
37 #include <yaz/rpn2cql.h>
38 #include <yaz/rpn2solr.h>
39 #include <yaz/pquery.h>
40 #include <yaz/cql.h>
41 #include <yaz/oid_db.h>
42 #include <yaz/diagbib1.h>
43 #include <yaz/log.h>
44 #include <yaz/zgdu.h>
45 #include <yaz/querytowrbuf.h>
46
47 namespace mp = metaproxy_1;
48 namespace yf = mp::filter;
49
50 namespace metaproxy_1 {
51     namespace filter {
52         struct Zoom::Searchable : boost::noncopyable {
53             std::string authentication;
54             std::string cfAuth;
55             std::string cfProxy;
56             std::string cfSubDb;
57             std::string udb;
58             std::string target;
59             std::string query_encoding;
60             std::string sru;
61             std::string request_syntax;
62             std::string element_set;
63             std::string record_encoding;
64             std::string transform_xsl_fname;
65             std::string urlRecipe;
66             bool use_turbomarc;
67             bool piggyback;
68             CCL_bibset ccl_bibset;
69             Searchable(CCL_bibset base);
70             ~Searchable();
71         };
72         class Zoom::Backend : boost::noncopyable {
73             friend class Impl;
74             friend class Frontend;
75             std::string zurl;
76             ZOOM_connection m_connection;
77             ZOOM_resultset m_resultset;
78             std::string m_frontend_database;
79             SearchablePtr sptr;
80             xsltStylesheetPtr xsp;
81         public:
82             Backend(SearchablePtr sptr);
83             ~Backend();
84             void connect(std::string zurl, int *error, char **addinfo,
85                          ODR odr);
86             void search_pqf(const char *pqf, Odr_int *hits,
87                             int *error, char **addinfo, ODR odr);
88             void search_cql(const char *cql, Odr_int *hits,
89                             int *error, char **addinfo, ODR odr);
90             void present(Odr_int start, Odr_int number, ZOOM_record *recs,
91                          int *error, char **addinfo, ODR odr);
92             void set_option(const char *name, const char *value);
93             const char *get_option(const char *name);
94             void get_zoom_error(int *error, char **addinfo, ODR odr);
95         };
96         class Zoom::Frontend : boost::noncopyable {
97             friend class Impl;
98             Impl *m_p;
99             bool m_is_virtual;
100             bool m_in_use;
101             yazpp_1::GDU m_init_gdu;
102             BackendPtr m_backend;
103             void handle_package(mp::Package &package);
104             void handle_search(mp::Package &package);
105             void handle_present(mp::Package &package);
106             BackendPtr get_backend_from_databases(std::string &database,
107                                                   int *error,
108                                                   char **addinfo,
109                                                   ODR odr);
110             Z_Records *get_records(Odr_int start,
111                                    Odr_int number_to_present,
112                                    int *error,
113                                    char **addinfo,
114                                    Odr_int *number_of_records_returned,
115                                    ODR odr, BackendPtr b,
116                                    Odr_oid *preferredRecordSyntax,
117                                    const char *element_set_name);
118         public:
119             Frontend(Impl *impl);
120             ~Frontend();
121         };
122         class Zoom::Impl {
123             friend class Frontend;
124         public:
125             Impl();
126             ~Impl();
127             void process(metaproxy_1::Package & package);
128             void configure(const xmlNode * ptr, bool test_only,
129                            const char *path);
130         private:
131             void configure_local_records(const xmlNode * ptr, bool test_only);
132             FrontendPtr get_frontend(mp::Package &package);
133             void release_frontend(mp::Package &package);
134             SearchablePtr parse_torus_record(const xmlNode *ptr);
135             struct cql_node *convert_cql_fields(struct cql_node *cn, ODR odr);
136             std::map<mp::Session, FrontendPtr> m_clients;            
137             boost::mutex m_mutex;
138             boost::condition m_cond_session_ready;
139             std::string torus_url;
140             std::map<std::string,std::string> fieldmap;
141             std::string xsldir;
142             std::string file_path;
143             CCL_bibset bibset;
144             std::string element_transform;
145             std::string element_raw;
146             std::map<std::string,SearchablePtr> s_map;
147         };
148     }
149 }
150
151 // define Pimpl wrapper forwarding to Impl
152  
153 yf::Zoom::Zoom() : m_p(new Impl)
154 {
155 }
156
157 yf::Zoom::~Zoom()
158 {  // must have a destructor because of boost::scoped_ptr
159 }
160
161 void yf::Zoom::configure(const xmlNode *xmlnode, bool test_only,
162                          const char *path)
163 {
164     m_p->configure(xmlnode, test_only, path);
165 }
166
167 void yf::Zoom::process(mp::Package &package) const
168 {
169     m_p->process(package);
170 }
171
172
173 // define Implementation stuff
174
175 yf::Zoom::Backend::Backend(SearchablePtr ptr) : sptr(ptr)
176 {
177     m_connection = ZOOM_connection_create(0);
178     m_resultset = 0;
179     xsp = 0;
180 }
181
182 yf::Zoom::Backend::~Backend()
183 {
184     if (xsp)
185         xsltFreeStylesheet(xsp);
186     ZOOM_connection_destroy(m_connection);
187     ZOOM_resultset_destroy(m_resultset);
188 }
189
190
191 void yf::Zoom::Backend::get_zoom_error(int *error, char **addinfo,
192                                        ODR odr)
193 {
194     const char *msg = 0;
195     const char *zoom_addinfo = 0;
196     *error = ZOOM_connection_error(m_connection, &msg, &zoom_addinfo);
197     if (*error)
198     {
199         if (*error >= ZOOM_ERROR_CONNECT)
200         {
201             // turn ZOOM diagnostic into a Bib-1 2: with addinfo=zoom err msg
202             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
203             *addinfo = (char *) odr_malloc(
204                 odr, 20 + strlen(msg) + 
205                 (zoom_addinfo ? strlen(zoom_addinfo) : 0));
206             strcpy(*addinfo, msg);
207             if (zoom_addinfo)
208             {
209                 strcat(*addinfo, ": ");
210                 strcat(*addinfo, zoom_addinfo);
211                 strcat(*addinfo, " ");
212             }
213         }
214         else
215         {
216             *addinfo = (char *) odr_malloc(
217                 odr, 20 + (zoom_addinfo ? strlen(zoom_addinfo) : 0));
218             **addinfo = '\0';
219             if (zoom_addinfo && *zoom_addinfo)
220             {
221                 strcpy(*addinfo, zoom_addinfo);
222                 strcat(*addinfo, " ");
223             }
224             strcat(*addinfo, "(backend)");
225         }
226     }
227 }
228
229 void yf::Zoom::Backend::connect(std::string zurl,
230                                 int *error, char **addinfo,
231                                 ODR odr)
232 {
233     ZOOM_connection_connect(m_connection, zurl.c_str(), 0);
234     get_zoom_error(error, addinfo, odr);
235 }
236
237 void yf::Zoom::Backend::search_pqf(const char *pqf, Odr_int *hits,
238                                    int *error, char **addinfo, ODR odr)
239 {
240     m_resultset = ZOOM_connection_search_pqf(m_connection, pqf);
241     get_zoom_error(error, addinfo, odr);
242     if (*error == 0)
243         *hits = ZOOM_resultset_size(m_resultset);
244     else
245         *hits = 0;
246 }
247
248 void yf::Zoom::Backend::search_cql(const char *cql, Odr_int *hits,
249                                    int *error, char **addinfo, ODR odr)
250 {
251     ZOOM_query q = ZOOM_query_create();
252
253     ZOOM_query_cql(q, cql);
254
255     m_resultset = ZOOM_connection_search(m_connection, q);
256     ZOOM_query_destroy(q);
257     get_zoom_error(error, addinfo, odr);
258     if (*error == 0)
259         *hits = ZOOM_resultset_size(m_resultset);
260     else
261         *hits = 0;
262 }
263
264 void yf::Zoom::Backend::present(Odr_int start, Odr_int number,
265                                 ZOOM_record *recs,
266                                 int *error, char **addinfo, ODR odr)
267 {
268     ZOOM_resultset_records(m_resultset, recs, start, number);
269     get_zoom_error(error, addinfo, odr);
270 }
271
272 void yf::Zoom::Backend::set_option(const char *name, const char *value)
273 {
274     ZOOM_connection_option_set(m_connection, name, value);
275     if (m_resultset)
276         ZOOM_resultset_option_set(m_resultset, name, value);
277 }
278
279 const char *yf::Zoom::Backend::get_option(const char *name)
280 {
281     return ZOOM_connection_option_get(m_connection, name);
282 }
283
284 yf::Zoom::Searchable::Searchable(CCL_bibset base)
285 {
286     piggyback = true;
287     use_turbomarc = true;
288     ccl_bibset = ccl_qual_dup(base);
289 }
290
291 yf::Zoom::Searchable::~Searchable()
292 {
293     ccl_qual_rm(&ccl_bibset);
294 }
295
296 yf::Zoom::Frontend::Frontend(Impl *impl) : 
297     m_p(impl), m_is_virtual(false), m_in_use(true)
298 {
299 }
300
301 yf::Zoom::Frontend::~Frontend()
302 {
303 }
304
305 yf::Zoom::FrontendPtr yf::Zoom::Impl::get_frontend(mp::Package &package)
306 {
307     boost::mutex::scoped_lock lock(m_mutex);
308
309     std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
310     
311     while(true)
312     {
313         it = m_clients.find(package.session());
314         if (it == m_clients.end())
315             break;
316         
317         if (!it->second->m_in_use)
318         {
319             it->second->m_in_use = true;
320             return it->second;
321         }
322         m_cond_session_ready.wait(lock);
323     }
324     FrontendPtr f(new Frontend(this));
325     m_clients[package.session()] = f;
326     f->m_in_use = true;
327     return f;
328 }
329
330 void yf::Zoom::Impl::release_frontend(mp::Package &package)
331 {
332     boost::mutex::scoped_lock lock(m_mutex);
333     std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
334     
335     it = m_clients.find(package.session());
336     if (it != m_clients.end())
337     {
338         if (package.session().is_closed())
339         {
340             m_clients.erase(it);
341         }
342         else
343         {
344             it->second->m_in_use = false;
345         }
346         m_cond_session_ready.notify_all();
347     }
348 }
349
350 yf::Zoom::Impl::Impl() : element_transform("pz2") , element_raw("raw")
351 {
352     bibset = ccl_qual_mk();
353 }
354
355 yf::Zoom::Impl::~Impl()
356
357     ccl_qual_rm(&bibset);
358 }
359
360 yf::Zoom::SearchablePtr yf::Zoom::Impl::parse_torus_record(const xmlNode *ptr)
361 {
362     Zoom::SearchablePtr s(new Searchable(bibset));
363     
364     for (ptr = ptr->children; ptr; ptr = ptr->next)
365     {
366         if (ptr->type != XML_ELEMENT_NODE)
367             continue;
368         if (!strcmp((const char *) ptr->name, "layer"))
369             ptr = ptr->children;
370         else if (!strcmp((const char *) ptr->name,
371                          "authentication"))
372         {
373             s->authentication = mp::xml::get_text(ptr);
374         }
375         else if (!strcmp((const char *) ptr->name,
376                          "cfAuth"))
377         {
378             s->cfAuth = mp::xml::get_text(ptr);
379         } 
380         else if (!strcmp((const char *) ptr->name,
381                          "cfProxy"))
382         {
383             s->cfProxy = mp::xml::get_text(ptr);
384         }  
385         else if (!strcmp((const char *) ptr->name,
386                          "cfSubDb"))
387         {
388             s->cfSubDb = mp::xml::get_text(ptr);
389         }  
390         else if (!strcmp((const char *) ptr->name, "udb"))
391         {
392             s->udb = mp::xml::get_text(ptr);
393         }
394         else if (!strcmp((const char *) ptr->name, "zurl"))
395         {
396             s->target = mp::xml::get_text(ptr);
397         }
398         else if (!strcmp((const char *) ptr->name, "sru"))
399         {
400             s->sru = mp::xml::get_text(ptr);
401         }
402         else if (!strcmp((const char *) ptr->name,
403                          "queryEncoding"))
404         {
405             s->query_encoding = mp::xml::get_text(ptr);
406         }
407         else if (!strcmp((const char *) ptr->name,
408                          "piggyback"))
409         {
410             s->piggyback = mp::xml::get_bool(ptr, true);
411         }
412         else if (!strcmp((const char *) ptr->name,
413                          "requestSyntax"))
414         {
415             s->request_syntax = mp::xml::get_text(ptr);
416         }
417         else if (!strcmp((const char *) ptr->name,
418                          "elementSet"))
419         {
420             s->element_set = mp::xml::get_text(ptr);
421         }
422         else if (!strcmp((const char *) ptr->name,
423                          "recordEncoding"))
424         {
425             s->record_encoding = mp::xml::get_text(ptr);
426         }
427         else if (!strcmp((const char *) ptr->name,
428                          "transform"))
429         {
430             s->transform_xsl_fname = mp::xml::get_text(ptr);
431         }
432         else if (!strcmp((const char *) ptr->name,
433                          "urlRecipe"))
434         {
435             s->urlRecipe = mp::xml::get_text(ptr);
436         }
437         else if (!strcmp((const char *) ptr->name,
438                          "useTurboMarc"))
439         {
440             ; // useTurboMarc is ignored
441         }
442         else if (!strncmp((const char *) ptr->name,
443                           "cclmap_", 7))
444         {
445             std::string value = mp::xml::get_text(ptr);
446             ccl_qual_fitem(s->ccl_bibset, value.c_str(),
447                            (const char *) ptr->name + 7);
448         }
449     }
450     return s;
451 }
452
453 void yf::Zoom::Impl::configure_local_records(const xmlNode *ptr, bool test_only)
454 {
455     while (ptr && ptr->type != XML_ELEMENT_NODE)
456         ptr = ptr->next;
457     
458     if (ptr)
459     {
460         if (!strcmp((const char *) ptr->name, "records"))
461         {
462             for (ptr = ptr->children; ptr; ptr = ptr->next)
463             {
464                 if (ptr->type != XML_ELEMENT_NODE)
465                     continue;
466                 if (!strcmp((const char *) ptr->name, "record"))
467                 {
468                     SearchablePtr s = parse_torus_record(ptr);
469                     if (s)
470                     {
471                         std::string udb = s->udb;
472                         if (udb.length())
473                             s_map[s->udb] = s;
474                         else
475                         {
476                             throw mp::filter::FilterException
477                                 ("No udb for local torus record");
478                         }
479                     }
480                 }
481                 else
482                 {
483                     throw mp::filter::FilterException
484                         ("Bad element " 
485                          + std::string((const char *) ptr->name)
486                          + " in zoom filter inside element "
487                          "<torus><records>");
488                 }
489             }
490         }
491         else
492         {
493             throw mp::filter::FilterException
494                 ("Bad element " 
495                  + std::string((const char *) ptr->name)
496                  + " in zoom filter inside element <torus>");
497         }
498     }
499 }
500
501 void yf::Zoom::Impl::configure(const xmlNode *ptr, bool test_only,
502                                const char *path)
503 {
504     if (path && *path)
505     {
506         file_path = path;
507     }
508     for (ptr = ptr->children; ptr; ptr = ptr->next)
509     {
510         if (ptr->type != XML_ELEMENT_NODE)
511             continue;
512         else if (!strcmp((const char *) ptr->name, "torus"))
513         {
514             const struct _xmlAttr *attr;
515             for (attr = ptr->properties; attr; attr = attr->next)
516             {
517                 if (!strcmp((const char *) attr->name, "url"))
518                     torus_url = mp::xml::get_text(attr->children);
519                 else if (!strcmp((const char *) attr->name, "xsldir"))
520                     xsldir = mp::xml::get_text(attr->children);
521                 else if (!strcmp((const char *) attr->name, "element_transform"))
522                     element_transform = mp::xml::get_text(attr->children);
523                 else if (!strcmp((const char *) attr->name, "element_raw"))
524                     element_raw = mp::xml::get_text(attr->children);
525                 else
526                     throw mp::filter::FilterException(
527                         "Bad attribute " + std::string((const char *)
528                                                        attr->name));
529             }
530             configure_local_records(ptr->children, test_only);
531         }
532         else if (!strcmp((const char *) ptr->name, "cclmap"))
533         {
534             const char *addinfo = 0;
535             ccl_xml_config(bibset, ptr, &addinfo);
536         }
537         else if (!strcmp((const char *) ptr->name, "fieldmap"))
538         {
539             const struct _xmlAttr *attr;
540             std::string ccl_field;
541             std::string cql_field;
542             for (attr = ptr->properties; attr; attr = attr->next)
543             {
544                 if (!strcmp((const char *) attr->name, "ccl"))
545                     ccl_field = mp::xml::get_text(attr->children);
546                 else if (!strcmp((const char *) attr->name, "cql"))
547                     cql_field = mp::xml::get_text(attr->children);
548                 else
549                     throw mp::filter::FilterException(
550                         "Bad attribute " + std::string((const char *)
551                                                        attr->name));
552             }
553             if (cql_field.length())
554                 fieldmap[cql_field] = ccl_field;
555         }
556         else
557         {
558             throw mp::filter::FilterException
559                 ("Bad element " 
560                  + std::string((const char *) ptr->name)
561                  + " in zoom filter");
562         }
563     }
564 }
565
566 yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
567     std::string &database, int *error, char **addinfo, ODR odr)
568 {
569     std::list<BackendPtr>::const_iterator map_it;
570     if (m_backend && m_backend->m_frontend_database == database)
571         return m_backend;
572
573     const char *sru_proxy = 0;
574     std::string db_args;
575     std::string torus_db;
576     size_t db_arg_pos = database.find(',');
577     if (db_arg_pos != std::string::npos)
578     {
579         torus_db = database.substr(0, db_arg_pos);
580         db_args = database.substr(db_arg_pos + 1);
581     }
582     else
583         torus_db = database;
584  
585     SearchablePtr sptr;
586
587     std::map<std::string,SearchablePtr>::iterator it;
588     it = m_p->s_map.find(torus_db);
589     if (it != m_p->s_map.end())
590         sptr = it->second;
591     else
592     {
593         xmlDoc *doc = mp::get_searchable(m_p->torus_url, torus_db);
594         if (!doc)
595         {
596             *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
597             *addinfo = odr_strdup(odr, database.c_str());
598             BackendPtr b;
599             return b;
600         }
601         const xmlNode *ptr = xmlDocGetRootElement(doc);
602         if (ptr)
603         {   // presumably ptr is a records element node
604             // parse first record in document
605             for (ptr = ptr->children; ptr; ptr = ptr->next)
606             {
607                 if (ptr->type == XML_ELEMENT_NODE
608                     && !strcmp((const char *) ptr->name, "record"))
609                 {
610                     sptr = m_p->parse_torus_record(ptr);
611                     break;
612                 }
613             }
614         }
615         xmlFreeDoc(doc);
616     }
617
618     if (!sptr)
619     {
620         *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
621         *addinfo = odr_strdup(odr, database.c_str());
622         BackendPtr b;
623         return b;
624     }
625         
626     xsltStylesheetPtr xsp = 0;
627     if (sptr->transform_xsl_fname.length())
628     {
629         const char *path = 0;
630
631         if (m_p->xsldir.length())
632             path = m_p->xsldir.c_str();
633         else
634             path = m_p->file_path.c_str();
635         std::string fname;
636
637         char fullpath[1024];
638         char *cp = yaz_filepath_resolve(sptr->transform_xsl_fname.c_str(),
639                                         path, 0, fullpath);
640         if (cp)
641             fname.assign(cp);
642         else
643         {
644             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
645             *addinfo = (char *)
646                 odr_malloc(odr, 40 + sptr->transform_xsl_fname.length());
647             sprintf(*addinfo, "File could not be read: %s", 
648                     sptr->transform_xsl_fname.c_str());
649             BackendPtr b;
650             return b;
651         }
652         xmlDoc *xsp_doc = xmlParseFile(fname.c_str());
653         if (!xsp_doc)
654         {
655             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
656             *addinfo = (char *) odr_malloc(odr, 40 + fname.length());
657             sprintf(*addinfo, "xmlParseFile failed. File: %s", fname.c_str());
658             BackendPtr b;
659             return b;
660         }
661         xsp = xsltParseStylesheetDoc(xsp_doc);
662         if (!xsp)
663         {
664             *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
665             *addinfo = odr_strdup(odr, "xsltParseStylesheetDoc failed");
666             BackendPtr b;
667             xmlFreeDoc(xsp_doc);
668             return b;
669         }
670     }
671
672     m_backend.reset();
673
674     BackendPtr b(new Backend(sptr));
675
676     b->xsp = xsp;
677     b->m_frontend_database = database;
678     std::string authentication = sptr->authentication;
679         
680     b->set_option("timeout", "40");
681
682     if (sptr->query_encoding.length())
683         b->set_option("rpnCharset", sptr->query_encoding.c_str());
684
685     if (sptr->cfAuth.length())
686     {
687         // A CF target
688         b->set_option("user", sptr->cfAuth.c_str());
689         if (db_args.length() == 0)
690         {
691             if (authentication.length())
692             {
693                 // no database (auth) args specified already.. and the
694                 // Torus authentication has it.. Generate the args that CF
695                 // understands..
696                 size_t found = authentication.find('/');
697                 if (found != std::string::npos)
698                 {
699                     db_args += "user=" + mp::util::uri_encode(authentication.substr(0, found))
700                         + "&password=" + mp::util::uri_encode(authentication.substr(found+1));
701                 }
702                 else
703                     db_args += "user=" + mp::util::uri_encode(authentication);
704             }
705             if (sptr->cfProxy.length())
706             {
707                 if (db_args.length())
708                     db_args += "&";
709                 db_args += "proxy=" + mp::util::uri_encode(sptr->cfProxy);
710             }
711         }
712         if (sptr->cfSubDb.length())
713         {
714             if (db_args.length())
715                 db_args += "&";
716             db_args += "subdatabase=" + mp::util::uri_encode(sptr->cfSubDb);
717         }
718     }
719     else
720     {
721         // A non-CF target
722         if (db_args.length())
723         {
724             // user has specified backend authentication
725             const char *param_user = 0;
726             const char *param_password = 0;
727             char **names;
728             char **values;
729             int i;
730             int no_parms = yaz_uri_to_array(db_args.c_str(),
731                                             odr, &names, &values);
732             for (i = 0; i < no_parms; i++)
733             {
734                 const char *name = names[i];
735                 const char *value = values[i];
736                 if (!strcmp(name, "user"))
737                     param_user = value;
738                 else if (!strcmp(name, "password"))
739                     param_password = value;
740                 else if (!strcmp(name, "proxy"))
741                     sru_proxy = value;
742                 else
743                 {
744                     BackendPtr notfound;
745                     char *msg = (char*) odr_malloc(odr, strlen(name) + 30);
746                     *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
747                     sprintf(msg, "Bad database argument: %s", name);
748                     *addinfo = msg;
749                     return notfound;
750                 }
751             }
752             if (param_user && param_password)
753             {
754                 char *auth = (char*) odr_malloc(
755                     odr, strlen(param_user) + strlen(param_password) + 2);
756                 strcpy(auth, param_user);
757                 strcat(auth, "/");
758                 strcat(auth, param_password);
759                 b->set_option("user", auth);
760             }
761             db_args.clear(); // no arguments to be passed (non-CF)
762         }
763         else
764         {
765             // use authentication from Torus, if given
766             if (authentication.length())
767                 b->set_option("user", authentication.c_str());
768         }
769     }
770
771     if (sru_proxy)
772         b->set_option("proxy", sru_proxy);
773
774     std::string url;
775     if (sptr->sru.length())
776     {
777         url = "http://" + sptr->target;
778         b->set_option("sru", sptr->sru.c_str());
779     }
780     else
781     {
782         url = sptr->target;
783     }
784     if (db_args.length())
785         url += "," + db_args;
786     yaz_log(YLOG_LOG, "url=%s", url.c_str());
787     b->connect(url, error, addinfo, odr);
788     if (*error == 0)
789     {
790         m_backend = b;
791     }
792     return b;
793 }
794
795 Z_Records *yf::Zoom::Frontend::get_records(Odr_int start,
796                                            Odr_int number_to_present,
797                                            int *error,
798                                            char **addinfo,
799                                            Odr_int *number_of_records_returned,
800                                            ODR odr,
801                                            BackendPtr b,
802                                            Odr_oid *preferredRecordSyntax,
803                                            const char *element_set_name)
804 {
805     *number_of_records_returned = 0;
806     Z_Records *records = 0;
807     bool enable_pz2_retrieval = false; // whether target profile is used
808     bool enable_pz2_transform = false; // whether XSLT is used as well
809     bool assume_marc8_charset = false;
810
811     if (start < 0 || number_to_present <= 0)
812         return records;
813     
814     if (number_to_present > 10000)
815         number_to_present = 10000;
816     
817     ZOOM_record *recs = (ZOOM_record *)
818         odr_malloc(odr, number_to_present * sizeof(*recs));
819
820     char oid_name_str[OID_STR_MAX];
821     const char *syntax_name = 0;
822     
823     if (preferredRecordSyntax &&
824         !oid_oidcmp(preferredRecordSyntax, yaz_oid_recsyn_xml)
825         && element_set_name)
826     {
827         if (!strcmp(element_set_name, m_p->element_transform.c_str()))
828         {
829             enable_pz2_retrieval = true;
830             enable_pz2_transform = true;
831         }
832         else if (!strcmp(element_set_name, m_p->element_raw.c_str()))
833         {
834             enable_pz2_retrieval = true;
835         }
836     }
837     
838     if (enable_pz2_retrieval)
839     {
840         if (b->sptr->request_syntax.length())
841         {
842             syntax_name = b->sptr->request_syntax.c_str();
843             const Odr_oid *syntax_oid = 
844                 yaz_string_to_oid(yaz_oid_std(), CLASS_RECSYN, syntax_name);
845             if (!oid_oidcmp(syntax_oid, yaz_oid_recsyn_usmarc)
846                 || !oid_oidcmp(syntax_oid, yaz_oid_recsyn_opac))
847                 assume_marc8_charset = true;
848         }
849     }
850     else if (preferredRecordSyntax)
851         syntax_name =
852             yaz_oid_to_string_buf(preferredRecordSyntax, 0, oid_name_str);
853
854     b->set_option("preferredRecordSyntax", syntax_name);
855
856     if (enable_pz2_retrieval)
857     {
858         element_set_name = 0;
859         if (b->sptr->element_set.length())
860             element_set_name = b->sptr->element_set.c_str();
861     }
862
863     b->set_option("elementSetName", element_set_name);
864
865     b->present(start, number_to_present, recs, error, addinfo, odr);
866
867     Odr_int i = 0;
868     if (!*error)
869     {
870         for (i = 0; i < number_to_present; i++)
871             if (!recs[i])
872                 break;
873     }
874     if (i > 0)
875     {  // only return records if no error and at least one record
876         char *odr_database = odr_strdup(odr,
877                                         b->m_frontend_database.c_str());
878         Z_NamePlusRecordList *npl = (Z_NamePlusRecordList *)
879             odr_malloc(odr, sizeof(*npl));
880         *number_of_records_returned = i;
881         npl->num_records = i;
882         npl->records = (Z_NamePlusRecord **)
883             odr_malloc(odr, i * sizeof(*npl->records));
884         for (i = 0; i < number_to_present; i++)
885         {
886             Z_NamePlusRecord *npr = 0;
887             const char *addinfo;
888             int sur_error = ZOOM_record_error(recs[i], 0 /* msg */,
889                                               &addinfo, 0 /* diagset */);
890                 
891             if (sur_error)
892             {
893                 npr = zget_surrogateDiagRec(odr, odr_database, sur_error,
894                                             addinfo);
895             }
896             else if (enable_pz2_retrieval)
897             {
898                 char rec_type_str[100];
899                 const char *record_encoding = 0;
900
901                 if (b->sptr->record_encoding.length())
902                     record_encoding = b->sptr->record_encoding.c_str();
903                 else if (assume_marc8_charset)
904                     record_encoding = "marc8";
905
906                 strcpy(rec_type_str, b->sptr->use_turbomarc ? "txml" : "xml");
907                 if (record_encoding)
908                 {
909                     strcat(rec_type_str, "; charset=");
910                     strcat(rec_type_str, record_encoding);
911                 }
912                 
913                 int rec_len;
914                 xmlChar *xmlrec_buf = 0;
915                 const char *rec_buf = ZOOM_record_get(recs[i], rec_type_str,
916                                                       &rec_len);
917                 if (rec_buf && b->xsp && enable_pz2_transform)
918                 {
919                     xmlDoc *rec_doc = xmlParseMemory(rec_buf, rec_len);
920                     if (rec_doc)
921                     { 
922                         xmlDoc *rec_res;
923                         rec_res = xsltApplyStylesheet(b->xsp, rec_doc, 0);
924
925                         if (rec_res)
926                             xsltSaveResultToString(&xmlrec_buf, &rec_len,
927                                                    rec_res, b->xsp);
928                         rec_buf = (const char *) xmlrec_buf;
929                         xmlFreeDoc(rec_doc);
930                         xmlFreeDoc(rec_res);
931                     }
932                 }
933
934                 if (rec_buf)
935                 {
936                     xmlDoc *doc = xmlParseMemory(rec_buf, rec_len);
937                     std::string res = 
938                         mp::xml::url_recipe_handle(doc, b->sptr->urlRecipe);
939                     if (res.length())
940                     {
941                         xmlNode *ptr = xmlDocGetRootElement(doc);
942                         while (ptr && ptr->type != XML_ELEMENT_NODE)
943                             ptr = ptr->next;
944                         xmlNode *c = 
945                             xmlNewChild(ptr, 0, BAD_CAST "generated-url", 0);
946                         xmlNode * t = xmlNewText(BAD_CAST res.c_str());
947                         xmlAddChild(c, t);
948
949                         if (xmlrec_buf)
950                             xmlFree(xmlrec_buf);
951
952                         xmlDocDumpMemory(doc, &xmlrec_buf, &rec_len);
953                         rec_buf = (const char *) xmlrec_buf;
954                     }
955                     xmlFreeDoc(doc);
956                 }
957                 if (rec_buf)
958                 {
959                     npr = (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
960                     npr->databaseName = odr_database;
961                     npr->which = Z_NamePlusRecord_databaseRecord;
962                     npr->u.databaseRecord =
963                         z_ext_record_xml(odr, rec_buf, rec_len);
964                 }
965                 else
966                 {
967                     npr = zget_surrogateDiagRec(
968                         odr, odr_database, 
969                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
970                         rec_type_str);
971                 }
972                 if (xmlrec_buf)
973                     xmlFree(xmlrec_buf);
974             }
975             else
976             {
977                 Z_External *ext =
978                     (Z_External *) ZOOM_record_get(recs[i], "ext", 0);
979                 if (ext)
980                 {
981                     npr = (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
982                     npr->databaseName = odr_database;
983                     npr->which = Z_NamePlusRecord_databaseRecord;
984                     npr->u.databaseRecord = ext;
985                 }
986                 else
987                 {
988                     npr = zget_surrogateDiagRec(
989                         odr, odr_database, 
990                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
991                         "ZOOM_record, type ext");
992                 }
993             }
994             npl->records[i] = npr;
995         }
996         records = (Z_Records*) odr_malloc(odr, sizeof(*records));
997         records->which = Z_Records_DBOSD;
998         records->u.databaseOrSurDiagnostics = npl;
999     }
1000     return records;
1001 }
1002     
1003 struct cql_node *yf::Zoom::Impl::convert_cql_fields(struct cql_node *cn,
1004                                                     ODR odr)
1005 {
1006     struct cql_node *r = 0;
1007     if (!cn)
1008         return 0;
1009     switch (cn->which)
1010     {
1011     case CQL_NODE_ST:
1012         if (cn->u.st.index)
1013         {
1014             std::map<std::string,std::string>::const_iterator it;
1015             it = fieldmap.find(cn->u.st.index);
1016             if (it == fieldmap.end())
1017                 return cn;
1018             if (it->second.length())
1019                 cn->u.st.index = odr_strdup(odr, it->second.c_str());
1020             else
1021                 cn->u.st.index = 0;
1022         }
1023         break;
1024     case CQL_NODE_BOOL:
1025         r = convert_cql_fields(cn->u.boolean.left, odr);
1026         if (!r)
1027             r = convert_cql_fields(cn->u.boolean.right, odr);
1028         break;
1029     case CQL_NODE_SORT:
1030         r = convert_cql_fields(cn->u.sort.search, odr);
1031         break;
1032     }
1033     return r;
1034 }
1035
1036 void yf::Zoom::Frontend::handle_search(mp::Package &package)
1037 {
1038     Z_GDU *gdu = package.request().get();
1039     Z_APDU *apdu_req = gdu->u.z3950;
1040     Z_APDU *apdu_res = 0;
1041     mp::odr odr;
1042     Z_SearchRequest *sr = apdu_req->u.searchRequest;
1043     if (sr->num_databaseNames != 1)
1044     {
1045         apdu_res = odr.create_searchResponse(
1046             apdu_req, YAZ_BIB1_TOO_MANY_DATABASES_SPECIFIED, 0);
1047         package.response() = apdu_res;
1048         return;
1049     }
1050
1051     int error = 0;
1052     char *addinfo = 0;
1053     std::string db(sr->databaseNames[0]);
1054     BackendPtr b = get_backend_from_databases(db, &error, &addinfo, odr);
1055     if (error)
1056     {
1057         apdu_res = 
1058             odr.create_searchResponse(apdu_req, error, addinfo);
1059         package.response() = apdu_res;
1060         return;
1061     }
1062
1063     b->set_option("setname", "default");
1064
1065     Odr_int hits = 0;
1066     Z_Query *query = sr->query;
1067     WRBUF ccl_wrbuf = 0;
1068     WRBUF pqf_wrbuf = 0;
1069
1070     if (query->which == Z_Query_type_1 || query->which == Z_Query_type_101)
1071     {
1072         // RPN
1073         pqf_wrbuf = wrbuf_alloc();
1074         yaz_rpnquery_to_wrbuf(pqf_wrbuf, query->u.type_1);
1075     }
1076     else if (query->which == Z_Query_type_2)
1077     {
1078         // CCL
1079         ccl_wrbuf = wrbuf_alloc();
1080         wrbuf_write(ccl_wrbuf, (const char *) query->u.type_2->buf,
1081                     query->u.type_2->len);
1082     }
1083     else if (query->which == Z_Query_type_104 &&
1084              query->u.type_104->which == Z_External_CQL)
1085     {
1086         // CQL
1087         const char *cql = query->u.type_104->u.cql;
1088         CQL_parser cp = cql_parser_create();
1089         int r = cql_parser_string(cp, cql);
1090         if (r)
1091         {
1092             cql_parser_destroy(cp);
1093             apdu_res = 
1094                 odr.create_searchResponse(apdu_req, 
1095                                           YAZ_BIB1_MALFORMED_QUERY,
1096                                           "CQL syntax error");
1097             package.response() = apdu_res;
1098             return;
1099         }
1100         struct cql_node *cn = cql_parser_result(cp);
1101         struct cql_node *cn_error = m_p->convert_cql_fields(cn, odr);
1102         if (cn_error)
1103         {
1104             // hopefully we are getting a ptr to a index+relation+term node
1105             addinfo = 0;
1106             if (cn_error->which == CQL_NODE_ST)
1107                 addinfo = cn_error->u.st.index;
1108
1109             apdu_res = 
1110                 odr.create_searchResponse(apdu_req, 
1111                                           YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1112                                           addinfo);
1113             package.response() = apdu_res;
1114             return;
1115         }
1116         char ccl_buf[1024];
1117
1118         r = cql_to_ccl_buf(cn, ccl_buf, sizeof(ccl_buf));
1119         if (r == 0)
1120         {
1121             ccl_wrbuf = wrbuf_alloc();
1122             wrbuf_puts(ccl_wrbuf, ccl_buf);
1123         }
1124         cql_parser_destroy(cp);
1125         if (r)
1126         {
1127             apdu_res = 
1128                 odr.create_searchResponse(apdu_req, 
1129                                           YAZ_BIB1_MALFORMED_QUERY,
1130                                           "CQL to CCL conversion error");
1131             package.response() = apdu_res;
1132             return;
1133         }
1134     }
1135     else
1136     {
1137         apdu_res = 
1138             odr.create_searchResponse(apdu_req, YAZ_BIB1_QUERY_TYPE_UNSUPP, 0);
1139         package.response() = apdu_res;
1140         return;
1141     }
1142
1143     if (ccl_wrbuf)
1144     {
1145         // CCL to PQF
1146         assert(pqf_wrbuf == 0);
1147         int cerror, cpos;
1148         struct ccl_rpn_node *cn;
1149         yaz_log(YLOG_LOG, "CCL: %s", wrbuf_cstr(ccl_wrbuf));
1150         cn = ccl_find_str(b->sptr->ccl_bibset, wrbuf_cstr(ccl_wrbuf),
1151                           &cerror, &cpos);
1152         wrbuf_destroy(ccl_wrbuf);
1153         if (!cn)
1154         {
1155             char *addinfo = odr_strdup(odr, ccl_err_msg(cerror));
1156             int z3950_diag = YAZ_BIB1_MALFORMED_QUERY;
1157
1158             switch (cerror)
1159             {
1160             case CCL_ERR_UNKNOWN_QUAL:
1161                 z3950_diag = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1162                 break;
1163             case CCL_ERR_TRUNC_NOT_LEFT: 
1164             case CCL_ERR_TRUNC_NOT_RIGHT:
1165             case CCL_ERR_TRUNC_NOT_BOTH:
1166                 z3950_diag = YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE;
1167                 break;
1168             }
1169             apdu_res = 
1170                 odr.create_searchResponse(apdu_req, z3950_diag, addinfo);
1171             package.response() = apdu_res;
1172             return;
1173         }
1174         pqf_wrbuf = wrbuf_alloc();
1175         ccl_pquery(pqf_wrbuf, cn);
1176         ccl_rpn_delete(cn);
1177     }
1178     
1179     assert(pqf_wrbuf);
1180     if (b->get_option("sru"))
1181     {
1182         int status = 0;
1183         Z_RPNQuery *zquery;
1184         zquery = p_query_rpn(odr, wrbuf_cstr(pqf_wrbuf));
1185         WRBUF wrb = wrbuf_alloc();
1186             
1187         if (!strcmp(b->get_option("sru"), "solr"))
1188         {
1189             solr_transform_t cqlt = solr_transform_create();
1190             
1191             status = solr_transform_rpn2solr_wrbuf(cqlt, wrb, zquery);
1192             
1193             solr_transform_close(cqlt);
1194         }
1195         else
1196         {
1197             cql_transform_t cqlt = cql_transform_create();
1198             
1199             status = cql_transform_rpn2cql_wrbuf(cqlt, wrb, zquery);
1200             
1201             cql_transform_close(cqlt);
1202         }
1203         if (status == 0)
1204         {
1205             yaz_log(YLOG_LOG, "search CQL: %s", wrbuf_cstr(wrb));
1206             b->search_cql(wrbuf_cstr(wrb), &hits, &error, &addinfo, odr);
1207         }
1208         
1209         wrbuf_destroy(wrb);
1210         wrbuf_destroy(pqf_wrbuf);
1211         if (status)
1212         {
1213             apdu_res = 
1214                 odr.create_searchResponse(apdu_req, YAZ_BIB1_MALFORMED_QUERY,
1215                                           "can not convert from RPN to CQL/SOLR");
1216             package.response() = apdu_res;
1217             return;
1218         }
1219     }
1220     else
1221     {
1222         yaz_log(YLOG_LOG, "search PQF: %s", wrbuf_cstr(pqf_wrbuf));
1223         b->search_pqf(wrbuf_cstr(pqf_wrbuf), &hits, &error, &addinfo, odr);
1224         wrbuf_destroy(pqf_wrbuf);
1225     }
1226     
1227     
1228     const char *element_set_name = 0;
1229     Odr_int number_to_present = 0;
1230     if (!error)
1231         mp::util::piggyback_sr(sr, hits, number_to_present, &element_set_name);
1232     
1233     Odr_int number_of_records_returned = 0;
1234     Z_Records *records = get_records(
1235         0, number_to_present, &error, &addinfo,
1236         &number_of_records_returned, odr, b, sr->preferredRecordSyntax,
1237         element_set_name);
1238     apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
1239     if (records)
1240     {
1241         apdu_res->u.searchResponse->records = records;
1242         apdu_res->u.searchResponse->numberOfRecordsReturned =
1243             odr_intdup(odr, number_of_records_returned);
1244     }
1245     apdu_res->u.searchResponse->resultCount = odr_intdup(odr, hits);
1246     package.response() = apdu_res;
1247 }
1248
1249 void yf::Zoom::Frontend::handle_present(mp::Package &package)
1250 {
1251     Z_GDU *gdu = package.request().get();
1252     Z_APDU *apdu_req = gdu->u.z3950;
1253     Z_APDU *apdu_res = 0;
1254     Z_PresentRequest *pr = apdu_req->u.presentRequest;
1255
1256     mp::odr odr;
1257     if (!m_backend)
1258     {
1259         package.response() = odr.create_presentResponse(
1260             apdu_req, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST, 0);
1261         return;
1262     }
1263     const char *element_set_name = 0;
1264     Z_RecordComposition *comp = pr->recordComposition;
1265     if (comp && comp->which != Z_RecordComp_simple)
1266     {
1267         package.response() = odr.create_presentResponse(
1268             apdu_req, 
1269             YAZ_BIB1_PRESENT_COMP_SPEC_PARAMETER_UNSUPP, 0);
1270         return;
1271     }
1272     if (comp && comp->u.simple->which == Z_ElementSetNames_generic)
1273         element_set_name = comp->u.simple->u.generic;
1274     Odr_int number_of_records_returned = 0;
1275     int error = 0;
1276     char *addinfo = 0;
1277     Z_Records *records = get_records(
1278         *pr->resultSetStartPoint - 1, *pr->numberOfRecordsRequested,
1279         &error, &addinfo, &number_of_records_returned, odr, m_backend,
1280         pr->preferredRecordSyntax, element_set_name);
1281
1282     apdu_res = odr.create_presentResponse(apdu_req, error, addinfo);
1283     if (records)
1284     {
1285         apdu_res->u.presentResponse->records = records;
1286         apdu_res->u.presentResponse->numberOfRecordsReturned =
1287             odr_intdup(odr, number_of_records_returned);
1288     }
1289     package.response() = apdu_res;
1290 }
1291
1292 void yf::Zoom::Frontend::handle_package(mp::Package &package)
1293 {
1294     Z_GDU *gdu = package.request().get();
1295     if (!gdu)
1296         ;
1297     else if (gdu->which == Z_GDU_Z3950)
1298     {
1299         Z_APDU *apdu_req = gdu->u.z3950;
1300         if (apdu_req->which == Z_APDU_initRequest)
1301         {
1302             mp::odr odr;
1303             package.response() = odr.create_close(
1304                 apdu_req,
1305                 Z_Close_protocolError,
1306                 "double init");
1307         }
1308         else if (apdu_req->which == Z_APDU_searchRequest)
1309         {
1310             handle_search(package);
1311         }
1312         else if (apdu_req->which == Z_APDU_presentRequest)
1313         {
1314             handle_present(package);
1315         }
1316         else
1317         {
1318             mp::odr odr;
1319             package.response() = odr.create_close(
1320                 apdu_req,
1321                 Z_Close_protocolError,
1322                 "zoom filter cannot handle this APDU");
1323             package.session().close();
1324         }
1325     }
1326     else
1327     {
1328         package.session().close();
1329     }
1330 }
1331
1332 void yf::Zoom::Impl::process(mp::Package &package)
1333 {
1334     FrontendPtr f = get_frontend(package);
1335     Z_GDU *gdu = package.request().get();
1336
1337     if (f->m_is_virtual)
1338     {
1339         f->handle_package(package);
1340     }
1341     else if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
1342              Z_APDU_initRequest)
1343     {
1344         Z_InitRequest *req = gdu->u.z3950->u.initRequest;
1345         f->m_init_gdu = gdu;
1346         
1347         mp::odr odr;
1348         Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
1349         Z_InitResponse *resp = apdu->u.initResponse;
1350         
1351         int i;
1352         static const int masks[] = {
1353             Z_Options_search,
1354             Z_Options_present,
1355             -1 
1356         };
1357         for (i = 0; masks[i] != -1; i++)
1358             if (ODR_MASK_GET(req->options, masks[i]))
1359                 ODR_MASK_SET(resp->options, masks[i]);
1360         
1361         static const int versions[] = {
1362             Z_ProtocolVersion_1,
1363             Z_ProtocolVersion_2,
1364             Z_ProtocolVersion_3,
1365             -1
1366         };
1367         for (i = 0; versions[i] != -1; i++)
1368             if (ODR_MASK_GET(req->protocolVersion, versions[i]))
1369                 ODR_MASK_SET(resp->protocolVersion, versions[i]);
1370             else
1371                 break;
1372         
1373         *resp->preferredMessageSize = *req->preferredMessageSize;
1374         *resp->maximumRecordSize = *req->maximumRecordSize;
1375         
1376         package.response() = apdu;
1377         f->m_is_virtual = true;
1378     }
1379     else
1380         package.move();
1381
1382     release_frontend(package);
1383 }
1384
1385
1386 static mp::filter::Base* filter_creator()
1387 {
1388     return new mp::filter::Zoom;
1389 }
1390
1391 extern "C" {
1392     struct metaproxy_1_filter_struct metaproxy_1_filter_zoom = {
1393         0,
1394         "zoom",
1395         filter_creator
1396     };
1397 }
1398
1399
1400 /*
1401  * Local variables:
1402  * c-basic-offset: 4
1403  * c-file-style: "Stroustrup"
1404  * indent-tabs-mode: nil
1405  * End:
1406  * vim: shiftwidth=4 tabstop=8 expandtab
1407  */
1408