Adding '(backend)' to addinfo for backend diagnostics
[metaproxy-moved-to-github.git] / src / filter_zoom.cpp
1 /* This file is part of Metaproxy.
2    Copyright (C) 2005-2011 Index Data
3
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17 */
18
19 #include "config.hpp"
20 #include "filter_zoom.hpp"
21 #include <yaz/zoom.h>
22 #include <yaz/yaz-version.h>
23 #include <yaz/srw.h>
24 #include <metaproxy/package.hpp>
25 #include <metaproxy/util.hpp>
26 #include <metaproxy/xmlutil.hpp>
27 #include "torus.hpp"
28
29 #include <libxslt/xsltutils.h>
30 #include <libxslt/transform.h>
31
32 #include <boost/thread/mutex.hpp>
33 #include <boost/thread/condition.hpp>
34 #include <yaz/ccl_xml.h>
35 #include <yaz/ccl.h>
36 #include <yaz/rpn2cql.h>
37 #include <yaz/rpn2solr.h>
38 #include <yaz/pquery.h>
39 #include <yaz/cql.h>
40 #include <yaz/oid_db.h>
41 #include <yaz/diagbib1.h>
42 #include <yaz/log.h>
43 #include <yaz/zgdu.h>
44 #include <yaz/querytowrbuf.h>
45
46 namespace mp = metaproxy_1;
47 namespace yf = mp::filter;
48
49 namespace metaproxy_1 {
50     namespace filter {
51         struct Zoom::Searchable : boost::noncopyable {
52             std::string authentication;
53             std::string cfAuth;
54             std::string cfProxy;
55             std::string cfSubDb;
56             std::string udb;
57             std::string target;
58             std::string query_encoding;
59             std::string sru;
60             std::string request_syntax;
61             std::string element_set;
62             std::string record_encoding;
63             std::string transform_xsl_fname;
64             std::string urlRecipe;
65             bool use_turbomarc;
66             bool piggyback;
67             CCL_bibset ccl_bibset;
68             Searchable(CCL_bibset base);
69             ~Searchable();
70         };
71         class Zoom::Backend : boost::noncopyable {
72             friend class Impl;
73             friend class Frontend;
74             std::string zurl;
75             ZOOM_connection m_connection;
76             ZOOM_resultset m_resultset;
77             std::string m_frontend_database;
78             SearchablePtr sptr;
79             xsltStylesheetPtr xsp;
80         public:
81             Backend(SearchablePtr sptr);
82             ~Backend();
83             void connect(std::string zurl, int *error, char **addinfo,
84                          ODR odr);
85             void search_pqf(const char *pqf, Odr_int *hits,
86                             int *error, char **addinfo, ODR odr);
87             void search_cql(const char *cql, Odr_int *hits,
88                             int *error, char **addinfo, ODR odr);
89             void present(Odr_int start, Odr_int number, ZOOM_record *recs,
90                          int *error, char **addinfo, ODR odr);
91             void set_option(const char *name, const char *value);
92             const char *get_option(const char *name);
93             void get_zoom_error(int *error, char **addinfo, ODR odr);
94         };
95         class Zoom::Frontend : boost::noncopyable {
96             friend class Impl;
97             Impl *m_p;
98             bool m_is_virtual;
99             bool m_in_use;
100             yazpp_1::GDU m_init_gdu;
101             BackendPtr m_backend;
102             void handle_package(mp::Package &package);
103             void handle_search(mp::Package &package);
104             void handle_present(mp::Package &package);
105             BackendPtr get_backend_from_databases(std::string &database,
106                                                   int *error,
107                                                   char **addinfo,
108                                                   ODR odr);
109             Z_Records *get_records(Odr_int start,
110                                    Odr_int number_to_present,
111                                    int *error,
112                                    char **addinfo,
113                                    Odr_int *number_of_records_returned,
114                                    ODR odr, BackendPtr b,
115                                    Odr_oid *preferredRecordSyntax,
116                                    const char *element_set_name);
117         public:
118             Frontend(Impl *impl);
119             ~Frontend();
120         };
121         class Zoom::Impl {
122             friend class Frontend;
123         public:
124             Impl();
125             ~Impl();
126             void process(metaproxy_1::Package & package);
127             void configure(const xmlNode * ptr, bool test_only,
128                            const char *path);
129         private:
130             void configure_local_records(const xmlNode * ptr, bool test_only);
131             FrontendPtr get_frontend(mp::Package &package);
132             void release_frontend(mp::Package &package);
133             SearchablePtr parse_torus_record(const xmlNode *ptr);
134             struct cql_node *convert_cql_fields(struct cql_node *cn, ODR odr);
135             std::map<mp::Session, FrontendPtr> m_clients;            
136             boost::mutex m_mutex;
137             boost::condition m_cond_session_ready;
138             std::string torus_url;
139             std::map<std::string,std::string> fieldmap;
140             std::string xsldir;
141             std::string file_path;
142             CCL_bibset bibset;
143             std::string element_transform;
144             std::string element_raw;
145             std::map<std::string,SearchablePtr> s_map;
146         };
147     }
148 }
149
150 // define Pimpl wrapper forwarding to Impl
151  
152 yf::Zoom::Zoom() : m_p(new Impl)
153 {
154 }
155
156 yf::Zoom::~Zoom()
157 {  // must have a destructor because of boost::scoped_ptr
158 }
159
160 void yf::Zoom::configure(const xmlNode *xmlnode, bool test_only,
161                          const char *path)
162 {
163     m_p->configure(xmlnode, test_only, path);
164 }
165
166 void yf::Zoom::process(mp::Package &package) const
167 {
168     m_p->process(package);
169 }
170
171
172 // define Implementation stuff
173
174 yf::Zoom::Backend::Backend(SearchablePtr ptr) : sptr(ptr)
175 {
176     m_connection = ZOOM_connection_create(0);
177     m_resultset = 0;
178     xsp = 0;
179 }
180
181 yf::Zoom::Backend::~Backend()
182 {
183     if (xsp)
184         xsltFreeStylesheet(xsp);
185     ZOOM_connection_destroy(m_connection);
186     ZOOM_resultset_destroy(m_resultset);
187 }
188
189
190 void yf::Zoom::Backend::get_zoom_error(int *error, char **addinfo,
191                                        ODR odr)
192 {
193     const char *msg = 0;
194     const char *zoom_addinfo = 0;
195     *error = ZOOM_connection_error(m_connection, &msg, &zoom_addinfo);
196     if (*error)
197     {
198         if (*error >= ZOOM_ERROR_CONNECT)
199         {
200             // turn ZOOM diagnostic into a Bib-1 2: with addinfo=zoom err msg
201             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
202             *addinfo = (char *) odr_malloc(
203                 odr, 20 + strlen(msg) + 
204                 (zoom_addinfo ? strlen(zoom_addinfo) : 0));
205             strcpy(*addinfo, msg);
206             if (zoom_addinfo)
207             {
208                 strcat(*addinfo, ": ");
209                 strcat(*addinfo, zoom_addinfo);
210                 strcat(*addinfo, " ");
211             }
212         }
213         else
214         {
215             *addinfo = (char *) odr_malloc(
216                 odr, 20 + (zoom_addinfo ? strlen(zoom_addinfo) : 0));
217             **addinfo = '\0';
218             if (zoom_addinfo && *zoom_addinfo)
219             {
220                 strcpy(*addinfo, zoom_addinfo);
221                 strcat(*addinfo, " ");
222             }
223             strcat(*addinfo, "(backend)");
224         }
225     }
226 }
227
228 void yf::Zoom::Backend::connect(std::string zurl,
229                                 int *error, char **addinfo,
230                                 ODR odr)
231 {
232     ZOOM_connection_connect(m_connection, zurl.c_str(), 0);
233     get_zoom_error(error, addinfo, odr);
234 }
235
236 void yf::Zoom::Backend::search_pqf(const char *pqf, Odr_int *hits,
237                                    int *error, char **addinfo, ODR odr)
238 {
239     m_resultset = ZOOM_connection_search_pqf(m_connection, pqf);
240     get_zoom_error(error, addinfo, odr);
241     if (*error == 0)
242         *hits = ZOOM_resultset_size(m_resultset);
243     else
244         *hits = 0;
245 }
246
247 void yf::Zoom::Backend::search_cql(const char *cql, Odr_int *hits,
248                                    int *error, char **addinfo, ODR odr)
249 {
250     ZOOM_query q = ZOOM_query_create();
251
252     ZOOM_query_cql(q, cql);
253
254     m_resultset = ZOOM_connection_search(m_connection, q);
255     ZOOM_query_destroy(q);
256     get_zoom_error(error, addinfo, odr);
257     if (*error == 0)
258         *hits = ZOOM_resultset_size(m_resultset);
259     else
260         *hits = 0;
261 }
262
263 void yf::Zoom::Backend::present(Odr_int start, Odr_int number,
264                                 ZOOM_record *recs,
265                                 int *error, char **addinfo, ODR odr)
266 {
267     ZOOM_resultset_records(m_resultset, recs, start, number);
268     get_zoom_error(error, addinfo, odr);
269 }
270
271 void yf::Zoom::Backend::set_option(const char *name, const char *value)
272 {
273     ZOOM_connection_option_set(m_connection, name, value);
274     if (m_resultset)
275         ZOOM_resultset_option_set(m_resultset, name, value);
276 }
277
278 const char *yf::Zoom::Backend::get_option(const char *name)
279 {
280     return ZOOM_connection_option_get(m_connection, name);
281 }
282
283 yf::Zoom::Searchable::Searchable(CCL_bibset base)
284 {
285     piggyback = true;
286     use_turbomarc = true;
287     ccl_bibset = ccl_qual_dup(base);
288 }
289
290 yf::Zoom::Searchable::~Searchable()
291 {
292     ccl_qual_rm(&ccl_bibset);
293 }
294
295 yf::Zoom::Frontend::Frontend(Impl *impl) : 
296     m_p(impl), m_is_virtual(false), m_in_use(true)
297 {
298 }
299
300 yf::Zoom::Frontend::~Frontend()
301 {
302 }
303
304 yf::Zoom::FrontendPtr yf::Zoom::Impl::get_frontend(mp::Package &package)
305 {
306     boost::mutex::scoped_lock lock(m_mutex);
307
308     std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
309     
310     while(true)
311     {
312         it = m_clients.find(package.session());
313         if (it == m_clients.end())
314             break;
315         
316         if (!it->second->m_in_use)
317         {
318             it->second->m_in_use = true;
319             return it->second;
320         }
321         m_cond_session_ready.wait(lock);
322     }
323     FrontendPtr f(new Frontend(this));
324     m_clients[package.session()] = f;
325     f->m_in_use = true;
326     return f;
327 }
328
329 void yf::Zoom::Impl::release_frontend(mp::Package &package)
330 {
331     boost::mutex::scoped_lock lock(m_mutex);
332     std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
333     
334     it = m_clients.find(package.session());
335     if (it != m_clients.end())
336     {
337         if (package.session().is_closed())
338         {
339             m_clients.erase(it);
340         }
341         else
342         {
343             it->second->m_in_use = false;
344         }
345         m_cond_session_ready.notify_all();
346     }
347 }
348
349 yf::Zoom::Impl::Impl() : element_transform("pz2") , element_raw("raw")
350 {
351     bibset = ccl_qual_mk();
352 }
353
354 yf::Zoom::Impl::~Impl()
355
356     ccl_qual_rm(&bibset);
357 }
358
359 yf::Zoom::SearchablePtr yf::Zoom::Impl::parse_torus_record(const xmlNode *ptr)
360 {
361     Zoom::SearchablePtr s(new Searchable(bibset));
362     
363     for (ptr = ptr->children; ptr; ptr = ptr->next)
364     {
365         if (ptr->type != XML_ELEMENT_NODE)
366             continue;
367         if (!strcmp((const char *) ptr->name, "layer"))
368             ptr = ptr->children;
369         else if (!strcmp((const char *) ptr->name,
370                          "authentication"))
371         {
372             s->authentication = mp::xml::get_text(ptr);
373         }
374         else if (!strcmp((const char *) ptr->name,
375                          "cfAuth"))
376         {
377             s->cfAuth = mp::xml::get_text(ptr);
378         } 
379         else if (!strcmp((const char *) ptr->name,
380                          "cfProxy"))
381         {
382             s->cfProxy = mp::xml::get_text(ptr);
383         }  
384         else if (!strcmp((const char *) ptr->name,
385                          "cfSubDb"))
386         {
387             s->cfSubDb = mp::xml::get_text(ptr);
388         }  
389         else if (!strcmp((const char *) ptr->name, "udb"))
390         {
391             s->udb = mp::xml::get_text(ptr);
392         }
393         else if (!strcmp((const char *) ptr->name, "zurl"))
394         {
395             s->target = mp::xml::get_text(ptr);
396         }
397         else if (!strcmp((const char *) ptr->name, "sru"))
398         {
399             s->sru = mp::xml::get_text(ptr);
400         }
401         else if (!strcmp((const char *) ptr->name,
402                          "queryEncoding"))
403         {
404             s->query_encoding = mp::xml::get_text(ptr);
405         }
406         else if (!strcmp((const char *) ptr->name,
407                          "piggyback"))
408         {
409             s->piggyback = mp::xml::get_bool(ptr, true);
410         }
411         else if (!strcmp((const char *) ptr->name,
412                          "requestSyntax"))
413         {
414             s->request_syntax = mp::xml::get_text(ptr);
415         }
416         else if (!strcmp((const char *) ptr->name,
417                          "elementSet"))
418         {
419             s->element_set = mp::xml::get_text(ptr);
420         }
421         else if (!strcmp((const char *) ptr->name,
422                          "recordEncoding"))
423         {
424             s->record_encoding = mp::xml::get_text(ptr);
425         }
426         else if (!strcmp((const char *) ptr->name,
427                          "transform"))
428         {
429             s->transform_xsl_fname = mp::xml::get_text(ptr);
430         }
431         else if (!strcmp((const char *) ptr->name,
432                          "urlRecipe"))
433         {
434             s->urlRecipe = mp::xml::get_text(ptr);
435         }
436         else if (!strcmp((const char *) ptr->name,
437                          "useTurboMarc"))
438         {
439             ; // useTurboMarc is ignored
440         }
441         else if (!strncmp((const char *) ptr->name,
442                           "cclmap_", 7))
443         {
444             std::string value = mp::xml::get_text(ptr);
445             ccl_qual_fitem(s->ccl_bibset, value.c_str(),
446                            (const char *) ptr->name + 7);
447         }
448     }
449     return s;
450 }
451
452 void yf::Zoom::Impl::configure_local_records(const xmlNode *ptr, bool test_only)
453 {
454     while (ptr && ptr->type != XML_ELEMENT_NODE)
455         ptr = ptr->next;
456     
457     if (ptr)
458     {
459         if (!strcmp((const char *) ptr->name, "records"))
460         {
461             for (ptr = ptr->children; ptr; ptr = ptr->next)
462             {
463                 if (ptr->type != XML_ELEMENT_NODE)
464                     continue;
465                 if (!strcmp((const char *) ptr->name, "record"))
466                 {
467                     SearchablePtr s = parse_torus_record(ptr);
468                     if (s)
469                     {
470                         std::string udb = s->udb;
471                         if (udb.length())
472                             s_map[s->udb] = s;
473                         else
474                         {
475                             throw mp::filter::FilterException
476                                 ("No udb for local torus record");
477                         }
478                     }
479                 }
480                 else
481                 {
482                     throw mp::filter::FilterException
483                         ("Bad element " 
484                          + std::string((const char *) ptr->name)
485                          + " in zoom filter inside element "
486                          "<torus><records>");
487                 }
488             }
489         }
490         else
491         {
492             throw mp::filter::FilterException
493                 ("Bad element " 
494                  + std::string((const char *) ptr->name)
495                  + " in zoom filter inside element <torus>");
496         }
497     }
498 }
499
500 void yf::Zoom::Impl::configure(const xmlNode *ptr, bool test_only,
501                                const char *path)
502 {
503     if (path && *path)
504     {
505         file_path = path;
506         if (path[strlen(path)-1] != '/')
507             file_path += "/";
508     }
509     for (ptr = ptr->children; ptr; ptr = ptr->next)
510     {
511         if (ptr->type != XML_ELEMENT_NODE)
512             continue;
513         else if (!strcmp((const char *) ptr->name, "torus"))
514         {
515             const struct _xmlAttr *attr;
516             for (attr = ptr->properties; attr; attr = attr->next)
517             {
518                 if (!strcmp((const char *) attr->name, "url"))
519                     torus_url = mp::xml::get_text(attr->children);
520                 else if (!strcmp((const char *) attr->name, "xsldir"))
521                     xsldir = mp::xml::get_text(attr->children);
522                 else if (!strcmp((const char *) attr->name, "element_transform"))
523                     element_transform = mp::xml::get_text(attr->children);
524                 else if (!strcmp((const char *) attr->name, "element_raw"))
525                     element_raw = mp::xml::get_text(attr->children);
526                 else
527                     throw mp::filter::FilterException(
528                         "Bad attribute " + std::string((const char *)
529                                                        attr->name));
530             }
531             configure_local_records(ptr->children, test_only);
532         }
533         else if (!strcmp((const char *) ptr->name, "cclmap"))
534         {
535             const char *addinfo = 0;
536             ccl_xml_config(bibset, ptr, &addinfo);
537         }
538         else if (!strcmp((const char *) ptr->name, "fieldmap"))
539         {
540             const struct _xmlAttr *attr;
541             std::string ccl_field;
542             std::string cql_field;
543             for (attr = ptr->properties; attr; attr = attr->next)
544             {
545                 if (!strcmp((const char *) attr->name, "ccl"))
546                     ccl_field = mp::xml::get_text(attr->children);
547                 else if (!strcmp((const char *) attr->name, "cql"))
548                     cql_field = mp::xml::get_text(attr->children);
549                 else
550                     throw mp::filter::FilterException(
551                         "Bad attribute " + std::string((const char *)
552                                                        attr->name));
553             }
554             if (cql_field.length())
555                 fieldmap[cql_field] = ccl_field;
556         }
557         else
558         {
559             throw mp::filter::FilterException
560                 ("Bad element " 
561                  + std::string((const char *) ptr->name)
562                  + " in zoom filter");
563         }
564     }
565 }
566
567 yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
568     std::string &database, int *error, char **addinfo, ODR odr)
569 {
570     std::list<BackendPtr>::const_iterator map_it;
571     if (m_backend && m_backend->m_frontend_database == database)
572         return m_backend;
573
574     const char *sru_proxy = 0;
575     std::string db_args;
576     std::string torus_db;
577     size_t db_arg_pos = database.find(',');
578     if (db_arg_pos != std::string::npos)
579     {
580         torus_db = database.substr(0, db_arg_pos);
581         db_args = database.substr(db_arg_pos + 1);
582     }
583     else
584         torus_db = database;
585  
586     SearchablePtr sptr;
587
588     std::map<std::string,SearchablePtr>::iterator it;
589     it = m_p->s_map.find(torus_db);
590     if (it != m_p->s_map.end())
591         sptr = it->second;
592     else
593     {
594         xmlDoc *doc = mp::get_searchable(m_p->torus_url, torus_db);
595         if (!doc)
596         {
597             *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
598             *addinfo = odr_strdup(odr, database.c_str());
599             BackendPtr b;
600             return b;
601         }
602         const xmlNode *ptr = xmlDocGetRootElement(doc);
603         if (ptr)
604         {   // presumably ptr is a records element node
605             // parse first record in document
606             for (ptr = ptr->children; ptr; ptr = ptr->next)
607             {
608                 if (ptr->type == XML_ELEMENT_NODE
609                     && !strcmp((const char *) ptr->name, "record"))
610                 {
611                     sptr = m_p->parse_torus_record(ptr);
612                     break;
613                 }
614             }
615         }
616         xmlFreeDoc(doc);
617     }
618
619     if (!sptr)
620     {
621         *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
622         *addinfo = odr_strdup(odr, database.c_str());
623         BackendPtr b;
624         return b;
625     }
626         
627     xsltStylesheetPtr xsp = 0;
628     if (sptr->transform_xsl_fname.length())
629     {
630         std::string fname;
631
632         if (m_p->xsldir.length()) 
633             fname = m_p->xsldir + "/" + sptr->transform_xsl_fname;
634         else
635             fname = m_p->file_path + sptr->transform_xsl_fname;
636         xmlDoc *xsp_doc = xmlParseFile(fname.c_str());
637         if (!xsp_doc)
638         {
639             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
640             *addinfo = (char *) odr_malloc(odr, 40 + strlen(fname.c_str()));
641             sprintf(*addinfo, "xmlParseFile failed. File %s", fname.c_str());
642             BackendPtr b;
643             return b;
644         }
645         xsp = xsltParseStylesheetDoc(xsp_doc);
646         if (!xsp)
647         {
648             *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
649             *addinfo = odr_strdup(odr, "xsltParseStylesheetDoc failed");
650             BackendPtr b;
651             xmlFreeDoc(xsp_doc);
652             return b;
653         }
654     }
655
656     m_backend.reset();
657
658     BackendPtr b(new Backend(sptr));
659
660     b->xsp = xsp;
661     b->m_frontend_database = database;
662     std::string authentication = sptr->authentication;
663         
664     b->set_option("timeout", "40");
665
666     if (sptr->query_encoding.length())
667         b->set_option("rpnCharset", sptr->query_encoding.c_str());
668
669     if (sptr->cfAuth.length())
670     {
671         // A CF target
672         b->set_option("user", sptr->cfAuth.c_str());
673         if (db_args.length() == 0)
674         {
675             if (authentication.length())
676             {
677                 // no database (auth) args specified already.. and the
678                 // Torus authentication has it.. Generate the args that CF
679                 // understands..
680                 size_t found = authentication.find('/');
681                 if (found != std::string::npos)
682                 {
683                     db_args += "user=" + mp::util::uri_encode(authentication.substr(0, found))
684                         + "&password=" + mp::util::uri_encode(authentication.substr(found+1));
685                 }
686                 else
687                     db_args += "user=" + mp::util::uri_encode(authentication);
688             }
689             if (sptr->cfProxy.length())
690             {
691                 if (db_args.length())
692                     db_args += "&";
693                 db_args += "proxy=" + mp::util::uri_encode(sptr->cfProxy);
694             }
695         }
696         if (sptr->cfSubDb.length())
697         {
698             if (db_args.length())
699                 db_args += "&";
700             db_args += "subdatabase=" + mp::util::uri_encode(sptr->cfSubDb);
701         }
702     }
703     else
704     {
705         // A non-CF target
706         if (db_args.length())
707         {
708             // user has specified backend authentication
709             const char *param_user = 0;
710             const char *param_password = 0;
711             char **names;
712             char **values;
713             int i;
714             int no_parms = yaz_uri_to_array(db_args.c_str(),
715                                             odr, &names, &values);
716             for (i = 0; i < no_parms; i++)
717             {
718                 const char *name = names[i];
719                 const char *value = values[i];
720                 if (!strcmp(name, "user"))
721                     param_user = value;
722                 else if (!strcmp(name, "password"))
723                     param_password = value;
724                 else if (!strcmp(name, "proxy"))
725                     sru_proxy = value;
726                 else
727                 {
728                     BackendPtr notfound;
729                     char *msg = (char*) odr_malloc(odr, strlen(name) + 30);
730                     *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
731                     sprintf(msg, "Bad database argument: %s", name);
732                     *addinfo = msg;
733                     return notfound;
734                 }
735             }
736             if (param_user && param_password)
737             {
738                 char *auth = (char*) odr_malloc(
739                     odr, strlen(param_user) + strlen(param_password) + 2);
740                 strcpy(auth, param_user);
741                 strcat(auth, "/");
742                 strcat(auth, param_password);
743                 b->set_option("user", auth);
744             }
745             db_args.clear(); // no arguments to be passed (non-CF)
746         }
747         else
748         {
749             // use authentication from Torus, if given
750             if (authentication.length())
751                 b->set_option("user", authentication.c_str());
752         }
753     }
754
755     if (sru_proxy)
756         b->set_option("proxy", sru_proxy);
757
758     std::string url;
759     if (sptr->sru.length())
760     {
761         url = "http://" + sptr->target;
762         b->set_option("sru", sptr->sru.c_str());
763     }
764     else
765     {
766         url = sptr->target;
767     }
768     if (db_args.length())
769         url += "," + db_args;
770     yaz_log(YLOG_LOG, "url=%s", url.c_str());
771     b->connect(url, error, addinfo, odr);
772     if (*error == 0)
773     {
774         m_backend = b;
775     }
776     return b;
777 }
778
779 Z_Records *yf::Zoom::Frontend::get_records(Odr_int start,
780                                            Odr_int number_to_present,
781                                            int *error,
782                                            char **addinfo,
783                                            Odr_int *number_of_records_returned,
784                                            ODR odr,
785                                            BackendPtr b,
786                                            Odr_oid *preferredRecordSyntax,
787                                            const char *element_set_name)
788 {
789     *number_of_records_returned = 0;
790     Z_Records *records = 0;
791     bool enable_pz2_retrieval = false; // whether target profile is used
792     bool enable_pz2_transform = false; // whether XSLT is used as well
793     bool assume_marc8_charset = false;
794
795     if (start < 0 || number_to_present <= 0)
796         return records;
797     
798     if (number_to_present > 10000)
799         number_to_present = 10000;
800     
801     ZOOM_record *recs = (ZOOM_record *)
802         odr_malloc(odr, number_to_present * sizeof(*recs));
803
804     char oid_name_str[OID_STR_MAX];
805     const char *syntax_name = 0;
806     
807     if (preferredRecordSyntax &&
808         !oid_oidcmp(preferredRecordSyntax, yaz_oid_recsyn_xml)
809         && element_set_name)
810     {
811         if (!strcmp(element_set_name, m_p->element_transform.c_str()))
812         {
813             enable_pz2_retrieval = true;
814             enable_pz2_transform = true;
815         }
816         else if (!strcmp(element_set_name, m_p->element_raw.c_str()))
817         {
818             enable_pz2_retrieval = true;
819         }
820     }
821     
822     if (enable_pz2_retrieval)
823     {
824         if (b->sptr->request_syntax.length())
825         {
826             syntax_name = b->sptr->request_syntax.c_str();
827             const Odr_oid *syntax_oid = 
828                 yaz_string_to_oid(yaz_oid_std(), CLASS_RECSYN, syntax_name);
829             if (!oid_oidcmp(syntax_oid, yaz_oid_recsyn_usmarc)
830                 || !oid_oidcmp(syntax_oid, yaz_oid_recsyn_opac))
831                 assume_marc8_charset = true;
832         }
833     }
834     else if (preferredRecordSyntax)
835         syntax_name =
836             yaz_oid_to_string_buf(preferredRecordSyntax, 0, oid_name_str);
837
838     b->set_option("preferredRecordSyntax", syntax_name);
839
840     if (enable_pz2_retrieval)
841     {
842         element_set_name = 0;
843         if (b->sptr->element_set.length())
844             element_set_name = b->sptr->element_set.c_str();
845     }
846
847     b->set_option("elementSetName", element_set_name);
848
849     b->present(start, number_to_present, recs, error, addinfo, odr);
850
851     Odr_int i = 0;
852     if (!*error)
853     {
854         for (i = 0; i < number_to_present; i++)
855             if (!recs[i])
856                 break;
857     }
858     if (i > 0)
859     {  // only return records if no error and at least one record
860         char *odr_database = odr_strdup(odr,
861                                         b->m_frontend_database.c_str());
862         Z_NamePlusRecordList *npl = (Z_NamePlusRecordList *)
863             odr_malloc(odr, sizeof(*npl));
864         *number_of_records_returned = i;
865         npl->num_records = i;
866         npl->records = (Z_NamePlusRecord **)
867             odr_malloc(odr, i * sizeof(*npl->records));
868         for (i = 0; i < number_to_present; i++)
869         {
870             Z_NamePlusRecord *npr = 0;
871             const char *addinfo;
872             int sur_error = ZOOM_record_error(recs[i], 0 /* msg */,
873                                               &addinfo, 0 /* diagset */);
874                 
875             if (sur_error)
876             {
877                 npr = zget_surrogateDiagRec(odr, odr_database, sur_error,
878                                             addinfo);
879             }
880             else if (enable_pz2_retrieval)
881             {
882                 char rec_type_str[100];
883                 const char *record_encoding = 0;
884
885                 if (b->sptr->record_encoding.length())
886                     record_encoding = b->sptr->record_encoding.c_str();
887                 else if (assume_marc8_charset)
888                     record_encoding = "marc8";
889
890                 strcpy(rec_type_str, b->sptr->use_turbomarc ? "txml" : "xml");
891                 if (record_encoding)
892                 {
893                     strcat(rec_type_str, "; charset=");
894                     strcat(rec_type_str, record_encoding);
895                 }
896                 
897                 int rec_len;
898                 xmlChar *xmlrec_buf = 0;
899                 const char *rec_buf = ZOOM_record_get(recs[i], rec_type_str,
900                                                       &rec_len);
901                 if (rec_buf && b->xsp && enable_pz2_transform)
902                 {
903                     xmlDoc *rec_doc = xmlParseMemory(rec_buf, rec_len);
904                     if (rec_doc)
905                     { 
906                         xmlDoc *rec_res;
907                         rec_res = xsltApplyStylesheet(b->xsp, rec_doc, 0);
908
909                         if (rec_res)
910                             xsltSaveResultToString(&xmlrec_buf, &rec_len,
911                                                    rec_res, b->xsp);
912                         rec_buf = (const char *) xmlrec_buf;
913                         xmlFreeDoc(rec_doc);
914                         xmlFreeDoc(rec_res);
915                     }
916                 }
917
918                 if (rec_buf)
919                 {
920                     xmlDoc *doc = xmlParseMemory(rec_buf, rec_len);
921                     std::string res = 
922                         mp::xml::url_recipe_handle(doc, b->sptr->urlRecipe);
923                     if (res.length())
924                     {
925                         xmlNode *ptr = xmlDocGetRootElement(doc);
926                         while (ptr && ptr->type != XML_ELEMENT_NODE)
927                             ptr = ptr->next;
928                         xmlNode *c = 
929                             xmlNewChild(ptr, 0, BAD_CAST "generated-url", 0);
930                         xmlNode * t = xmlNewText(BAD_CAST res.c_str());
931                         xmlAddChild(c, t);
932
933                         if (xmlrec_buf)
934                             xmlFree(xmlrec_buf);
935
936                         xmlDocDumpMemory(doc, &xmlrec_buf, &rec_len);
937                         rec_buf = (const char *) xmlrec_buf;
938                     }
939                     xmlFreeDoc(doc);
940                 }
941                 if (rec_buf)
942                 {
943                     npr = (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
944                     npr->databaseName = odr_database;
945                     npr->which = Z_NamePlusRecord_databaseRecord;
946                     npr->u.databaseRecord =
947                         z_ext_record_xml(odr, rec_buf, rec_len);
948                 }
949                 else
950                 {
951                     npr = zget_surrogateDiagRec(
952                         odr, odr_database, 
953                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
954                         rec_type_str);
955                 }
956                 if (xmlrec_buf)
957                     xmlFree(xmlrec_buf);
958             }
959             else
960             {
961                 Z_External *ext =
962                     (Z_External *) ZOOM_record_get(recs[i], "ext", 0);
963                 if (ext)
964                 {
965                     npr = (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
966                     npr->databaseName = odr_database;
967                     npr->which = Z_NamePlusRecord_databaseRecord;
968                     npr->u.databaseRecord = ext;
969                 }
970                 else
971                 {
972                     npr = zget_surrogateDiagRec(
973                         odr, odr_database, 
974                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
975                         "ZOOM_record, type ext");
976                 }
977             }
978             npl->records[i] = npr;
979         }
980         records = (Z_Records*) odr_malloc(odr, sizeof(*records));
981         records->which = Z_Records_DBOSD;
982         records->u.databaseOrSurDiagnostics = npl;
983     }
984     return records;
985 }
986     
987 struct cql_node *yf::Zoom::Impl::convert_cql_fields(struct cql_node *cn,
988                                                     ODR odr)
989 {
990     struct cql_node *r = 0;
991     if (!cn)
992         return 0;
993     switch (cn->which)
994     {
995     case CQL_NODE_ST:
996         if (cn->u.st.index)
997         {
998             std::map<std::string,std::string>::const_iterator it;
999             it = fieldmap.find(cn->u.st.index);
1000             if (it == fieldmap.end())
1001                 return cn;
1002             if (it->second.length())
1003                 cn->u.st.index = odr_strdup(odr, it->second.c_str());
1004             else
1005                 cn->u.st.index = 0;
1006         }
1007         break;
1008     case CQL_NODE_BOOL:
1009         r = convert_cql_fields(cn->u.boolean.left, odr);
1010         if (!r)
1011             r = convert_cql_fields(cn->u.boolean.right, odr);
1012         break;
1013     case CQL_NODE_SORT:
1014         r = convert_cql_fields(cn->u.sort.search, odr);
1015         break;
1016     }
1017     return r;
1018 }
1019
1020 void yf::Zoom::Frontend::handle_search(mp::Package &package)
1021 {
1022     Z_GDU *gdu = package.request().get();
1023     Z_APDU *apdu_req = gdu->u.z3950;
1024     Z_APDU *apdu_res = 0;
1025     mp::odr odr;
1026     Z_SearchRequest *sr = apdu_req->u.searchRequest;
1027     if (sr->num_databaseNames != 1)
1028     {
1029         apdu_res = odr.create_searchResponse(
1030             apdu_req, YAZ_BIB1_TOO_MANY_DATABASES_SPECIFIED, 0);
1031         package.response() = apdu_res;
1032         return;
1033     }
1034
1035     int error = 0;
1036     char *addinfo = 0;
1037     std::string db(sr->databaseNames[0]);
1038     BackendPtr b = get_backend_from_databases(db, &error, &addinfo, odr);
1039     if (error)
1040     {
1041         apdu_res = 
1042             odr.create_searchResponse(apdu_req, error, addinfo);
1043         package.response() = apdu_res;
1044         return;
1045     }
1046
1047     b->set_option("setname", "default");
1048
1049     Odr_int hits = 0;
1050     Z_Query *query = sr->query;
1051     WRBUF ccl_wrbuf = 0;
1052     WRBUF pqf_wrbuf = 0;
1053
1054     if (query->which == Z_Query_type_1 || query->which == Z_Query_type_101)
1055     {
1056         // RPN
1057         pqf_wrbuf = wrbuf_alloc();
1058         yaz_rpnquery_to_wrbuf(pqf_wrbuf, query->u.type_1);
1059     }
1060     else if (query->which == Z_Query_type_2)
1061     {
1062         // CCL
1063         ccl_wrbuf = wrbuf_alloc();
1064         wrbuf_write(ccl_wrbuf, (const char *) query->u.type_2->buf,
1065                     query->u.type_2->len);
1066     }
1067     else if (query->which == Z_Query_type_104 &&
1068              query->u.type_104->which == Z_External_CQL)
1069     {
1070         // CQL
1071         const char *cql = query->u.type_104->u.cql;
1072         CQL_parser cp = cql_parser_create();
1073         int r = cql_parser_string(cp, cql);
1074         if (r)
1075         {
1076             cql_parser_destroy(cp);
1077             apdu_res = 
1078                 odr.create_searchResponse(apdu_req, 
1079                                           YAZ_BIB1_MALFORMED_QUERY,
1080                                           "CQL syntax error");
1081             package.response() = apdu_res;
1082             return;
1083         }
1084         struct cql_node *cn = cql_parser_result(cp);
1085         struct cql_node *cn_error = m_p->convert_cql_fields(cn, odr);
1086         if (cn_error)
1087         {
1088             // hopefully we are getting a ptr to a index+relation+term node
1089             addinfo = 0;
1090             if (cn_error->which == CQL_NODE_ST)
1091                 addinfo = cn_error->u.st.index;
1092
1093             apdu_res = 
1094                 odr.create_searchResponse(apdu_req, 
1095                                           YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1096                                           addinfo);
1097             package.response() = apdu_res;
1098             return;
1099         }
1100         char ccl_buf[1024];
1101
1102         r = cql_to_ccl_buf(cn, ccl_buf, sizeof(ccl_buf));
1103         if (r == 0)
1104         {
1105             ccl_wrbuf = wrbuf_alloc();
1106             wrbuf_puts(ccl_wrbuf, ccl_buf);
1107         }
1108         cql_parser_destroy(cp);
1109         if (r)
1110         {
1111             apdu_res = 
1112                 odr.create_searchResponse(apdu_req, 
1113                                           YAZ_BIB1_MALFORMED_QUERY,
1114                                           "CQL to CCL conversion error");
1115             package.response() = apdu_res;
1116             return;
1117         }
1118     }
1119     else
1120     {
1121         apdu_res = 
1122             odr.create_searchResponse(apdu_req, YAZ_BIB1_QUERY_TYPE_UNSUPP, 0);
1123         package.response() = apdu_res;
1124         return;
1125     }
1126
1127     if (ccl_wrbuf)
1128     {
1129         // CCL to PQF
1130         assert(pqf_wrbuf == 0);
1131         int cerror, cpos;
1132         struct ccl_rpn_node *cn;
1133         yaz_log(YLOG_LOG, "CCL: %s", wrbuf_cstr(ccl_wrbuf));
1134         cn = ccl_find_str(b->sptr->ccl_bibset, wrbuf_cstr(ccl_wrbuf),
1135                           &cerror, &cpos);
1136         wrbuf_destroy(ccl_wrbuf);
1137         if (!cn)
1138         {
1139             char *addinfo = odr_strdup(odr, ccl_err_msg(cerror));
1140             int z3950_diag = YAZ_BIB1_MALFORMED_QUERY;
1141
1142             switch (cerror)
1143             {
1144             case CCL_ERR_UNKNOWN_QUAL:
1145                 z3950_diag = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1146                 break;
1147             case CCL_ERR_TRUNC_NOT_LEFT: 
1148             case CCL_ERR_TRUNC_NOT_RIGHT:
1149             case CCL_ERR_TRUNC_NOT_BOTH:
1150                 z3950_diag = YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE;
1151                 break;
1152             }
1153             apdu_res = 
1154                 odr.create_searchResponse(apdu_req, z3950_diag, addinfo);
1155             package.response() = apdu_res;
1156             return;
1157         }
1158         pqf_wrbuf = wrbuf_alloc();
1159         ccl_pquery(pqf_wrbuf, cn);
1160         ccl_rpn_delete(cn);
1161     }
1162     
1163     assert(pqf_wrbuf);
1164     if (b->get_option("sru"))
1165     {
1166         int status = 0;
1167         Z_RPNQuery *zquery;
1168         zquery = p_query_rpn(odr, wrbuf_cstr(pqf_wrbuf));
1169         WRBUF wrb = wrbuf_alloc();
1170             
1171         if (!strcmp(b->get_option("sru"), "solr"))
1172         {
1173             solr_transform_t cqlt = solr_transform_create();
1174             
1175             status = solr_transform_rpn2solr_wrbuf(cqlt, wrb, zquery);
1176             
1177             solr_transform_close(cqlt);
1178         }
1179         else
1180         {
1181             cql_transform_t cqlt = cql_transform_create();
1182             
1183             status = cql_transform_rpn2cql_wrbuf(cqlt, wrb, zquery);
1184             
1185             cql_transform_close(cqlt);
1186         }
1187         if (status == 0)
1188         {
1189             yaz_log(YLOG_LOG, "search CQL: %s", wrbuf_cstr(wrb));
1190             b->search_cql(wrbuf_cstr(wrb), &hits, &error, &addinfo, odr);
1191         }
1192         
1193         wrbuf_destroy(wrb);
1194         wrbuf_destroy(pqf_wrbuf);
1195         if (status)
1196         {
1197             apdu_res = 
1198                 odr.create_searchResponse(apdu_req, YAZ_BIB1_MALFORMED_QUERY,
1199                                           "can not convert from RPN to CQL/SOLR");
1200             package.response() = apdu_res;
1201             return;
1202         }
1203     }
1204     else
1205     {
1206         yaz_log(YLOG_LOG, "search PQF: %s", wrbuf_cstr(pqf_wrbuf));
1207         b->search_pqf(wrbuf_cstr(pqf_wrbuf), &hits, &error, &addinfo, odr);
1208         wrbuf_destroy(pqf_wrbuf);
1209     }
1210     
1211     
1212     const char *element_set_name = 0;
1213     Odr_int number_to_present = 0;
1214     if (!error)
1215         mp::util::piggyback_sr(sr, hits, number_to_present, &element_set_name);
1216     
1217     Odr_int number_of_records_returned = 0;
1218     Z_Records *records = get_records(
1219         0, number_to_present, &error, &addinfo,
1220         &number_of_records_returned, odr, b, sr->preferredRecordSyntax,
1221         element_set_name);
1222     apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
1223     if (records)
1224     {
1225         apdu_res->u.searchResponse->records = records;
1226         apdu_res->u.searchResponse->numberOfRecordsReturned =
1227             odr_intdup(odr, number_of_records_returned);
1228     }
1229     apdu_res->u.searchResponse->resultCount = odr_intdup(odr, hits);
1230     package.response() = apdu_res;
1231 }
1232
1233 void yf::Zoom::Frontend::handle_present(mp::Package &package)
1234 {
1235     Z_GDU *gdu = package.request().get();
1236     Z_APDU *apdu_req = gdu->u.z3950;
1237     Z_APDU *apdu_res = 0;
1238     Z_PresentRequest *pr = apdu_req->u.presentRequest;
1239
1240     mp::odr odr;
1241     if (!m_backend)
1242     {
1243         package.response() = odr.create_presentResponse(
1244             apdu_req, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST, 0);
1245         return;
1246     }
1247     const char *element_set_name = 0;
1248     Z_RecordComposition *comp = pr->recordComposition;
1249     if (comp && comp->which != Z_RecordComp_simple)
1250     {
1251         package.response() = odr.create_presentResponse(
1252             apdu_req, 
1253             YAZ_BIB1_PRESENT_COMP_SPEC_PARAMETER_UNSUPP, 0);
1254         return;
1255     }
1256     if (comp && comp->u.simple->which == Z_ElementSetNames_generic)
1257         element_set_name = comp->u.simple->u.generic;
1258     Odr_int number_of_records_returned = 0;
1259     int error = 0;
1260     char *addinfo = 0;
1261     Z_Records *records = get_records(
1262         *pr->resultSetStartPoint - 1, *pr->numberOfRecordsRequested,
1263         &error, &addinfo, &number_of_records_returned, odr, m_backend,
1264         pr->preferredRecordSyntax, element_set_name);
1265
1266     apdu_res = odr.create_presentResponse(apdu_req, error, addinfo);
1267     if (records)
1268     {
1269         apdu_res->u.presentResponse->records = records;
1270         apdu_res->u.presentResponse->numberOfRecordsReturned =
1271             odr_intdup(odr, number_of_records_returned);
1272     }
1273     package.response() = apdu_res;
1274 }
1275
1276 void yf::Zoom::Frontend::handle_package(mp::Package &package)
1277 {
1278     Z_GDU *gdu = package.request().get();
1279     if (!gdu)
1280         ;
1281     else if (gdu->which == Z_GDU_Z3950)
1282     {
1283         Z_APDU *apdu_req = gdu->u.z3950;
1284         if (apdu_req->which == Z_APDU_initRequest)
1285         {
1286             mp::odr odr;
1287             package.response() = odr.create_close(
1288                 apdu_req,
1289                 Z_Close_protocolError,
1290                 "double init");
1291         }
1292         else if (apdu_req->which == Z_APDU_searchRequest)
1293         {
1294             handle_search(package);
1295         }
1296         else if (apdu_req->which == Z_APDU_presentRequest)
1297         {
1298             handle_present(package);
1299         }
1300         else
1301         {
1302             mp::odr odr;
1303             package.response() = odr.create_close(
1304                 apdu_req,
1305                 Z_Close_protocolError,
1306                 "zoom filter cannot handle this APDU");
1307             package.session().close();
1308         }
1309     }
1310     else
1311     {
1312         package.session().close();
1313     }
1314 }
1315
1316 void yf::Zoom::Impl::process(mp::Package &package)
1317 {
1318     FrontendPtr f = get_frontend(package);
1319     Z_GDU *gdu = package.request().get();
1320
1321     if (f->m_is_virtual)
1322     {
1323         f->handle_package(package);
1324     }
1325     else if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
1326              Z_APDU_initRequest)
1327     {
1328         Z_InitRequest *req = gdu->u.z3950->u.initRequest;
1329         f->m_init_gdu = gdu;
1330         
1331         mp::odr odr;
1332         Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
1333         Z_InitResponse *resp = apdu->u.initResponse;
1334         
1335         int i;
1336         static const int masks[] = {
1337             Z_Options_search,
1338             Z_Options_present,
1339             -1 
1340         };
1341         for (i = 0; masks[i] != -1; i++)
1342             if (ODR_MASK_GET(req->options, masks[i]))
1343                 ODR_MASK_SET(resp->options, masks[i]);
1344         
1345         static const int versions[] = {
1346             Z_ProtocolVersion_1,
1347             Z_ProtocolVersion_2,
1348             Z_ProtocolVersion_3,
1349             -1
1350         };
1351         for (i = 0; versions[i] != -1; i++)
1352             if (ODR_MASK_GET(req->protocolVersion, versions[i]))
1353                 ODR_MASK_SET(resp->protocolVersion, versions[i]);
1354             else
1355                 break;
1356         
1357         *resp->preferredMessageSize = *req->preferredMessageSize;
1358         *resp->maximumRecordSize = *req->maximumRecordSize;
1359         
1360         package.response() = apdu;
1361         f->m_is_virtual = true;
1362     }
1363     else
1364         package.move();
1365
1366     release_frontend(package);
1367 }
1368
1369
1370 static mp::filter::Base* filter_creator()
1371 {
1372     return new mp::filter::Zoom;
1373 }
1374
1375 extern "C" {
1376     struct metaproxy_1_filter_struct metaproxy_1_filter_zoom = {
1377         0,
1378         "zoom",
1379         filter_creator
1380     };
1381 }
1382
1383
1384 /*
1385  * Local variables:
1386  * c-basic-offset: 4
1387  * c-file-style: "Stroustrup"
1388  * indent-tabs-mode: nil
1389  * End:
1390  * vim: shiftwidth=4 tabstop=8 expandtab
1391  */
1392