zoom: Torus setting: authenticationMode
[metaproxy-moved-to-github.git] / src / filter_zoom.cpp
1 /* This file is part of Metaproxy.
2    Copyright (C) 2005-2013 Index Data
3
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17 */
18
19 #include "config.hpp"
20
21 #include <stdlib.h>
22 #include <sys/types.h>
23 #include "filter_zoom.hpp"
24 #include <metaproxy/package.hpp>
25 #include <metaproxy/util.hpp>
26 #include <metaproxy/xmlutil.hpp>
27 #include <yaz/comstack.h>
28 #include <yaz/poll.h>
29 #include "torus.hpp"
30
31 #include <libxslt/xsltutils.h>
32 #include <libxslt/transform.h>
33
34 #include <boost/thread/mutex.hpp>
35 #include <boost/thread/condition.hpp>
36
37 #include <yaz/yaz-version.h>
38 #include <yaz/tpath.h>
39 #include <yaz/srw.h>
40 #include <yaz/ccl_xml.h>
41 #include <yaz/ccl.h>
42 #include <yaz/rpn2cql.h>
43 #include <yaz/rpn2solr.h>
44 #include <yaz/pquery.h>
45 #include <yaz/cql.h>
46 #include <yaz/oid_db.h>
47 #include <yaz/diagbib1.h>
48 #include <yaz/log.h>
49 #include <yaz/zgdu.h>
50 #include <yaz/querytowrbuf.h>
51 #include <yaz/sortspec.h>
52 #include <yaz/tokenizer.h>
53 #include <yaz/zoom.h>
54
55 namespace mp = metaproxy_1;
56 namespace yf = mp::filter;
57
58 namespace metaproxy_1 {
59     namespace filter {
60         class Zoom::Searchable : boost::noncopyable {
61           public:
62             std::string authentication;
63             std::string authenticationMode;
64             std::string cfAuth;
65             std::string cfProxy;
66             std::string cfSubDB;
67             std::string udb;
68             std::string target;
69             std::string query_encoding;
70             std::string sru;
71             std::string sru_version;
72             std::string request_syntax;
73             std::string element_set;
74             std::string record_encoding;
75             std::string transform_xsl_fname;
76             std::string transform_xsl_content;
77             std::string urlRecipe;
78             std::string contentConnector;
79             std::string sortStrategy;
80             std::string extraArgs;
81             std::string rpn2cql_fname;
82             bool use_turbomarc;
83             bool piggyback;
84             CCL_bibset ccl_bibset;
85             std::map<std::string, std::string> sortmap;
86             Searchable(CCL_bibset base);
87             ~Searchable();
88         };
89         class Zoom::Backend : boost::noncopyable {
90             friend class Impl;
91             friend class Frontend;
92             std::string zurl;
93             mp::wrbuf m_apdu_wrbuf;
94             ZOOM_connection m_connection;
95             ZOOM_resultset m_resultset;
96             std::string m_frontend_database;
97             SearchablePtr sptr;
98             xsltStylesheetPtr xsp;
99             std::string cproxy_host;
100             bool enable_cproxy;
101             bool enable_explain;
102             xmlDoc *explain_doc;
103             std::string m_proxy;
104             cql_transform_t cqlt;
105         public:
106             Backend();
107             ~Backend();
108             void connect(std::string zurl, int *error, char **addinfo,
109                          ODR odr);
110             void search(ZOOM_query q, Odr_int *hits,
111                         int *error, char **addinfo, ODR odr);
112             void present(Odr_int start, Odr_int number, ZOOM_record *recs,
113                          int *error, char **addinfo, ODR odr);
114             void set_option(const char *name, const char *value);
115             void set_option(const char *name, std::string value);
116             const char *get_option(const char *name);
117             void get_zoom_error(int *error, char **addinfo, ODR odr);
118         };
119         class Zoom::Frontend : boost::noncopyable {
120             friend class Impl;
121             Impl *m_p;
122             bool m_is_virtual;
123             bool m_in_use;
124             std::string session_realm;
125             yazpp_1::GDU m_init_gdu;
126             BackendPtr m_backend;
127             void handle_package(mp::Package &package);
128             void handle_search(mp::Package &package);
129
130             void auth(mp::Package &package, Z_InitRequest *req,
131                       int *error, char **addinfo, ODR odr);
132
133             BackendPtr explain_search(mp::Package &package,
134                                       std::string &database,
135                                       int *error,
136                                       char **addinfo,
137                                       mp::odr &odr,
138                                       std::string torus_url,
139                                       std::string &torus_db,
140                                       std::string &realm);
141             void handle_present(mp::Package &package);
142             BackendPtr get_backend_from_databases(mp::Package &package,
143                                                   std::string &database,
144                                                   int *error,
145                                                   char **addinfo,
146                                                   mp::odr &odr,
147                                                   int *proxy_step);
148
149             bool create_content_session(mp::Package &package,
150                                         BackendPtr b,
151                                         int *error,
152                                         char **addinfo,
153                                         ODR odr,
154                                         std::string authentication,
155                                         std::string proxy,
156                                         std::string realm);
157
158             void prepare_elements(BackendPtr b,
159                                   Odr_oid *preferredRecordSyntax,
160                                   const char *element_set_name,
161                                   bool &enable_pz2_retrieval,
162                                   bool &enable_pz2_transform,
163                                   bool &enable_record_transform,
164                                   bool &assume_marc8_charset);
165
166             Z_Records *get_records(Package &package,
167                                    Odr_int start,
168                                    Odr_int number_to_present,
169                                    int *error,
170                                    char **addinfo,
171                                    Odr_int *number_of_records_returned,
172                                    ODR odr, BackendPtr b,
173                                    Odr_oid *preferredRecordSyntax,
174                                    const char *element_set_name);
175             Z_Records *get_explain_records(Package &package,
176                                            Odr_int start,
177                                            Odr_int number_to_present,
178                                            int *error,
179                                            char **addinfo,
180                                            Odr_int *number_of_records_returned,
181                                            ODR odr, BackendPtr b,
182                                            Odr_oid *preferredRecordSyntax,
183                                            const char *element_set_name);
184             bool retry(mp::Package &package,
185                        mp::odr &odr,
186                        BackendPtr b,
187                        int &error, char **addinfo,
188                        int &proxy_step, int &same_retries,
189                        int &proxy_retries);
190             void log_diagnostic(mp::Package &package,
191                                 int error, const char *addinfo);
192         public:
193             Frontend(Impl *impl);
194             ~Frontend();
195         };
196         class Zoom::Impl {
197             friend class Frontend;
198         public:
199             Impl();
200             ~Impl();
201             void process(metaproxy_1::Package & package);
202             void configure(const xmlNode * ptr, bool test_only,
203                            const char *path);
204         private:
205             void configure_local_records(const xmlNode * ptr, bool test_only);
206             bool check_proxy(const char *proxy);
207
208
209
210             FrontendPtr get_frontend(mp::Package &package);
211             void release_frontend(mp::Package &package);
212             SearchablePtr parse_torus_record(const xmlNode *ptr);
213             struct cql_node *convert_cql_fields(struct cql_node *cn, ODR odr);
214             std::map<mp::Session, FrontendPtr> m_clients;
215             boost::mutex m_mutex;
216             boost::condition m_cond_session_ready;
217             std::string torus_searchable_url;
218             std::string torus_content_url;
219             std::string torus_auth_url;
220             std::string default_realm;
221             std::map<std::string,std::string> fieldmap;
222             std::string xsldir;
223             std::string file_path;
224             std::string content_proxy_server;
225             std::string content_tmp_file;
226             std::string content_config_file;
227             bool apdu_log;
228             CCL_bibset bibset;
229             std::string element_transform;
230             std::string element_raw;
231             std::string element_passthru;
232             std::string proxy;
233             xsltStylesheetPtr explain_xsp;
234             xsltStylesheetPtr record_xsp;
235             std::map<std::string,SearchablePtr> s_map;
236             std::string zoom_timeout;
237             int proxy_timeout;
238         };
239     }
240 }
241
242
243 static xmlNode *xml_node_search(xmlNode *ptr, int *num, int m)
244 {
245     while (ptr)
246     {
247         if (ptr->type == XML_ELEMENT_NODE &&
248             !strcmp((const char *) ptr->name, "recordData"))
249         {
250             (*num)++;
251             if (m == *num)
252                 return ptr;
253         }
254         else  // else: we don't want to find nested nodes
255         {
256             xmlNode *ret_node = xml_node_search(ptr->children, num, m);
257             if (ret_node)
258                 return ret_node;
259         }
260         ptr = ptr->next;
261     }
262     return 0;
263 }
264
265 // define Pimpl wrapper forwarding to Impl
266
267 yf::Zoom::Zoom() : m_p(new Impl)
268 {
269 }
270
271 yf::Zoom::~Zoom()
272 {  // must have a destructor because of boost::scoped_ptr
273 }
274
275 void yf::Zoom::configure(const xmlNode *xmlnode, bool test_only,
276                          const char *path)
277 {
278     m_p->configure(xmlnode, test_only, path);
279 }
280
281 void yf::Zoom::process(mp::Package &package) const
282 {
283     m_p->process(package);
284 }
285
286
287 // define Implementation stuff
288
289 yf::Zoom::Backend::Backend()
290 {
291     m_connection = ZOOM_connection_create(0);
292     ZOOM_connection_save_apdu_wrbuf(m_connection, m_apdu_wrbuf);
293     m_resultset = 0;
294     xsp = 0;
295     enable_cproxy = true;
296     enable_explain = false;
297     explain_doc = 0;
298     cqlt = 0;
299 }
300
301 yf::Zoom::Backend::~Backend()
302 {
303     if (xsp)
304         xsltFreeStylesheet(xsp);
305     if (explain_doc)
306         xmlFreeDoc(explain_doc);
307     cql_transform_close(cqlt);
308     ZOOM_connection_destroy(m_connection);
309     ZOOM_resultset_destroy(m_resultset);
310 }
311
312
313 void yf::Zoom::Backend::get_zoom_error(int *error, char **addinfo,
314                                        ODR odr)
315 {
316     const char *msg = 0;
317     const char *zoom_addinfo = 0;
318     const char *dset = 0;
319     int error0 = ZOOM_connection_error_x(m_connection, &msg,
320                                          &zoom_addinfo, &dset);
321     if (error0)
322     {
323         if (!dset)
324             dset = "Unknown";
325
326         if (!strcmp(dset, "info:srw/diagnostic/1"))
327             *error = yaz_diag_srw_to_bib1(error0);
328         else if (!strcmp(dset, "Bib-1"))
329             *error = error0;
330         else if (!strcmp(dset, "ZOOM"))
331         {
332             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
333             if (error0 == ZOOM_ERROR_INIT)
334                 *error = YAZ_BIB1_INIT_AC_AUTHENTICATION_SYSTEM_ERROR;
335             else if (error0 == ZOOM_ERROR_DECODE)
336             {
337                 if (zoom_addinfo)
338                 {
339                     if (strstr(zoom_addinfo, "Authentication") ||
340                         strstr(zoom_addinfo, "authentication"))
341                         *error = YAZ_BIB1_INIT_AC_AUTHENTICATION_SYSTEM_ERROR;
342                 }
343             }
344         }
345         else
346             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
347
348         *addinfo = (char *) odr_malloc(
349             odr, 30 + strlen(dset) + strlen(msg) +
350             (zoom_addinfo ? strlen(zoom_addinfo) : 0));
351         **addinfo = '\0';
352         if (zoom_addinfo && *zoom_addinfo)
353         {
354             strcpy(*addinfo, zoom_addinfo);
355             strcat(*addinfo, " ");
356         }
357         sprintf(*addinfo + strlen(*addinfo), "(%s %d %s)", dset, error0, msg);
358     }
359 }
360
361 void yf::Zoom::Backend::connect(std::string zurl,
362                                 int *error, char **addinfo,
363                                 ODR odr)
364 {
365     size_t h = zurl.find_first_of('#');
366     if (h != std::string::npos)
367         zurl.erase(h);
368     ZOOM_connection_connect(m_connection, zurl.length() ? zurl.c_str() : 0, 0);
369     get_zoom_error(error, addinfo, odr);
370
371 }
372
373 void yf::Zoom::Backend::search(ZOOM_query q, Odr_int *hits,
374                                int *error, char **addinfo, ODR odr)
375 {
376     ZOOM_resultset_destroy(m_resultset);
377     m_resultset = ZOOM_connection_search(m_connection, q);
378     get_zoom_error(error, addinfo, odr);
379     if (*error == 0)
380         *hits = ZOOM_resultset_size(m_resultset);
381     else
382         *hits = 0;
383 }
384
385 void yf::Zoom::Backend::present(Odr_int start, Odr_int number,
386                                 ZOOM_record *recs,
387                                 int *error, char **addinfo, ODR odr)
388 {
389     ZOOM_resultset_records(m_resultset, recs, start, number);
390     get_zoom_error(error, addinfo, odr);
391 }
392
393 void yf::Zoom::Backend::set_option(const char *name, const char *value)
394 {
395     ZOOM_connection_option_set(m_connection, name, value);
396     if (m_resultset)
397         ZOOM_resultset_option_set(m_resultset, name, value);
398 }
399
400 void yf::Zoom::Backend::set_option(const char *name, std::string value)
401 {
402     set_option(name, value.c_str());
403 }
404
405 const char *yf::Zoom::Backend::get_option(const char *name)
406 {
407     return ZOOM_connection_option_get(m_connection, name);
408 }
409
410 yf::Zoom::Searchable::Searchable(CCL_bibset base)
411 {
412     piggyback = true;
413     use_turbomarc = true;
414     sortStrategy = "embed";
415     ccl_bibset = ccl_qual_dup(base);
416 }
417
418 yf::Zoom::Searchable::~Searchable()
419 {
420     ccl_qual_rm(&ccl_bibset);
421 }
422
423 yf::Zoom::Frontend::Frontend(Impl *impl) :
424     m_p(impl), m_is_virtual(false), m_in_use(true)
425 {
426 }
427
428 yf::Zoom::Frontend::~Frontend()
429 {
430 }
431
432 yf::Zoom::FrontendPtr yf::Zoom::Impl::get_frontend(mp::Package &package)
433 {
434     boost::mutex::scoped_lock lock(m_mutex);
435
436     std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
437
438     while(true)
439     {
440         it = m_clients.find(package.session());
441         if (it == m_clients.end())
442             break;
443
444         if (!it->second->m_in_use)
445         {
446             it->second->m_in_use = true;
447             return it->second;
448         }
449         m_cond_session_ready.wait(lock);
450     }
451     FrontendPtr f(new Frontend(this));
452     m_clients[package.session()] = f;
453     f->m_in_use = true;
454     return f;
455 }
456
457 void yf::Zoom::Impl::release_frontend(mp::Package &package)
458 {
459     boost::mutex::scoped_lock lock(m_mutex);
460     std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
461
462     it = m_clients.find(package.session());
463     if (it != m_clients.end())
464     {
465         if (package.session().is_closed())
466         {
467             m_clients.erase(it);
468         }
469         else
470         {
471             it->second->m_in_use = false;
472         }
473         m_cond_session_ready.notify_all();
474     }
475 }
476
477 yf::Zoom::Impl::Impl() :
478     apdu_log(false), element_transform("pz2") , element_raw("raw") ,
479     element_passthru("F"),
480     zoom_timeout("40"), proxy_timeout(1)
481 {
482     bibset = ccl_qual_mk();
483
484     explain_xsp = 0;
485     record_xsp = 0;
486     srand((unsigned int) time(0));
487 }
488
489 yf::Zoom::Impl::~Impl()
490 {
491     if (explain_xsp)
492         xsltFreeStylesheet(explain_xsp);
493     ccl_qual_rm(&bibset);
494 }
495
496 yf::Zoom::SearchablePtr yf::Zoom::Impl::parse_torus_record(const xmlNode *ptr)
497 {
498     Zoom::SearchablePtr s(new Searchable(bibset));
499
500     for (ptr = ptr->children; ptr; ptr = ptr->next)
501     {
502         if (ptr->type != XML_ELEMENT_NODE)
503             continue;
504         if (!strcmp((const char *) ptr->name, "layer"))
505             ptr = ptr->children;
506         else if (!strcmp((const char *) ptr->name,
507                          "authentication"))
508         {
509             s->authentication = mp::xml::get_text(ptr);
510         }
511         else if (!strcmp((const char *) ptr->name,
512                          "authenticationMode"))
513         {
514             s->authenticationMode = mp::xml::get_text(ptr);
515         }
516         else if (!strcmp((const char *) ptr->name,
517                          "cfAuth"))
518         {
519             s->cfAuth = mp::xml::get_text(ptr);
520         }
521         else if (!strcmp((const char *) ptr->name,
522                          "cfProxy"))
523         {
524             s->cfProxy = mp::xml::get_text(ptr);
525         }
526         else if (!strcmp((const char *) ptr->name,
527                          "cfSubDB"))
528         {
529             s->cfSubDB = mp::xml::get_text(ptr);
530         }
531         else if (!strcmp((const char *) ptr->name,
532                          "contentConnector"))
533         {
534             s->contentConnector = mp::xml::get_text(ptr);
535         }
536         else if (!strcmp((const char *) ptr->name, "udb"))
537         {
538             s->udb = mp::xml::get_text(ptr);
539         }
540         else if (!strcmp((const char *) ptr->name, "zurl"))
541         {
542             s->target = mp::xml::get_text(ptr);
543         }
544         else if (!strcmp((const char *) ptr->name, "sru"))
545         {
546             s->sru = mp::xml::get_text(ptr);
547         }
548         else if (!strcmp((const char *) ptr->name, "SRUVersion") ||
549                  !strcmp((const char *) ptr->name, "sruVersion"))
550         {
551             s->sru_version = mp::xml::get_text(ptr);
552         }
553         else if (!strcmp((const char *) ptr->name,
554                          "queryEncoding"))
555         {
556             s->query_encoding = mp::xml::get_text(ptr);
557         }
558         else if (!strcmp((const char *) ptr->name,
559                          "piggyback"))
560         {
561             s->piggyback = mp::xml::get_bool(ptr, true);
562         }
563         else if (!strcmp((const char *) ptr->name,
564                          "requestSyntax"))
565         {
566             s->request_syntax = mp::xml::get_text(ptr);
567         }
568         else if (!strcmp((const char *) ptr->name,
569                          "elementSet"))
570         {
571             s->element_set = mp::xml::get_text(ptr);
572         }
573         else if (!strcmp((const char *) ptr->name,
574                          "recordEncoding"))
575         {
576             s->record_encoding = mp::xml::get_text(ptr);
577         }
578         else if (!strcmp((const char *) ptr->name,
579                          "transform"))
580         {
581             s->transform_xsl_fname = mp::xml::get_text(ptr);
582         }
583         else if (!strcmp((const char *) ptr->name,
584                          "literalTransform"))
585         {
586             s->transform_xsl_content = mp::xml::get_text(ptr);
587         }
588         else if (!strcmp((const char *) ptr->name,
589                          "urlRecipe"))
590         {
591             s->urlRecipe = mp::xml::get_text(ptr);
592         }
593         else if (!strcmp((const char *) ptr->name,
594                          "useTurboMarc"))
595         {
596             ; // useTurboMarc is ignored
597         }
598         else if (!strncmp((const char *) ptr->name,
599                           "cclmap_", 7))
600         {
601             std::string value = mp::xml::get_text(ptr);
602             if (value.length() > 0)
603             {
604                 ccl_qual_fitem(s->ccl_bibset, value.c_str(),
605                                (const char *) ptr->name + 7);
606             }
607         }
608         else if (!strncmp((const char *) ptr->name,
609                           "sortmap_", 8))
610         {
611             std::string value = mp::xml::get_text(ptr);
612             s->sortmap[(const char *) ptr->name + 8] = value;
613         }
614         else if (!strcmp((const char *) ptr->name,
615                           "sortStrategy"))
616         {
617             s->sortStrategy = mp::xml::get_text(ptr);
618         }
619         else if (!strcmp((const char *) ptr->name,
620                           "extraArgs"))
621         {
622             s->extraArgs = mp::xml::get_text(ptr);
623         }
624         else if (!strcmp((const char *) ptr->name, "rpn2cql"))
625             s->rpn2cql_fname = mp::xml::get_text(ptr);
626     }
627     return s;
628 }
629
630 void yf::Zoom::Impl::configure_local_records(const xmlNode *ptr, bool test_only)
631 {
632     while (ptr && ptr->type != XML_ELEMENT_NODE)
633         ptr = ptr->next;
634
635     if (ptr)
636     {
637         if (!strcmp((const char *) ptr->name, "records"))
638         {
639             for (ptr = ptr->children; ptr; ptr = ptr->next)
640             {
641                 if (ptr->type != XML_ELEMENT_NODE)
642                     continue;
643                 if (!strcmp((const char *) ptr->name, "record"))
644                 {
645                     SearchablePtr s = parse_torus_record(ptr);
646                     if (s)
647                     {
648                         std::string udb = s->udb;
649                         if (udb.length())
650                             s_map[s->udb] = s;
651                         else
652                         {
653                             throw mp::filter::FilterException
654                                 ("No udb for local torus record");
655                         }
656                     }
657                 }
658                 else
659                 {
660                     throw mp::filter::FilterException
661                         ("Bad element "
662                          + std::string((const char *) ptr->name)
663                          + " in zoom filter inside element "
664                          "<torus><records>");
665                 }
666             }
667         }
668         else
669         {
670             throw mp::filter::FilterException
671                 ("Bad element "
672                  + std::string((const char *) ptr->name)
673                  + " in zoom filter inside element <torus>");
674         }
675     }
676 }
677
678 void yf::Zoom::Impl::configure(const xmlNode *ptr, bool test_only,
679                                const char *path)
680 {
681     std::string explain_xslt_fname;
682     std::string record_xslt_fname;
683
684     if (path && *path)
685     {
686         file_path = path;
687     }
688     for (ptr = ptr->children; ptr; ptr = ptr->next)
689     {
690         if (ptr->type != XML_ELEMENT_NODE)
691             continue;
692         else if (!strcmp((const char *) ptr->name, "torus"))
693         {
694             const struct _xmlAttr *attr;
695             for (attr = ptr->properties; attr; attr = attr->next)
696             {
697                 if (!strcmp((const char *) attr->name, "url"))
698                     torus_searchable_url = mp::xml::get_text(attr->children);
699                 else if (!strcmp((const char *) attr->name, "content_url"))
700                     torus_content_url = mp::xml::get_text(attr->children);
701                 else if (!strcmp((const char *) attr->name, "auth_url"))
702                     torus_auth_url = mp::xml::get_text(attr->children);
703                 else if (!strcmp((const char *) attr->name, "realm"))
704                     default_realm = mp::xml::get_text(attr->children);
705                 else if (!strcmp((const char *) attr->name, "xsldir"))
706                     xsldir = mp::xml::get_text(attr->children);
707                 else if (!strcmp((const char *) attr->name, "element_transform"))
708                     element_transform = mp::xml::get_text(attr->children);
709                 else if (!strcmp((const char *) attr->name, "element_raw"))
710                     element_raw = mp::xml::get_text(attr->children);
711                 else if (!strcmp((const char *) attr->name, "element_passthru"))
712                     element_passthru = mp::xml::get_text(attr->children);
713                 else if (!strcmp((const char *) attr->name, "proxy"))
714                     proxy = mp::xml::get_text(attr->children);
715                 else if (!strcmp((const char *) attr->name, "explain_xsl"))
716                     explain_xslt_fname = mp::xml::get_text(attr->children);
717                 else if (!strcmp((const char *) attr->name, "record_xsl"))
718                     record_xslt_fname = mp::xml::get_text(attr->children);
719                 else
720                     throw mp::filter::FilterException(
721                         "Bad attribute " + std::string((const char *)
722                                                        attr->name));
723             }
724             // If content_url is not given, use value of searchable, to
725             // ensure backwards compatibility
726             if (!torus_content_url.length())
727                 torus_content_url = torus_searchable_url;
728             configure_local_records(ptr->children, test_only);
729         }
730         else if (!strcmp((const char *) ptr->name, "cclmap"))
731         {
732             const char *addinfo = 0;
733             ccl_xml_config(bibset, ptr, &addinfo);
734         }
735         else if (!strcmp((const char *) ptr->name, "fieldmap"))
736         {
737             const struct _xmlAttr *attr;
738             std::string ccl_field;
739             std::string cql_field;
740             for (attr = ptr->properties; attr; attr = attr->next)
741             {
742                 if (!strcmp((const char *) attr->name, "ccl"))
743                     ccl_field = mp::xml::get_text(attr->children);
744                 else if (!strcmp((const char *) attr->name, "cql"))
745                     cql_field = mp::xml::get_text(attr->children);
746                 else
747                     throw mp::filter::FilterException(
748                         "Bad attribute " + std::string((const char *)
749                                                        attr->name));
750             }
751             if (cql_field.length())
752                 fieldmap[cql_field] = ccl_field;
753         }
754         else if (!strcmp((const char *) ptr->name, "contentProxy"))
755         {
756             const struct _xmlAttr *attr;
757             for (attr = ptr->properties; attr; attr = attr->next)
758             {
759                 if (!strcmp((const char *) attr->name, "server"))
760                 {
761                     yaz_log(YLOG_WARN,
762                             "contentProxy's server attribute is deprecated");
763                     yaz_log(YLOG_LOG,
764                             "Specify config_file instead. For example:");
765                     yaz_log(YLOG_LOG,
766                             " content_file=\"/etc/cf-proxy/cproxy.cfg\"");
767                     content_proxy_server = mp::xml::get_text(attr->children);
768                 }
769                 else if (!strcmp((const char *) attr->name, "tmp_file"))
770                     content_tmp_file = mp::xml::get_text(attr->children);
771                 else if (!strcmp((const char *) attr->name, "config_file"))
772                     content_config_file = mp::xml::get_text(attr->children);
773                 else
774                     throw mp::filter::FilterException(
775                         "Bad attribute " + std::string((const char *)
776                                                        attr->name));
777             }
778         }
779         else if (!strcmp((const char *) ptr->name, "log"))
780         {
781             const struct _xmlAttr *attr;
782             for (attr = ptr->properties; attr; attr = attr->next)
783             {
784                 if (!strcmp((const char *) attr->name, "apdu"))
785                     apdu_log = mp::xml::get_bool(attr->children, false);
786                 else
787                     throw mp::filter::FilterException(
788                         "Bad attribute " + std::string((const char *)
789                                                        attr->name));
790             }
791         }
792         else if (!strcmp((const char *) ptr->name, "zoom"))
793         {
794             const struct _xmlAttr *attr;
795             for (attr = ptr->properties; attr; attr = attr->next)
796             {
797                 if (!strcmp((const char *) attr->name, "timeout"))
798                     zoom_timeout = mp::xml::get_text(attr->children);
799                 else if (!strcmp((const char *) attr->name, "proxy_timeout"))
800                     proxy_timeout = mp::xml::get_int(attr->children, 1);
801                 else
802                     throw mp::filter::FilterException(
803                         "Bad attribute " + std::string((const char *)
804                                                        attr->name));
805             }
806         }
807         else
808         {
809             throw mp::filter::FilterException
810                 ("Bad element "
811                  + std::string((const char *) ptr->name)
812                  + " in zoom filter");
813         }
814     }
815
816     if (explain_xslt_fname.length())
817     {
818         const char *path = 0;
819
820         if (xsldir.length())
821             path = xsldir.c_str();
822         else
823             path = file_path.c_str();
824
825         char fullpath[1024];
826         char *cp = yaz_filepath_resolve(explain_xslt_fname.c_str(),
827                                         path, 0, fullpath);
828         if (!cp)
829         {
830             throw mp::filter::FilterException
831                 ("Cannot read XSLT " + explain_xslt_fname);
832         }
833
834         xmlDoc *xsp_doc = xmlParseFile(cp);
835         if (!xsp_doc)
836         {
837             throw mp::filter::FilterException
838                 ("Cannot parse XSLT " + explain_xslt_fname);
839         }
840
841         explain_xsp = xsltParseStylesheetDoc(xsp_doc);
842         if (!explain_xsp)
843         {
844             xmlFreeDoc(xsp_doc);
845             throw mp::filter::FilterException
846                 ("Cannot parse XSLT " + explain_xslt_fname);
847
848         }
849     }
850
851     if (record_xslt_fname.length())
852     {
853         const char *path = 0;
854
855         if (xsldir.length())
856             path = xsldir.c_str();
857         else
858             path = file_path.c_str();
859
860         char fullpath[1024];
861         char *cp = yaz_filepath_resolve(record_xslt_fname.c_str(),
862                                         path, 0, fullpath);
863         if (!cp)
864         {
865             throw mp::filter::FilterException
866                 ("Cannot read XSLT " + record_xslt_fname);
867         }
868
869         xmlDoc *xsp_doc = xmlParseFile(cp);
870         if (!xsp_doc)
871         {
872             throw mp::filter::FilterException
873                 ("Cannot parse XSLT " + record_xslt_fname);
874         }
875
876         record_xsp = xsltParseStylesheetDoc(xsp_doc);
877         if (!record_xsp)
878         {
879             xmlFreeDoc(xsp_doc);
880             throw mp::filter::FilterException
881                 ("Cannot parse XSLT " + record_xslt_fname);
882
883         }
884     }
885 }
886
887 bool yf::Zoom::Frontend::create_content_session(mp::Package &package,
888                                                 BackendPtr b,
889                                                 int *error, char **addinfo,
890                                                 ODR odr,
891                                                 std::string authentication,
892                                                 std::string proxy,
893                                                 std::string realm)
894 {
895     if (b->sptr->contentConnector.length())
896     {
897         std::string proxyhostname;
898         std::string tmp_file;
899         bool legacy_format = false;
900
901         if (m_p->content_proxy_server.length())
902         {
903             proxyhostname = m_p->content_proxy_server;
904             legacy_format = true;
905         }
906
907         if (m_p->content_tmp_file.length())
908             tmp_file = m_p->content_tmp_file;
909
910         if (m_p->content_config_file.length())
911         {
912             FILE *inf = fopen(m_p->content_config_file.c_str(), "r");
913             if (inf)
914             {
915                 char buf[1024];
916                 while (fgets(buf, sizeof(buf)-1, inf))
917                 {
918                     char *cp;
919                     cp = strchr(buf, '#');
920                     if (cp)
921                         *cp = '\0';
922                     cp = strchr(buf, '\n');
923                     if (cp)
924                         *cp = '\0';
925                     cp = strchr(buf, ':');
926                     if (cp)
927                     {
928                         char *cp1 = cp;
929                         while (cp1 != buf && cp1[-1] == ' ')
930                             cp1--;
931                         *cp1 = '\0';
932                         cp++;
933                         while (*cp == ' ')
934                             cp++;
935                         if (!strcmp(buf, "proxyhostname"))
936                             proxyhostname = cp;
937                         if (!strcmp(buf, "sessiondir") && *cp)
938                         {
939                             if (cp[strlen(cp)-1] == '/')
940                                 cp[strlen(cp)-1] = '\0';
941                             tmp_file = std::string(cp) + std::string("/cf.XXXXXX.p");
942                         }
943                     }
944                 }
945                 fclose(inf);
946             }
947             else
948             {
949                 package.log("zoom", YLOG_WARN|YLOG_ERRNO,
950                             "unable to open content config %s",
951                             m_p->content_config_file.c_str());
952                 *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
953                 *addinfo = (char *)  odr_malloc(odr, 70 + tmp_file.length());
954                 sprintf(*addinfo, "zoom: unable to open content config %s",
955                         m_p->content_config_file.c_str());
956                 return false;
957             }
958         }
959
960         if (proxyhostname.length() == 0)
961         {
962             package.log("zoom", YLOG_WARN, "no proxyhostname");
963             return true;
964         }
965         if (tmp_file.length() == 0)
966         {
967             package.log("zoom", YLOG_WARN, "no tmp_file");
968             return true;
969         }
970
971         char *fname = xstrdup(tmp_file.c_str());
972         char *xx = strstr(fname, "XXXXXX");
973         if (!xx)
974         {
975             package.log("zoom", YLOG_WARN, "bad tmp_file %s", tmp_file.c_str());
976             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
977             *addinfo = (char *)  odr_malloc(odr, 60 + tmp_file.length());
978             sprintf(*addinfo, "zoom: bad format of content tmp_file: %s",
979                     tmp_file.c_str());
980             xfree(fname);
981             return false;
982         }
983         char tmp_char = xx[6];
984         sprintf(xx, "%06d", ((unsigned) rand()) % 1000000);
985         if (legacy_format)
986             b->cproxy_host = std::string(xx) + "." + proxyhostname;
987         else
988             b->cproxy_host = proxyhostname + "/" + xx;
989         xx[6] = tmp_char;
990
991         FILE *file = fopen(fname, "w");
992         if (!file)
993         {
994             package.log("zoom", YLOG_WARN|YLOG_ERRNO, "create %s", fname);
995             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
996             *addinfo = (char *) odr_malloc(odr, 50 + strlen(fname));
997             sprintf(*addinfo, "zoom: could not create %s", fname);
998             xfree(fname);
999             return false;
1000         }
1001         mp::wrbuf w;
1002         wrbuf_puts(w, "#content_proxy\n");
1003         wrbuf_printf(w, "connector: %s\n", b->sptr->contentConnector.c_str());
1004         if (authentication.length())
1005             wrbuf_printf(w, "auth: %s\n", authentication.c_str());
1006         if (proxy.length())
1007             wrbuf_printf(w, "proxy: %s\n", proxy.c_str());
1008         if (realm.length())
1009             wrbuf_printf(w, "realm: %s\n", realm.c_str());
1010
1011         fwrite(w.buf(), 1, w.len(), file);
1012         fclose(file);
1013         package.log("zoom", YLOG_LOG, "content file: %s", fname);
1014         xfree(fname);
1015     }
1016     return true;
1017 }
1018
1019 yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
1020     mp::Package &package,
1021     std::string &database, int *error, char **addinfo, mp::odr &odr,
1022     int *proxy_step)
1023 {
1024     bool connection_reuse = false;
1025     std::string proxy;
1026
1027     std::list<BackendPtr>::const_iterator map_it;
1028     if (m_backend && !m_backend->enable_explain &&
1029         m_backend->m_frontend_database == database)
1030     {
1031         connection_reuse = true;
1032         proxy = m_backend->m_proxy;
1033     }
1034
1035     std::string input_args;
1036     std::string torus_db;
1037     size_t db_arg_pos = database.find(',');
1038     if (db_arg_pos != std::string::npos)
1039     {
1040         torus_db = database.substr(0, db_arg_pos);
1041         input_args = database.substr(db_arg_pos + 1);
1042     }
1043     else
1044         torus_db = database;
1045
1046     std::string authentication;
1047     std::string content_authentication;
1048     std::string content_proxy;
1049     std::string realm = session_realm;
1050     if (realm.length() == 0)
1051         realm = m_p->default_realm;
1052
1053     const char *param_user = 0;
1054     const char *param_password = 0;
1055     const char *param_content_user = 0;
1056     const char *param_content_password = 0;
1057     const char *param_nocproxy = 0;
1058     int no_parms = 0;
1059
1060     char **names;
1061     char **values;
1062     int no_out_args = 0;
1063     if (input_args.length())
1064         no_parms = yaz_uri_to_array(input_args.c_str(),
1065                                     odr, &names, &values);
1066     // adding 10 because we'll be adding other URL args
1067     const char **out_names = (const char **)
1068         odr_malloc(odr, (10 + no_parms) * sizeof(*out_names));
1069     const char **out_values = (const char **)
1070         odr_malloc(odr, (10 + no_parms) * sizeof(*out_values));
1071
1072     // may be changed if it's a content connection
1073     std::string torus_url = m_p->torus_searchable_url;
1074     int i;
1075     for (i = 0; i < no_parms; i++)
1076     {
1077         const char *name = names[i];
1078         const char *value = values[i];
1079         assert(name);
1080         assert(value);
1081         if (!strcmp(name, "user"))
1082             param_user = value;
1083         else if (!strcmp(name, "password"))
1084             param_password = value;
1085         else if (!strcmp(name, "content-user"))
1086             param_content_user = value;
1087         else if (!strcmp(name, "content-password"))
1088             param_content_password = value;
1089         else if (!strcmp(name, "content-proxy"))
1090             content_proxy = value;
1091         else if (!strcmp(name, "nocproxy"))
1092             param_nocproxy = value;
1093         else if (!strcmp(name, "proxy"))
1094         {
1095             char **dstr;
1096             int dnum = 0;
1097             nmem_strsplit(((ODR) odr)->mem, ",", value, &dstr, &dnum);
1098             if (connection_reuse)
1099             {
1100                 // find the step after our current proxy
1101                 int i;
1102                 for (i = 0; i < dnum; i++)
1103                     if (!strcmp(proxy.c_str(), dstr[i]))
1104                         break;
1105                 if (i >= dnum - 1)
1106                     *proxy_step = 0;
1107                 else
1108                     *proxy_step = i + 1;
1109             }
1110             else
1111             {
1112                 // step is known.. Guess our proxy from it
1113                 if (*proxy_step >= dnum)
1114                     *proxy_step = 0;
1115                 else
1116                 {
1117                     proxy = dstr[*proxy_step];
1118
1119                     (*proxy_step)++;
1120                     if (*proxy_step == dnum)
1121                         *proxy_step = 0;
1122                 }
1123             }
1124         }
1125         else if (!strcmp(name, "cproxysession"))
1126         {
1127             out_names[no_out_args] = name;
1128             out_values[no_out_args++] = value;
1129             torus_url = m_p->torus_content_url;
1130         }
1131         else if (!strcmp(name, "realm") && session_realm.length() == 0)
1132             realm = value;
1133         else if (!strcmp(name, "torus_url") && session_realm.length() == 0)
1134             torus_url = value;
1135         else if (name[0] == 'x' && name[1] == '-')
1136         {
1137             out_names[no_out_args] = name;
1138             out_values[no_out_args++] = value;
1139         }
1140         else
1141         {
1142             BackendPtr notfound;
1143             char *msg = (char*) odr_malloc(odr, strlen(name) + 30);
1144             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
1145             sprintf(msg, "zoom: bad database argument: %s", name);
1146             *addinfo = msg;
1147             return notfound;
1148         }
1149     }
1150     if (proxy.length())
1151         package.log("zoom", YLOG_LOG, "proxy: %s", proxy.c_str());
1152
1153     if (connection_reuse)
1154     {
1155         m_backend->connect("", error, addinfo, odr);
1156         return m_backend;
1157     }
1158
1159     if (param_user)
1160     {
1161         authentication = std::string(param_user);
1162         if (param_password)
1163             authentication += "/" + std::string(param_password);
1164     }
1165     if (param_content_user)
1166     {
1167         content_authentication = std::string(param_content_user);
1168         if (param_content_password)
1169             content_authentication += "/" + std::string(param_content_password);
1170     }
1171
1172     if (torus_db.compare("IR-Explain---1") == 0)
1173         return explain_search(package, database, error, addinfo, odr, torus_url,
1174                               torus_db, realm);
1175
1176     SearchablePtr sptr;
1177
1178     std::map<std::string,SearchablePtr>::iterator it;
1179     it = m_p->s_map.find(torus_db);
1180     if (it != m_p->s_map.end())
1181         sptr = it->second;
1182     else if (torus_url.length() > 0)
1183     {
1184         std::string torus_query = "udb==" + torus_db;
1185         xmlDoc *doc = mp::get_searchable(package,torus_url, torus_db,
1186                                          torus_query,
1187                                          realm, m_p->proxy);
1188         if (!doc)
1189         {
1190             *error = YAZ_BIB1_UNSPECIFIED_ERROR;
1191             *addinfo = odr_strdup(odr, "Torus server unavailable or "
1192                                   "incorrectly configured");
1193             BackendPtr b;
1194             return b;
1195         }
1196         const xmlNode *ptr = xmlDocGetRootElement(doc);
1197         if (ptr && ptr->type == XML_ELEMENT_NODE)
1198         {
1199             if (!strcmp((const char *) ptr->name, "record"))
1200             {
1201                 sptr = m_p->parse_torus_record(ptr);
1202             }
1203             else if (!strcmp((const char *) ptr->name, "records"))
1204             {
1205                 for (ptr = ptr->children; ptr; ptr = ptr->next)
1206                 {
1207                     if (ptr->type == XML_ELEMENT_NODE
1208                         && !strcmp((const char *) ptr->name, "record"))
1209                     {
1210                         if (sptr)
1211                         {
1212                             *error = YAZ_BIB1_UNSPECIFIED_ERROR;
1213                             *addinfo = (char*)
1214                                 odr_malloc(odr, 40 + torus_db.length());
1215                             sprintf(*addinfo, "multiple records for udb=%s",
1216                                     database.c_str());
1217                             xmlFreeDoc(doc);
1218                             BackendPtr b;
1219                             return b;
1220                         }
1221                         sptr = m_p->parse_torus_record(ptr);
1222                     }
1223                 }
1224             }
1225             else
1226             {
1227                 *error = YAZ_BIB1_UNSPECIFIED_ERROR;
1228                 *addinfo = (char*) odr_malloc(
1229                     odr, 40 + strlen((const char *) ptr->name));
1230                 sprintf(*addinfo, "bad root element for torus: %s", ptr->name);
1231                 xmlFreeDoc(doc);
1232                 BackendPtr b;
1233                 return b;
1234             }
1235         }
1236         xmlFreeDoc(doc);
1237     }
1238
1239     if (!sptr)
1240     {
1241         *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
1242         *addinfo = odr_strdup(odr, torus_db.c_str());
1243         BackendPtr b;
1244         return b;
1245     }
1246
1247     xsltStylesheetPtr xsp = 0;
1248     if (sptr->transform_xsl_content.length())
1249     {
1250         xmlDoc *xsp_doc = xmlParseMemory(sptr->transform_xsl_content.c_str(),
1251                                          sptr->transform_xsl_content.length());
1252         if (!xsp_doc)
1253         {
1254             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
1255             *addinfo = odr_strdup(odr, "zoom: xmlParseMemory failed "
1256                                   "for literalTransform XSL");
1257             BackendPtr b;
1258             return b;
1259         }
1260         xsp = xsltParseStylesheetDoc(xsp_doc);
1261         if (!xsp)
1262         {
1263             *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
1264             *addinfo =
1265                 odr_strdup(odr,"zoom: xsltParseStylesheetDoc failed "
1266                            "for literalTransform XSL");
1267             BackendPtr b;
1268             xmlFreeDoc(xsp_doc);
1269             return b;
1270         }
1271     }
1272     else if (sptr->transform_xsl_fname.length())
1273     {
1274         const char *path = 0;
1275
1276         if (m_p->xsldir.length())
1277             path = m_p->xsldir.c_str();
1278         else
1279             path = m_p->file_path.c_str();
1280         std::string fname;
1281
1282         char fullpath[1024];
1283         char *cp = yaz_filepath_resolve(sptr->transform_xsl_fname.c_str(),
1284                                         path, 0, fullpath);
1285         if (cp)
1286             fname.assign(cp);
1287         else
1288         {
1289             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
1290             *addinfo = (char *)
1291                 odr_malloc(odr, 40 + sptr->transform_xsl_fname.length());
1292             sprintf(*addinfo, "zoom: could not open file %s",
1293                     sptr->transform_xsl_fname.c_str());
1294             BackendPtr b;
1295             return b;
1296         }
1297         xmlDoc *xsp_doc = xmlParseFile(fname.c_str());
1298         if (!xsp_doc)
1299         {
1300             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
1301             *addinfo = (char *) odr_malloc(odr, 50 + fname.length());
1302             sprintf(*addinfo, "zoom: xmlParseFile failed for file %s",
1303                     fname.c_str());
1304             BackendPtr b;
1305             return b;
1306         }
1307         xsp = xsltParseStylesheetDoc(xsp_doc);
1308         if (!xsp)
1309         {
1310             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
1311             *addinfo = (char *) odr_malloc(odr, 50 + fname.length());
1312             sprintf(*addinfo, "zoom: xsltParseStylesheetDoc failed "
1313                     "for file %s", fname.c_str());
1314             BackendPtr b;
1315             xmlFreeDoc(xsp_doc);
1316             return b;
1317         }
1318     }
1319
1320     cql_transform_t cqlt = 0;
1321     if (sptr->rpn2cql_fname.length())
1322     {
1323         char fullpath[1024];
1324         char *cp = yaz_filepath_resolve(sptr->rpn2cql_fname.c_str(),
1325                                         m_p->file_path.c_str(), 0, fullpath);
1326         if (cp)
1327             cqlt = cql_transform_open_fname(fullpath);
1328     }
1329     else
1330         cqlt = cql_transform_create();
1331
1332     if (!cqlt)
1333     {
1334         *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
1335         *addinfo = odr_strdup(odr, "zoom: missing/invalid cql2rpn file");
1336         BackendPtr b;
1337         xsltFreeStylesheet(xsp);
1338         return b;
1339     }
1340
1341     m_backend.reset();
1342
1343     BackendPtr b(new Backend);
1344
1345     b->cqlt = cqlt;
1346     b->sptr = sptr;
1347     b->xsp = xsp;
1348     b->m_frontend_database = database;
1349     b->enable_cproxy = param_nocproxy ? false : true;
1350
1351     if (sptr->query_encoding.length())
1352         b->set_option("rpnCharset", sptr->query_encoding);
1353
1354     std::string extraArgs = sptr->extraArgs;
1355
1356     b->set_option("timeout", m_p->zoom_timeout.c_str());
1357
1358     if (m_p->apdu_log)
1359         b->set_option("apdulog", "1");
1360
1361     if (sptr->piggyback && sptr->sru.length())
1362         b->set_option("count", "1"); /* some SRU servers INSIST on getting
1363                                         maximumRecords > 0 */
1364     b->set_option("piggyback", sptr->piggyback ? "1" : "0");
1365
1366     if (authentication.length() == 0)
1367         authentication = sptr->authentication;
1368
1369     if (proxy.length() == 0)
1370         proxy = sptr->cfProxy;
1371     b->m_proxy = proxy;
1372
1373     if (sptr->cfAuth.length())
1374     {
1375         // A CF target
1376         b->set_option("user", sptr->cfAuth);
1377         if (authentication.length())
1378         {
1379             size_t found = authentication.find('/');
1380             if (found != std::string::npos)
1381             {
1382                 out_names[no_out_args] = "user";
1383                 out_values[no_out_args++] =
1384                     odr_strdup(odr, authentication.substr(0, found).c_str());
1385
1386                 out_names[no_out_args] = "password";
1387                 out_values[no_out_args++] =
1388                     odr_strdup(odr, authentication.substr(found+1).c_str());
1389             }
1390             else
1391             {
1392                 out_names[no_out_args] = "user";
1393                 out_values[no_out_args++] =
1394                     odr_strdup(odr, authentication.c_str());
1395             }
1396         }
1397         if (proxy.length())
1398         {
1399             out_names[no_out_args] = "proxy";
1400             out_values[no_out_args++] = odr_strdup(odr, proxy.c_str());
1401         }
1402         if (sptr->cfSubDB.length())
1403         {
1404             out_names[no_out_args] = "subdatabase";
1405             out_values[no_out_args++] = odr_strdup(odr, sptr->cfSubDB.c_str());
1406         }
1407         if (param_nocproxy)
1408         {
1409             out_names[no_out_args] = "nocproxy";
1410             out_values[no_out_args++] = odr_strdup(odr, param_nocproxy);
1411         }
1412     }
1413     else
1414     {
1415         if (sptr->sru.length() == 0)
1416             b->set_option("user", authentication); /* Z39.50 */
1417         else
1418         {
1419             std::string user;
1420             std::string password;
1421             std::string authtype = sptr->authenticationMode;
1422
1423             {
1424                 const char *cstr = authentication.c_str();
1425                 const char *cp1 = strchr(cstr, '/');
1426                 if (cp1)
1427                 {
1428                     password.assign(cp1 + 1);
1429                     user.assign(cstr, cp1 - cstr);
1430                 }
1431                 else
1432                     user.assign(cstr);
1433             }
1434
1435             if (authtype.compare("url") == 0)
1436             {
1437                 /* SRU URL encoding of auth stuff */
1438                 ODR o = odr_createmem(ODR_ENCODE);
1439                 char *path = 0;
1440                 const char *names[3];
1441                 const char *values[3];
1442
1443                 names[0] = "x-username";
1444                 values[0] = user.c_str();
1445                 names[1] = "x-password";
1446                 values[1] = password.c_str();
1447                 names[2] = 0;
1448                 values[2] = 0;
1449
1450                 yaz_array_to_uri(&path, o, (char **) names, (char **) values);
1451                 if (extraArgs.length())
1452                     extraArgs.append("&");
1453                 extraArgs.append(path);
1454                 odr_destroy(o);
1455             }
1456             else
1457             {
1458                 b->set_option("user", user);
1459                 if (password.length())
1460                     b->set_option("password", password);
1461             }
1462         }
1463         if (proxy.length())
1464             b->set_option("proxy", proxy);
1465     }
1466     if (extraArgs.length())
1467         b->set_option("extraArgs", extraArgs);
1468
1469     std::string url(sptr->target);
1470     if (sptr->sru.length())
1471     {
1472         b->set_option("sru", sptr->sru);
1473         if (url.find("://") == std::string::npos)
1474             url = "http://" + url;
1475         if (sptr->sru_version.length())
1476             b->set_option("sru_version", sptr->sru_version);
1477     }
1478     if (no_out_args)
1479     {
1480         char *x_args = 0;
1481         out_names[no_out_args] = 0; // terminate list
1482
1483         yaz_array_to_uri(&x_args, odr, (char **) out_names,
1484                          (char **) out_values);
1485         url += "," + std::string(x_args);
1486     }
1487     package.log("zoom", YLOG_LOG, "url: %s", url.c_str());
1488     b->connect(url, error, addinfo, odr);
1489     if (*error == 0 && b->enable_cproxy)
1490         create_content_session(package, b, error, addinfo, odr,
1491                                content_authentication.length() ?
1492                                content_authentication : authentication,
1493                                content_proxy.length() ? content_proxy : proxy,
1494                                realm);
1495     if (*error == 0)
1496         m_backend = b;
1497     return b;
1498 }
1499
1500 void yf::Zoom::Frontend::prepare_elements(BackendPtr b,
1501                                           Odr_oid *preferredRecordSyntax,
1502                                           const char *element_set_name,
1503                                           bool &enable_pz2_retrieval,
1504                                           bool &enable_pz2_transform,
1505                                           bool &enable_record_transform,
1506                                           bool &assume_marc8_charset)
1507 {
1508     char oid_name_str[OID_STR_MAX];
1509     const char *syntax_name = 0;
1510
1511     if (preferredRecordSyntax &&
1512         !oid_oidcmp(preferredRecordSyntax, yaz_oid_recsyn_xml))
1513     {
1514         if (element_set_name &&
1515             !strcmp(element_set_name, m_p->element_transform.c_str()))
1516         {
1517             enable_pz2_retrieval = true;
1518             enable_pz2_transform = true;
1519         }
1520         else if (element_set_name &&
1521                  !strcmp(element_set_name, m_p->element_raw.c_str()))
1522         {
1523             enable_pz2_retrieval = true;
1524         }
1525         else if (m_p->record_xsp)
1526         {
1527             enable_pz2_retrieval = true;
1528             enable_pz2_transform = true;
1529             enable_record_transform = true;
1530         }
1531     }
1532
1533     if (enable_pz2_retrieval)
1534     {
1535         std::string configured_request_syntax = b->sptr->request_syntax;
1536         if (configured_request_syntax.length())
1537         {
1538             syntax_name = configured_request_syntax.c_str();
1539             const Odr_oid *syntax_oid =
1540                 yaz_string_to_oid(yaz_oid_std(), CLASS_RECSYN, syntax_name);
1541             if (!oid_oidcmp(syntax_oid, yaz_oid_recsyn_usmarc)
1542                 || !oid_oidcmp(syntax_oid, yaz_oid_recsyn_opac))
1543                 assume_marc8_charset = true;
1544         }
1545     }
1546     else if (preferredRecordSyntax)
1547         syntax_name =
1548             yaz_oid_to_string_buf(preferredRecordSyntax, 0, oid_name_str);
1549
1550     if (b->sptr->sru.length())
1551         syntax_name = "XML";
1552
1553     b->set_option("preferredRecordSyntax", syntax_name);
1554
1555     if (enable_pz2_retrieval)
1556     {
1557         if (element_set_name && !strcmp(element_set_name,
1558                                         m_p->element_passthru.c_str()))
1559             ;
1560         else
1561         {
1562             element_set_name = 0;
1563             if (b->sptr->element_set.length())
1564                 element_set_name = b->sptr->element_set.c_str();
1565         }
1566     }
1567
1568     b->set_option("elementSetName", element_set_name);
1569     if (b->sptr->sru.length() && element_set_name)
1570         b->set_option("schema", element_set_name);
1571 }
1572
1573 Z_Records *yf::Zoom::Frontend::get_explain_records(
1574     mp::Package &package,
1575     Odr_int start,
1576     Odr_int number_to_present,
1577     int *error,
1578     char **addinfo,
1579     Odr_int *number_of_records_returned,
1580     ODR odr,
1581     BackendPtr b,
1582     Odr_oid *preferredRecordSyntax,
1583     const char *element_set_name)
1584 {
1585     Odr_int i;
1586     Z_Records *records = 0;
1587
1588     if (!b->explain_doc)
1589     {
1590         return records;
1591     }
1592     if (number_to_present > 10000)
1593         number_to_present = 10000;
1594
1595     xmlNode *ptr = xmlDocGetRootElement(b->explain_doc);
1596
1597     Z_NamePlusRecordList *npl = (Z_NamePlusRecordList *)
1598         odr_malloc(odr, sizeof(*npl));
1599     npl->records = (Z_NamePlusRecord **)
1600         odr_malloc(odr, number_to_present * sizeof(*npl->records));
1601
1602     for (i = 0; i < number_to_present; i++)
1603     {
1604         int num = 0;
1605         xmlNode *res = xml_node_search(ptr, &num, start + i + 1);
1606         if (!res)
1607             break;
1608         xmlBufferPtr xml_buf = xmlBufferCreate();
1609         xmlNode *tmp_node = xmlCopyNode(res->children, 1);
1610         xmlNodeDump(xml_buf, tmp_node->doc, tmp_node, 0, 0);
1611
1612         Z_NamePlusRecord *npr =
1613             (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
1614         npr->databaseName = odr_strdup(odr, b->m_frontend_database.c_str());
1615         npr->which = Z_NamePlusRecord_databaseRecord;
1616         npr->u.databaseRecord =
1617             z_ext_record_xml(odr,
1618                              (const char *) xml_buf->content, xml_buf->use);
1619         npl->records[i] = npr;
1620         xmlFreeNode(tmp_node);
1621         xmlBufferFree(xml_buf);
1622     }
1623     records = (Z_Records*) odr_malloc(odr, sizeof(*records));
1624     records->which = Z_Records_DBOSD;
1625     records->u.databaseOrSurDiagnostics = npl;
1626
1627     npl->num_records = i;
1628     *number_of_records_returned = i;
1629     return records;
1630 }
1631
1632
1633 Z_Records *yf::Zoom::Frontend::get_records(mp::Package &package,
1634                                            Odr_int start,
1635                                            Odr_int number_to_present,
1636                                            int *error,
1637                                            char **addinfo,
1638                                            Odr_int *number_of_records_returned,
1639                                            ODR odr,
1640                                            BackendPtr b,
1641                                            Odr_oid *preferredRecordSyntax,
1642                                            const char *element_set_name)
1643 {
1644     *number_of_records_returned = 0;
1645     Z_Records *records = 0;
1646     bool enable_pz2_retrieval = false; // whether target profile is used
1647     bool enable_pz2_transform = false; // whether XSLT is used as well
1648     bool assume_marc8_charset = false;
1649     bool enable_record_transform = false;
1650
1651     prepare_elements(b, preferredRecordSyntax,
1652                      element_set_name,
1653                      enable_pz2_retrieval,
1654                      enable_pz2_transform,
1655                      enable_record_transform,
1656                      assume_marc8_charset);
1657
1658     package.log("zoom", YLOG_LOG, "pz2_retrieval: %s . pz2_transform: %s",
1659                 enable_pz2_retrieval ? "yes" : "no",
1660                 enable_pz2_transform ? "yes" : "no");
1661
1662     if (start < 0 || number_to_present <=0)
1663         return records;
1664
1665     if (number_to_present > 10000)
1666         number_to_present = 10000;
1667
1668     ZOOM_record *recs = (ZOOM_record *)
1669         odr_malloc(odr, (size_t) number_to_present * sizeof(*recs));
1670
1671     b->present(start, number_to_present, recs, error, addinfo, odr);
1672
1673     int i = 0;
1674     if (!*error)
1675     {
1676         for (i = 0; i < number_to_present; i++)
1677             if (!recs[i])
1678                 break;
1679     }
1680     if (i > 0)
1681     {  // only return records if no error and at least one record
1682
1683         const char *xsl_parms[3];
1684         mp::wrbuf cproxy_host;
1685
1686         if (b->enable_cproxy && b->cproxy_host.length())
1687         {
1688             wrbuf_puts(cproxy_host, "\"");
1689             wrbuf_puts(cproxy_host, b->cproxy_host.c_str());
1690             wrbuf_puts(cproxy_host, "/\"");
1691
1692             xsl_parms[0] = "cproxyhost";
1693             xsl_parms[1] = wrbuf_cstr(cproxy_host);
1694             xsl_parms[2] = 0;
1695         }
1696         else
1697         {
1698             xsl_parms[0] = 0;
1699         }
1700
1701         char *odr_database = odr_strdup(odr,
1702                                         b->m_frontend_database.c_str());
1703         Z_NamePlusRecordList *npl = (Z_NamePlusRecordList *)
1704             odr_malloc(odr, sizeof(*npl));
1705         *number_of_records_returned = i;
1706         npl->num_records = i;
1707         npl->records = (Z_NamePlusRecord **)
1708             odr_malloc(odr, i * sizeof(*npl->records));
1709         for (i = 0; i < number_to_present; i++)
1710         {
1711             Z_NamePlusRecord *npr = 0;
1712             const char *addinfo;
1713
1714             int sur_error = ZOOM_record_error(recs[i], 0 /* msg */,
1715                                               &addinfo, 0 /* diagset */);
1716
1717             if (sur_error)
1718             {
1719                 log_diagnostic(package, sur_error, addinfo);
1720                 npr = zget_surrogateDiagRec(odr, odr_database, sur_error,
1721                                             addinfo);
1722             }
1723             else if (enable_pz2_retrieval)
1724             {
1725                 char rec_type_str[100];
1726                 const char *record_encoding = 0;
1727
1728                 if (b->sptr->record_encoding.length())
1729                     record_encoding = b->sptr->record_encoding.c_str();
1730                 else if (assume_marc8_charset)
1731                     record_encoding = "marc8";
1732
1733                 strcpy(rec_type_str, b->sptr->use_turbomarc ? "txml" : "xml");
1734                 if (record_encoding)
1735                 {
1736                     strcat(rec_type_str, "; charset=");
1737                     strcat(rec_type_str, record_encoding);
1738                 }
1739
1740                 package.log("zoom", YLOG_LOG, "Getting record of type %s",
1741                             rec_type_str);
1742                 int rec_len;
1743                 xmlChar *xmlrec_buf = 0;
1744                 const char *rec_buf = ZOOM_record_get(recs[i], rec_type_str,
1745                                                       &rec_len);
1746                 if (!rec_buf && !npr)
1747                 {
1748                     std::string addinfo("ZOOM_record_get failed for type ");
1749
1750                     int error = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1751                     addinfo += rec_type_str;
1752                     log_diagnostic(package, error, addinfo.c_str());
1753                     npr = zget_surrogateDiagRec(odr, odr_database,
1754                                                 error, addinfo.c_str());
1755                 }
1756                 else
1757                 {
1758                     package.log_write(rec_buf, rec_len);
1759                     package.log_write("\r\n", 2);
1760                 }
1761
1762                 if (rec_buf && b->xsp && enable_pz2_transform)
1763                 {
1764                     xmlDoc *rec_doc = xmlParseMemory(rec_buf, rec_len);
1765                     if (!rec_doc)
1766                     {
1767                         const char *addinfo = "xml parse failed for record";
1768                         int error = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1769                         log_diagnostic(package, error, addinfo);
1770                         npr = zget_surrogateDiagRec(
1771                             odr, odr_database, error, addinfo);
1772                     }
1773                     else
1774                     {
1775                         // first stage XSLT - per target
1776                         xsltStylesheetPtr xsp = b->xsp;
1777                         xmlDoc *rec_res = xsltApplyStylesheet(xsp, rec_doc,
1778                                                               xsl_parms);
1779                         // insert generated-url
1780                         if (rec_res)
1781                         {
1782                             std::string res =
1783                                 mp::xml::url_recipe_handle(rec_res,
1784                                                            b->sptr->urlRecipe);
1785                             if (res.length())
1786                             {
1787                                 xmlNode *ptr = xmlDocGetRootElement(rec_res);
1788                                 while (ptr && ptr->type != XML_ELEMENT_NODE)
1789                                     ptr = ptr->next;
1790                                 xmlNode *c =
1791                                     xmlNewChild(ptr, 0, BAD_CAST "metadata", 0);
1792                                 xmlNewProp(c, BAD_CAST "type", BAD_CAST
1793                                            "generated-url");
1794                                 xmlNode * t = xmlNewText(BAD_CAST res.c_str());
1795                                 xmlAddChild(c, t);
1796                             }
1797                         }
1798                         // second stage XSLT - common
1799                         if (rec_res && m_p->record_xsp &&
1800                             enable_record_transform)
1801                         {
1802                             xmlDoc *tmp_doc = rec_res;
1803
1804                             xsp = m_p->record_xsp;
1805                             rec_res = xsltApplyStylesheet(xsp, tmp_doc,
1806                                                           xsl_parms);
1807                             xmlFreeDoc(tmp_doc);
1808                         }
1809                         // get result out of it
1810                         if (rec_res)
1811                         {
1812                             xsltSaveResultToString(&xmlrec_buf, &rec_len,
1813                                                    rec_res, xsp);
1814                             rec_buf = (const char *) xmlrec_buf;
1815                             package.log_write(rec_buf, rec_len);
1816
1817                             xmlFreeDoc(rec_res);
1818                         }
1819                         if (!rec_buf)
1820                         {
1821                             std::string addinfo;
1822                             int error =
1823                                 YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1824
1825                             addinfo = "xslt apply failed for "
1826                                 + b->sptr->transform_xsl_fname;
1827                             log_diagnostic(package, error, addinfo.c_str());
1828                             npr = zget_surrogateDiagRec(
1829                                 odr, odr_database, error, addinfo.c_str());
1830                         }
1831                         xmlFreeDoc(rec_doc);
1832                     }
1833                 }
1834
1835                 if (!npr)
1836                 {
1837                     if (!rec_buf)
1838                         npr = zget_surrogateDiagRec(
1839                             odr, odr_database,
1840                             YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
1841                             rec_type_str);
1842                     else
1843                     {
1844                         npr = (Z_NamePlusRecord *)
1845                             odr_malloc(odr, sizeof(*npr));
1846                         npr->databaseName = odr_database;
1847                         npr->which = Z_NamePlusRecord_databaseRecord;
1848                         npr->u.databaseRecord =
1849                             z_ext_record_xml(odr, rec_buf, rec_len);
1850                     }
1851                 }
1852                 if (xmlrec_buf)
1853                     xmlFree(xmlrec_buf);
1854             }
1855             else
1856             {
1857                 Z_External *ext =
1858                     (Z_External *) ZOOM_record_get(recs[i], "ext", 0);
1859                 if (ext)
1860                 {
1861                     npr = (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
1862                     npr->databaseName = odr_database;
1863                     npr->which = Z_NamePlusRecord_databaseRecord;
1864                     npr->u.databaseRecord = ext;
1865                 }
1866                 else
1867                 {
1868                     npr = zget_surrogateDiagRec(
1869                         odr, odr_database,
1870                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
1871                         "ZOOM_record, type ext");
1872                 }
1873             }
1874             npl->records[i] = npr;
1875         }
1876         records = (Z_Records*) odr_malloc(odr, sizeof(*records));
1877         records->which = Z_Records_DBOSD;
1878         records->u.databaseOrSurDiagnostics = npl;
1879     }
1880     return records;
1881 }
1882
1883 struct cql_node *yf::Zoom::Impl::convert_cql_fields(struct cql_node *cn,
1884                                                     ODR odr)
1885 {
1886     struct cql_node *r = 0;
1887     if (!cn)
1888         return 0;
1889     switch (cn->which)
1890     {
1891     case CQL_NODE_ST:
1892         if (cn->u.st.index)
1893         {
1894             std::map<std::string,std::string>::const_iterator it;
1895             it = fieldmap.find(cn->u.st.index);
1896             if (it == fieldmap.end())
1897                 return cn;
1898             if (it->second.length())
1899                 cn->u.st.index = odr_strdup(odr, it->second.c_str());
1900             else
1901                 cn->u.st.index = 0;
1902         }
1903         break;
1904     case CQL_NODE_BOOL:
1905         r = convert_cql_fields(cn->u.boolean.left, odr);
1906         if (!r)
1907             r = convert_cql_fields(cn->u.boolean.right, odr);
1908         break;
1909     case CQL_NODE_SORT:
1910         r = convert_cql_fields(cn->u.sort.search, odr);
1911         break;
1912     }
1913     return r;
1914 }
1915
1916 void yf::Zoom::Frontend::log_diagnostic(mp::Package &package,
1917                                         int error, const char *addinfo)
1918 {
1919     const char *err_msg = yaz_diag_bib1_str(error);
1920     if (addinfo)
1921         package.log("zoom", YLOG_WARN, "Diagnostic %d %s: %s",
1922                     error, err_msg, addinfo);
1923     else
1924         package.log("zoom", YLOG_WARN, "Diagnostic %d %s:",
1925                     error, err_msg);
1926 }
1927
1928 yf::Zoom::BackendPtr yf::Zoom::Frontend::explain_search(mp::Package &package,
1929                                                         std::string &database,
1930                                                         int *error,
1931                                                         char **addinfo,
1932                                                         mp::odr &odr,
1933                                                         std::string torus_url,
1934                                                         std::string &torus_db,
1935                                                         std::string &realm)
1936 {
1937     m_backend.reset();
1938
1939     BackendPtr b(new Backend);
1940
1941     b->m_frontend_database = database;
1942     b->enable_explain = true;
1943
1944     Z_GDU *gdu = package.request().get();
1945     Z_APDU *apdu_req = gdu->u.z3950;
1946     Z_SearchRequest *sr = apdu_req->u.searchRequest;
1947     Z_Query *query = sr->query;
1948
1949     if (!m_p->explain_xsp)
1950     {
1951         *error = YAZ_BIB1_UNSPECIFIED_ERROR;
1952         *addinfo =
1953             odr_strdup(odr, "IR-Explain---1 unsupported. "
1954                        "Torus explain_xsl not defined");
1955         return m_backend;
1956     }
1957     else if (query->which == Z_Query_type_104 &&
1958         query->u.type_104->which == Z_External_CQL)
1959     {
1960         std::string torus_query(query->u.type_104->u.cql);
1961         xmlDoc *doc = mp::get_searchable(package, torus_url, "",
1962                                          torus_query,
1963                                          realm, m_p->proxy);
1964         if (m_p->explain_xsp)
1965         {
1966             xmlDoc *rec_res =  xsltApplyStylesheet(m_p->explain_xsp, doc, 0);
1967
1968             xmlFreeDoc(doc);
1969             doc = rec_res;
1970         }
1971         if (!doc)
1972         {
1973             *error = YAZ_BIB1_UNSPECIFIED_ERROR;
1974             *addinfo = odr_strdup(odr, "Torus server unavailable or "
1975                                   "incorrectly configured");
1976         }
1977         else
1978         {
1979             xmlNode *ptr = xmlDocGetRootElement(doc);
1980             int hits = 0;
1981
1982             xml_node_search(ptr, &hits, 0);
1983
1984             Z_APDU *apdu_res = odr.create_searchResponse(apdu_req, 0, 0);
1985             apdu_res->u.searchResponse->resultCount = odr_intdup(odr, hits);
1986             package.response() = apdu_res;
1987             m_backend = b;
1988         }
1989         if (b->explain_doc)
1990             xmlFreeDoc(b->explain_doc);
1991         b->explain_doc = doc;
1992         return m_backend;
1993     }
1994     else
1995     {
1996         *error = YAZ_BIB1_QUERY_TYPE_UNSUPP;
1997         *addinfo = odr_strdup(odr, "IR-Explain---1 only supports CQL");
1998         return m_backend;
1999     }
2000 }
2001
2002 static bool wait_conn(COMSTACK cs, int secs)
2003 {
2004     struct yaz_poll_fd pfd;
2005
2006     yaz_poll_add(pfd.input_mask, yaz_poll_except);
2007     if (cs->io_pending && CS_WANT_WRITE)
2008         yaz_poll_add(pfd.input_mask, yaz_poll_write);
2009     if (cs->io_pending & CS_WANT_READ)
2010         yaz_poll_add(pfd.input_mask, yaz_poll_read);
2011
2012     pfd.fd = cs_fileno(cs);
2013     pfd.client_data = 0;
2014
2015     int ret = yaz_poll(&pfd, 1, secs, 0);
2016     return ret > 0;
2017 }
2018
2019 bool yf::Zoom::Impl::check_proxy(const char *proxy)
2020 {
2021     COMSTACK conn = 0;
2022     const char *uri = "http://localhost/";
2023     void *add;
2024     mp::odr odr;
2025     bool outcome = false;
2026     conn = cs_create_host_proxy(uri, 0, &add, proxy);
2027
2028     if (!conn)
2029         return false;
2030
2031     Z_GDU *gdu = z_get_HTTP_Request_uri(odr, uri, 0, 1);
2032     gdu->u.HTTP_Request->method = odr_strdup(odr, "GET");
2033
2034     if (z_GDU(odr, &gdu, 0, 0))
2035     {
2036         int len;
2037         char *buf = odr_getbuf(odr, &len, 0);
2038
2039         int ret = cs_connect(conn, add);
2040         if (ret > 0 || (ret == 0 && wait_conn(conn, 1)))
2041         {
2042             while (1)
2043             {
2044                 ret = cs_put(conn, buf, len);
2045                 if (ret != 1)
2046                     break;
2047                 if (!wait_conn(conn, proxy_timeout))
2048                     break;
2049             }
2050             if (ret == 0)
2051                 outcome = true;
2052         }
2053     }
2054     cs_close(conn);
2055     return outcome;
2056 }
2057
2058 bool yf::Zoom::Frontend::retry(mp::Package &package,
2059                                mp::odr &odr,
2060                                BackendPtr b,
2061                                int &error, char **addinfo,
2062                                int &proxy_step, int &same_retries,
2063                                int &proxy_retries)
2064 {
2065     if (b && b->m_proxy.length() && !m_p->check_proxy(b->m_proxy.c_str()))
2066     {
2067         log_diagnostic(package, error, *addinfo);
2068         package.log("zoom", YLOG_LOG, "proxy %s fails", b->m_proxy.c_str());
2069         m_backend.reset();
2070         if (proxy_step) // there is a failover
2071         {
2072             proxy_retries++;
2073             package.log("zoom", YLOG_WARN, "search failed: trying next proxy");
2074             return true;
2075         }
2076         error = YAZ_BIB1_PROXY_FAILURE;
2077         *addinfo = odr_strdup(odr, b->m_proxy.c_str());
2078     }
2079     else if (same_retries == 0 && proxy_retries == 0)
2080     {
2081         log_diagnostic(package, error, *addinfo);
2082         same_retries++;
2083         package.log("zoom", YLOG_WARN, "search failed: retry");
2084         m_backend.reset();
2085         proxy_step = 0;
2086         return true;
2087     }
2088     return false;
2089 }
2090
2091 void yf::Zoom::Frontend::handle_search(mp::Package &package)
2092 {
2093     Z_GDU *gdu = package.request().get();
2094     Z_APDU *apdu_req = gdu->u.z3950;
2095     Z_APDU *apdu_res = 0;
2096     mp::odr odr;
2097     Z_SearchRequest *sr = apdu_req->u.searchRequest;
2098     if (sr->num_databaseNames != 1)
2099     {
2100         int error = YAZ_BIB1_TOO_MANY_DATABASES_SPECIFIED;
2101         log_diagnostic(package, error, 0);
2102         apdu_res = odr.create_searchResponse(apdu_req, error, 0);
2103         package.response() = apdu_res;
2104         return;
2105     }
2106     int proxy_step = 0;
2107     int same_retries = 0;
2108     int proxy_retries = 0;
2109
2110 next_proxy:
2111
2112     int error = 0;
2113     char *addinfo = 0;
2114     std::string db(sr->databaseNames[0]);
2115
2116     BackendPtr b = get_backend_from_databases(package, db, &error,
2117                                               &addinfo, odr, &proxy_step);
2118     if (error)
2119     {
2120         if (retry(package, odr, b, error, &addinfo, proxy_step,
2121                   same_retries, proxy_retries))
2122             goto next_proxy;
2123     }
2124     if (error)
2125     {
2126         log_diagnostic(package, error, addinfo);
2127         apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
2128         package.response() = apdu_res;
2129         return;
2130     }
2131     if (!b || b->enable_explain)
2132         return;
2133
2134     b->set_option("setname", "default");
2135
2136     bool enable_pz2_retrieval = false;
2137     bool enable_pz2_transform = false;
2138     bool enable_record_transform = false;
2139     bool assume_marc8_charset = false;
2140     prepare_elements(b, sr->preferredRecordSyntax, 0 /*element_set_name */,
2141                      enable_pz2_retrieval,
2142                      enable_pz2_transform,
2143                      enable_record_transform,
2144                      assume_marc8_charset);
2145
2146     Odr_int hits = 0;
2147     Z_Query *query = sr->query;
2148     mp::wrbuf ccl_wrbuf;
2149     mp::wrbuf pqf_wrbuf;
2150     std::string sortkeys;
2151
2152     if (query->which == Z_Query_type_1 || query->which == Z_Query_type_101)
2153     {
2154         // RPN
2155         yaz_rpnquery_to_wrbuf(pqf_wrbuf, query->u.type_1);
2156     }
2157     else if (query->which == Z_Query_type_2)
2158     {
2159         // CCL
2160         wrbuf_write(ccl_wrbuf, (const char *) query->u.type_2->buf,
2161                     query->u.type_2->len);
2162     }
2163     else if (query->which == Z_Query_type_104 &&
2164              query->u.type_104->which == Z_External_CQL)
2165     {
2166         // CQL
2167         const char *cql = query->u.type_104->u.cql;
2168         CQL_parser cp = cql_parser_create();
2169         int r = cql_parser_string(cp, cql);
2170         package.log("zoom", YLOG_LOG, "CQL: %s", cql);
2171         if (r)
2172         {
2173             cql_parser_destroy(cp);
2174             error = YAZ_BIB1_MALFORMED_QUERY;
2175             const char *addinfo = "CQL syntax error";
2176             log_diagnostic(package, error, addinfo);
2177             apdu_res =
2178                 odr.create_searchResponse(apdu_req, error, addinfo);
2179             package.response() = apdu_res;
2180             return;
2181         }
2182         struct cql_node *cn = cql_parser_result(cp);
2183         struct cql_node *cn_error = m_p->convert_cql_fields(cn, odr);
2184         if (cn_error)
2185         {
2186             // hopefully we are getting a ptr to a index+relation+term node
2187             error = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
2188             addinfo = 0;
2189             if (cn_error->which == CQL_NODE_ST)
2190                 addinfo = cn_error->u.st.index;
2191
2192             log_diagnostic(package, error, addinfo);
2193             apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
2194             package.response() = apdu_res;
2195             cql_parser_destroy(cp);
2196             return;
2197         }
2198         r = cql_to_ccl(cn, wrbuf_vp_puts,  ccl_wrbuf);
2199         if (r)
2200         {
2201             error = YAZ_BIB1_MALFORMED_QUERY;
2202             const char *addinfo = "CQL to CCL conversion error";
2203
2204             log_diagnostic(package, error, addinfo);
2205             apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
2206             package.response() = apdu_res;
2207             cql_parser_destroy(cp);
2208             return;
2209         }
2210
2211         mp::wrbuf sru_sortkeys_wrbuf;
2212         if (cql_sortby_to_sortkeys(cn, wrbuf_vp_puts, sru_sortkeys_wrbuf))
2213         {
2214             error = YAZ_BIB1_ILLEGAL_SORT_RELATION;
2215             const char *addinfo = "CQL to CCL sortby conversion";
2216
2217             log_diagnostic(package, error, addinfo);
2218             apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
2219             package.response() = apdu_res;
2220             cql_parser_destroy(cp);
2221             return;
2222         }
2223         mp::wrbuf sort_spec_wrbuf;
2224         yaz_srw_sortkeys_to_sort_spec(wrbuf_cstr(sru_sortkeys_wrbuf),
2225                                       sort_spec_wrbuf);
2226         yaz_tok_cfg_t tc = yaz_tok_cfg_create();
2227         yaz_tok_parse_t tp =
2228             yaz_tok_parse_buf(tc, wrbuf_cstr(sort_spec_wrbuf));
2229         yaz_tok_cfg_destroy(tc);
2230
2231         /* go through sortspec and map fields */
2232         int token = yaz_tok_move(tp);
2233         while (token != YAZ_TOK_EOF)
2234         {
2235             if (token == YAZ_TOK_STRING)
2236             {
2237                 const char *field = yaz_tok_parse_string(tp);
2238                 std::map<std::string,std::string>::iterator it;
2239                 it = b->sptr->sortmap.find(field);
2240                 if (it != b->sptr->sortmap.end())
2241                     sortkeys += it->second;
2242                 else
2243                     sortkeys += field;
2244             }
2245             sortkeys += " ";
2246             token = yaz_tok_move(tp);
2247             if (token == YAZ_TOK_STRING)
2248             {
2249                 sortkeys += yaz_tok_parse_string(tp);
2250             }
2251             if (token != YAZ_TOK_EOF)
2252             {
2253                 sortkeys += " ";
2254                 token = yaz_tok_move(tp);
2255             }
2256         }
2257         yaz_tok_parse_destroy(tp);
2258         cql_parser_destroy(cp);
2259     }
2260     else
2261     {
2262         error = YAZ_BIB1_QUERY_TYPE_UNSUPP;
2263         const char *addinfo = 0;
2264         log_diagnostic(package, error, addinfo);
2265         apdu_res =  odr.create_searchResponse(apdu_req, error, addinfo);
2266         package.response() = apdu_res;
2267         return;
2268     }
2269
2270     if (ccl_wrbuf.len())
2271     {
2272         // CCL to PQF
2273         assert(pqf_wrbuf.len() == 0);
2274         int cerror, cpos;
2275         struct ccl_rpn_node *cn;
2276         package.log("zoom", YLOG_LOG, "CCL: %s", wrbuf_cstr(ccl_wrbuf));
2277         cn = ccl_find_str(b->sptr->ccl_bibset, wrbuf_cstr(ccl_wrbuf),
2278                           &cerror, &cpos);
2279         if (!cn)
2280         {
2281             char *addinfo = odr_strdup_null(odr, ccl_err_msg(cerror));
2282             error = YAZ_BIB1_MALFORMED_QUERY;
2283
2284             switch (cerror)
2285             {
2286             case CCL_ERR_UNKNOWN_QUAL:
2287             case CCL_ERR_TRUNC_NOT_LEFT:
2288             case CCL_ERR_TRUNC_NOT_RIGHT:
2289             case CCL_ERR_TRUNC_NOT_BOTH:
2290 #ifdef CCL_ERR_TRUNC_NOT_EMBED
2291             case CCL_ERR_TRUNC_NOT_EMBED:
2292 #endif
2293 #ifdef CCL_ERR_TRUNC_NOT_SINGLE
2294             case CCL_ERR_TRUNC_NOT_SINGLE:
2295 #endif
2296                 error = YAZ_BIB1_UNSUPP_SEARCH;
2297                 break;
2298             }
2299             log_diagnostic(package, error, addinfo);
2300             apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
2301             package.response() = apdu_res;
2302             return;
2303         }
2304         ccl_pquery(pqf_wrbuf, cn);
2305         package.log("zoom", YLOG_LOG, "RPN: %s", wrbuf_cstr(pqf_wrbuf));
2306         ccl_rpn_delete(cn);
2307     }
2308
2309     assert(pqf_wrbuf.len());
2310
2311     ZOOM_query q = ZOOM_query_create();
2312     ZOOM_query_sortby2(q, b->sptr->sortStrategy.c_str(), sortkeys.c_str());
2313
2314     if (b->get_option("sru"))
2315     {
2316         int status = 0;
2317         Z_RPNQuery *zquery;
2318         zquery = p_query_rpn(odr, wrbuf_cstr(pqf_wrbuf));
2319         mp::wrbuf wrb;
2320
2321         if (!strcmp(b->get_option("sru"), "solr"))
2322         {
2323             solr_transform_t cqlt = solr_transform_create();
2324
2325             status = solr_transform_rpn2solr_wrbuf(cqlt, wrb, zquery);
2326
2327             solr_transform_close(cqlt);
2328         }
2329         else
2330         {
2331             status = cql_transform_rpn2cql_wrbuf(b->cqlt, wrb, zquery);
2332         }
2333         if (status == 0)
2334         {
2335             ZOOM_query_cql(q, wrbuf_cstr(wrb));
2336             package.log("zoom", YLOG_LOG, "CQL: %s", wrbuf_cstr(wrb));
2337             b->search(q, &hits, &error, &addinfo, odr);
2338         }
2339         ZOOM_query_destroy(q);
2340
2341         if (status)
2342         {
2343             error = YAZ_BIB1_MALFORMED_QUERY;
2344             const char *addinfo = "can not convert from RPN to CQL/Solr";
2345             log_diagnostic(package, error, addinfo);
2346             apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
2347             package.response() = apdu_res;
2348             return;
2349         }
2350     }
2351     else
2352     {
2353         ZOOM_query_prefix(q, wrbuf_cstr(pqf_wrbuf));
2354         package.log("zoom", YLOG_LOG, "search PQF: %s", wrbuf_cstr(pqf_wrbuf));
2355         b->search(q, &hits, &error, &addinfo, odr);
2356         ZOOM_query_destroy(q);
2357     }
2358
2359     if (error)
2360     {
2361         if (retry(package, odr, b, error, &addinfo, proxy_step,
2362                   same_retries, proxy_retries))
2363             goto next_proxy;
2364     }
2365
2366     const char *element_set_name = 0;
2367     Odr_int number_to_present = 0;
2368     if (!error)
2369         mp::util::piggyback_sr(sr, hits, number_to_present, &element_set_name);
2370
2371     Odr_int number_of_records_returned = 0;
2372     Z_Records *records = get_records(
2373         package,
2374         0, number_to_present, &error, &addinfo,
2375         &number_of_records_returned, odr, b, sr->preferredRecordSyntax,
2376         element_set_name);
2377     if (error)
2378         log_diagnostic(package, error, addinfo);
2379     apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
2380     if (records)
2381     {
2382         apdu_res->u.searchResponse->records = records;
2383         apdu_res->u.searchResponse->numberOfRecordsReturned =
2384             odr_intdup(odr, number_of_records_returned);
2385     }
2386     apdu_res->u.searchResponse->resultCount = odr_intdup(odr, hits);
2387     package.response() = apdu_res;
2388 }
2389
2390 void yf::Zoom::Frontend::handle_present(mp::Package &package)
2391 {
2392     Z_GDU *gdu = package.request().get();
2393     Z_APDU *apdu_req = gdu->u.z3950;
2394     Z_APDU *apdu_res = 0;
2395     Z_PresentRequest *pr = apdu_req->u.presentRequest;
2396
2397     mp::odr odr;
2398     if (!m_backend)
2399     {
2400         package.response() = odr.create_presentResponse(
2401             apdu_req, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST, 0);
2402         return;
2403     }
2404     const char *element_set_name = 0;
2405     Z_RecordComposition *comp = pr->recordComposition;
2406     if (comp && comp->which != Z_RecordComp_simple)
2407     {
2408         package.response() = odr.create_presentResponse(
2409             apdu_req,
2410             YAZ_BIB1_PRESENT_COMP_SPEC_PARAMETER_UNSUPP, 0);
2411         return;
2412     }
2413     if (comp && comp->u.simple->which == Z_ElementSetNames_generic)
2414         element_set_name = comp->u.simple->u.generic;
2415     Odr_int number_of_records_returned = 0;
2416     int error = 0;
2417     char *addinfo = 0;
2418
2419     if (m_backend->enable_explain)
2420     {
2421         Z_Records *records =
2422             get_explain_records(
2423                 package,
2424                 *pr->resultSetStartPoint - 1, *pr->numberOfRecordsRequested,
2425                 &error, &addinfo, &number_of_records_returned, odr, m_backend,
2426                 pr->preferredRecordSyntax, element_set_name);
2427
2428         apdu_res = odr.create_presentResponse(apdu_req, error, addinfo);
2429         if (records)
2430         {
2431             apdu_res->u.presentResponse->records = records;
2432             apdu_res->u.presentResponse->numberOfRecordsReturned =
2433                 odr_intdup(odr, number_of_records_returned);
2434         }
2435         package.response() = apdu_res;
2436     }
2437     else
2438     {
2439         Z_Records *records =
2440             get_records(package,
2441                         *pr->resultSetStartPoint - 1, *pr->numberOfRecordsRequested,
2442                         &error, &addinfo, &number_of_records_returned, odr, m_backend,
2443                         pr->preferredRecordSyntax, element_set_name);
2444
2445         apdu_res = odr.create_presentResponse(apdu_req, error, addinfo);
2446         if (records)
2447         {
2448             apdu_res->u.presentResponse->records = records;
2449             apdu_res->u.presentResponse->numberOfRecordsReturned =
2450                 odr_intdup(odr, number_of_records_returned);
2451         }
2452         package.response() = apdu_res;
2453     }
2454 }
2455
2456 void yf::Zoom::Frontend::handle_package(mp::Package &package)
2457 {
2458     Z_GDU *gdu = package.request().get();
2459     if (!gdu)
2460         ;
2461     else if (gdu->which == Z_GDU_Z3950)
2462     {
2463         Z_APDU *apdu_req = gdu->u.z3950;
2464
2465         if (m_backend)
2466             wrbuf_rewind(m_backend->m_apdu_wrbuf);
2467         if (apdu_req->which == Z_APDU_initRequest)
2468         {
2469             mp::odr odr;
2470             package.response() = odr.create_close(
2471                 apdu_req,
2472                 Z_Close_protocolError,
2473                 "double init");
2474         }
2475         else if (apdu_req->which == Z_APDU_searchRequest)
2476         {
2477             handle_search(package);
2478         }
2479         else if (apdu_req->which == Z_APDU_presentRequest)
2480         {
2481             handle_present(package);
2482         }
2483         else
2484         {
2485             mp::odr odr;
2486             package.response() = odr.create_close(
2487                 apdu_req,
2488                 Z_Close_protocolError,
2489                 "zoom filter cannot handle this APDU");
2490             package.session().close();
2491         }
2492         if (m_backend)
2493         {
2494             WRBUF w = m_backend->m_apdu_wrbuf;
2495             package.log_write(wrbuf_buf(w), wrbuf_len(w));
2496         }
2497     }
2498     else
2499     {
2500         package.session().close();
2501     }
2502 }
2503
2504 std::string escape_cql_term(std::string inp)
2505 {
2506     std::string res;
2507     size_t l = inp.length();
2508     size_t i;
2509     for (i = 0; i < l; i++)
2510     {
2511         if (strchr("*?^\"", inp[i]))
2512             res += "\\";
2513         res += inp[i];
2514     }
2515     return res;
2516 }
2517
2518 void yf::Zoom::Frontend::auth(mp::Package &package, Z_InitRequest *req,
2519                               int *error, char **addinfo, ODR odr)
2520 {
2521     if (m_p->torus_auth_url.length() == 0)
2522         return;
2523
2524     std::string user;
2525     std::string password;
2526     if (req->idAuthentication)
2527     {
2528         Z_IdAuthentication *auth = req->idAuthentication;
2529         switch (auth->which)
2530         {
2531         case Z_IdAuthentication_open:
2532             if (auth->u.open)
2533             {
2534                 const char *cp = strchr(auth->u.open, '/');
2535                 if (cp)
2536                 {
2537                     user.assign(auth->u.open, cp - auth->u.open);
2538                     password.assign(cp + 1);
2539                 }
2540             }
2541             break;
2542         case Z_IdAuthentication_idPass:
2543             if (auth->u.idPass->userId)
2544                 user.assign(auth->u.idPass->userId);
2545             if (auth->u.idPass->password)
2546                 password.assign(auth->u.idPass->password);
2547             break;
2548         }
2549     }
2550
2551     std::string ip = package.origin().get_address();
2552     yaz_log(YLOG_LOG, "IP=%s", ip.c_str());
2553
2554     std::string torus_query;
2555     int failure_code;
2556
2557     if (user.length() && password.length())
2558     {
2559         torus_query = "userName==\"" + escape_cql_term(user) +
2560             "\" and password==\"" + escape_cql_term(password) + "\"";
2561         failure_code = YAZ_BIB1_INIT_AC_BAD_USERID_AND_OR_PASSWORD;
2562     }
2563     else
2564     {
2565         const char *ip_cstr = ip.c_str();
2566         const char *cp = strchr(ip_cstr, ':');
2567         if (cp)
2568             ip_cstr = cp + 1;
2569
2570         torus_query = "ip encloses/net.ipaddress \"";
2571         torus_query += escape_cql_term(std::string(ip_cstr));
2572         torus_query += "\"";
2573         failure_code = YAZ_BIB1_INIT_AC_BLOCKED_NETWORK_ADDRESS;
2574     }
2575
2576     std::string dummy_db;
2577     std::string dummy_realm;
2578     xmlDoc *doc = mp::get_searchable(package, m_p->torus_auth_url, dummy_db,
2579                                      torus_query, dummy_realm, m_p->proxy);
2580     if (!doc)
2581     {
2582         // something fundamental broken in lookup.
2583         *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
2584         *addinfo = odr_strdup(odr, "zoom: torus server unavailable or "
2585                               "incorrectly configured.");
2586         return;
2587     }
2588     const xmlNode *ptr = xmlDocGetRootElement(doc);
2589     if (ptr && ptr->type == XML_ELEMENT_NODE)
2590     {
2591         if (strcmp((const char *) ptr->name, "records") == 0)
2592         {
2593             ptr = ptr->children;
2594             while (ptr && ptr->type != XML_ELEMENT_NODE)
2595                 ptr = ptr->next;
2596         }
2597         if (ptr && strcmp((const char *) ptr->name, "record") == 0)
2598         {
2599             ptr = ptr->children;
2600             while (ptr && ptr->type != XML_ELEMENT_NODE)
2601                 ptr = ptr->next;
2602         }
2603         if (ptr && strcmp((const char *) ptr->name, "layer") == 0)
2604         {
2605             ptr = ptr->children;
2606             while (ptr && ptr->type != XML_ELEMENT_NODE)
2607                 ptr = ptr->next;
2608         }
2609         while (ptr)
2610         {
2611             if (ptr && ptr->type == XML_ELEMENT_NODE &&
2612                 !strcmp((const char *) ptr->name, "identityId"))
2613                 break;
2614             ptr = ptr->next;
2615         }
2616     }
2617     if (!ptr)
2618     {
2619         *error = failure_code;
2620         return;
2621     }
2622     session_realm = mp::xml::get_text(ptr);
2623 }
2624
2625 void yf::Zoom::Impl::process(mp::Package &package)
2626 {
2627     FrontendPtr f = get_frontend(package);
2628     Z_GDU *gdu = package.request().get();
2629
2630     if (f->m_is_virtual)
2631     {
2632         f->handle_package(package);
2633     }
2634     else if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
2635              Z_APDU_initRequest)
2636     {
2637         Z_InitRequest *req = gdu->u.z3950->u.initRequest;
2638         f->m_init_gdu = gdu;
2639
2640         mp::odr odr;
2641         Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
2642         Z_InitResponse *resp = apdu->u.initResponse;
2643
2644         int i;
2645         static const int masks[] = {
2646             Z_Options_search,
2647             Z_Options_present,
2648             -1
2649         };
2650         for (i = 0; masks[i] != -1; i++)
2651             if (ODR_MASK_GET(req->options, masks[i]))
2652                 ODR_MASK_SET(resp->options, masks[i]);
2653
2654         static const int versions[] = {
2655             Z_ProtocolVersion_1,
2656             Z_ProtocolVersion_2,
2657             Z_ProtocolVersion_3,
2658             -1
2659         };
2660         for (i = 0; versions[i] != -1; i++)
2661             if (ODR_MASK_GET(req->protocolVersion, versions[i]))
2662                 ODR_MASK_SET(resp->protocolVersion, versions[i]);
2663             else
2664                 break;
2665
2666         *resp->preferredMessageSize = *req->preferredMessageSize;
2667         *resp->maximumRecordSize = *req->maximumRecordSize;
2668
2669         int error = 0;
2670         char *addinfo = 0;
2671         f->auth(package, req, &error, &addinfo, odr);
2672         if (error)
2673         {
2674             resp->userInformationField =
2675                 zget_init_diagnostics(odr, error, addinfo);
2676             *resp->result = 0;
2677             package.session().close();
2678         }
2679         else
2680             f->m_is_virtual = true;
2681         package.response() = apdu;
2682     }
2683     else
2684         package.move();
2685
2686     release_frontend(package);
2687 }
2688
2689
2690 static mp::filter::Base* filter_creator()
2691 {
2692     return new mp::filter::Zoom;
2693 }
2694
2695 extern "C" {
2696     struct metaproxy_1_filter_struct metaproxy_1_filter_zoom = {
2697         0,
2698         "zoom",
2699         filter_creator
2700     };
2701 }
2702
2703
2704 /*
2705  * Local variables:
2706  * c-basic-offset: 4
2707  * c-file-style: "Stroustrup"
2708  * indent-tabs-mode: nil
2709  * End:
2710  * vim: shiftwidth=4 tabstop=8 expandtab
2711  */
2712