src/filter_zoom.cpp

   1 /* This file is part of Metaproxy.
   2    Copyright (C) 2005-2012 Index Data
   3
   4 Metaproxy is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17 */
  18
  19 #include "config.hpp"
  20
  21 #include <stdlib.h>
  22 #include <sys/types.h>
  23 #include "filter_zoom.hpp"
  24 #include <metaproxy/package.hpp>
  25 #include <metaproxy/util.hpp>
  26 #include <metaproxy/xmlutil.hpp>
  27 #include <yaz/comstack.h>
  28 #include <yaz/poll.h>
  29 #include "torus.hpp"
  30
  31 #include <libxslt/xsltutils.h>
  32 #include <libxslt/transform.h>
  33
  34 #include <boost/thread/mutex.hpp>
  35 #include <boost/thread/condition.hpp>
  36
  37 #include <yaz/yaz-version.h>
  38 #include <yaz/tpath.h>
  39 #include <yaz/srw.h>
  40 #include <yaz/ccl_xml.h>
  41 #include <yaz/ccl.h>
  42 #include <yaz/rpn2cql.h>
  43 #include <yaz/rpn2solr.h>
  44 #include <yaz/pquery.h>
  45 #include <yaz/cql.h>
  46 #include <yaz/oid_db.h>
  47 #include <yaz/diagbib1.h>
  48 #include <yaz/log.h>
  49 #include <yaz/zgdu.h>
  50 #include <yaz/querytowrbuf.h>
  51 #include <yaz/sortspec.h>
  52 #include <yaz/tokenizer.h>
  53 #include <yaz/zoom.h>
  54
  55 namespace mp = metaproxy_1;
  56 namespace yf = mp::filter;
  57
  58 namespace metaproxy_1 {
  59     namespace filter {
  60         class Zoom::Searchable : boost::noncopyable {
  61           public:
  62             std::string authentication;
  63             std::string cfAuth;
  64             std::string cfProxy;
  65             std::string cfSubDB;
  66             std::string udb;
  67             std::string target;
  68             std::string query_encoding;
  69             std::string sru;
  70             std::string sru_version;
  71             std::string request_syntax;
  72             std::string element_set;
  73             std::string record_encoding;
  74             std::string transform_xsl_fname;
  75             std::string transform_xsl_content;
  76             std::string urlRecipe;
  77             std::string contentConnector;
  78             std::string sortStrategy;
  79             std::string rpn2cql_fname;
  80             bool use_turbomarc;
  81             bool piggyback;
  82             CCL_bibset ccl_bibset;
  83             std::map<std::string, std::string> sortmap;
  84             Searchable(CCL_bibset base);
  85             ~Searchable();
  86         };
  87         class Zoom::Backend : boost::noncopyable {
  88             friend class Impl;
  89             friend class Frontend;
  90             std::string zurl;
  91             mp::wrbuf m_apdu_wrbuf;
  92             ZOOM_connection m_connection;
  93             ZOOM_resultset m_resultset;
  94             std::string m_frontend_database;
  95             SearchablePtr sptr;
  96             xsltStylesheetPtr xsp;
  97             std::string cproxy_host;
  98             bool enable_cproxy;
  99             bool enable_explain;
 100             xmlDoc *explain_doc;
 101             std::string m_proxy;
 102             cql_transform_t cqlt;
 103         public:
 104             Backend();
 105             ~Backend();
 106             void connect(std::string zurl, int *error, char **addinfo,
 107                          ODR odr);
 108             void search(ZOOM_query q, Odr_int *hits,
 109                         int *error, char **addinfo, ODR odr);
 110             void present(Odr_int start, Odr_int number, ZOOM_record *recs,
 111                          int *error, char **addinfo, ODR odr);
 112             void set_option(const char *name, const char *value);
 113             void set_option(const char *name, std::string value);
 114             const char *get_option(const char *name);
 115             void get_zoom_error(int *error, char **addinfo, ODR odr);
 116         };
 117         class Zoom::Frontend : boost::noncopyable {
 118             friend class Impl;
 119             Impl *m_p;
 120             bool m_is_virtual;
 121             bool m_in_use;
 122             std::string session_realm;
 123             yazpp_1::GDU m_init_gdu;
 124             BackendPtr m_backend;
 125             void handle_package(mp::Package &package);
 126             void handle_search(mp::Package &package);
 127
 128             void auth(mp::Package &package, Z_InitRequest *req,
 129                       int *error, char **addinfo, ODR odr);
 130
 131             BackendPtr explain_search(mp::Package &package,
 132                                       std::string &database,
 133                                       int *error,
 134                                       char **addinfo,
 135                                       mp::odr &odr,
 136                                       std::string torus_url,
 137                                       std::string &torus_db,
 138                                       std::string &realm);
 139             void handle_present(mp::Package &package);
 140             BackendPtr get_backend_from_databases(mp::Package &package,
 141                                                   std::string &database,
 142                                                   int *error,
 143                                                   char **addinfo,
 144                                                   mp::odr &odr,
 145                                                   int *proxy_step);
 146
 147             bool create_content_session(mp::Package &package,
 148                                         BackendPtr b,
 149                                         int *error,
 150                                         char **addinfo,
 151                                         ODR odr,
 152                                         std::string authentication,
 153                                         std::string proxy,
 154                                         std::string realm);
 155
 156             void prepare_elements(BackendPtr b,
 157                                   Odr_oid *preferredRecordSyntax,
 158                                   const char *element_set_name,
 159                                   bool &enable_pz2_retrieval,
 160                                   bool &enable_pz2_transform,
 161                                   bool &enable_record_transform,
 162                                   bool &assume_marc8_charset);
 163
 164             Z_Records *get_records(Package &package,
 165                                    Odr_int start,
 166                                    Odr_int number_to_present,
 167                                    int *error,
 168                                    char **addinfo,
 169                                    Odr_int *number_of_records_returned,
 170                                    ODR odr, BackendPtr b,
 171                                    Odr_oid *preferredRecordSyntax,
 172                                    const char *element_set_name);
 173             Z_Records *get_explain_records(Package &package,
 174                                            Odr_int start,
 175                                            Odr_int number_to_present,
 176                                            int *error,
 177                                            char **addinfo,
 178                                            Odr_int *number_of_records_returned,
 179                                            ODR odr, BackendPtr b,
 180                                            Odr_oid *preferredRecordSyntax,
 181                                            const char *element_set_name);
 182             bool retry(mp::Package &package,
 183                        mp::odr &odr,
 184                        BackendPtr b,
 185                        int &error, char **addinfo,
 186                        int &proxy_step, int &same_retries,
 187                        int &proxy_retries);
 188             void log_diagnostic(mp::Package &package,
 189                                 int error, const char *addinfo);
 190         public:
 191             Frontend(Impl *impl);
 192             ~Frontend();
 193         };
 194         class Zoom::Impl {
 195             friend class Frontend;
 196         public:
 197             Impl();
 198             ~Impl();
 199             void process(metaproxy_1::Package & package);
 200             void configure(const xmlNode * ptr, bool test_only,
 201                            const char *path);
 202         private:
 203             void configure_local_records(const xmlNode * ptr, bool test_only);
 204             bool check_proxy(const char *proxy);
 205
 206
 207
 208             FrontendPtr get_frontend(mp::Package &package);
 209             void release_frontend(mp::Package &package);
 210             SearchablePtr parse_torus_record(const xmlNode *ptr);
 211             struct cql_node *convert_cql_fields(struct cql_node *cn, ODR odr);
 212             std::map<mp::Session, FrontendPtr> m_clients;
 213             boost::mutex m_mutex;
 214             boost::condition m_cond_session_ready;
 215             std::string torus_searchable_url;
 216             std::string torus_content_url;
 217             std::string torus_auth_url;
 218             std::string default_realm;
 219             std::map<std::string,std::string> fieldmap;
 220             std::string xsldir;
 221             std::string file_path;
 222             std::string content_proxy_server;
 223             std::string content_tmp_file;
 224             std::string content_config_file;
 225             bool apdu_log;
 226             CCL_bibset bibset;
 227             std::string element_transform;
 228             std::string element_raw;
 229             std::string proxy;
 230             xsltStylesheetPtr explain_xsp;
 231             xsltStylesheetPtr record_xsp;
 232             std::map<std::string,SearchablePtr> s_map;
 233             std::string zoom_timeout;
 234             int proxy_timeout;
 235         };
 236     }
 237 }
 238
 239
 240 static xmlNode *xml_node_search(xmlNode *ptr, int *num, int m)
 241 {
 242     while (ptr)
 243     {
 244         if (ptr->type == XML_ELEMENT_NODE &&
 245             !strcmp((const char *) ptr->name, "recordData"))
 246         {
 247             (*num)++;
 248             if (m == *num)
 249                 return ptr;
 250         }
 251         else  // else: we don't want to find nested nodes
 252         {
 253             xmlNode *ret_node = xml_node_search(ptr->children, num, m);
 254             if (ret_node)
 255                 return ret_node;
 256         }
 257         ptr = ptr->next;
 258     }
 259     return 0;
 260 }
 261
 262 // define Pimpl wrapper forwarding to Impl
 263
 264 yf::Zoom::Zoom() : m_p(new Impl)
 265 {
 266 }
 267
 268 yf::Zoom::~Zoom()
 269 {  // must have a destructor because of boost::scoped_ptr
 270 }
 271
 272 void yf::Zoom::configure(const xmlNode *xmlnode, bool test_only,
 273                          const char *path)
 274 {
 275     m_p->configure(xmlnode, test_only, path);
 276 }
 277
 278 void yf::Zoom::process(mp::Package &package) const
 279 {
 280     m_p->process(package);
 281 }
 282
 283
 284 // define Implementation stuff
 285
 286 yf::Zoom::Backend::Backend()
 287 {
 288     m_connection = ZOOM_connection_create(0);
 289     ZOOM_connection_save_apdu_wrbuf(m_connection, m_apdu_wrbuf);
 290     m_resultset = 0;
 291     xsp = 0;
 292     enable_cproxy = true;
 293     enable_explain = false;
 294     explain_doc = 0;
 295     cqlt = 0;
 296 }
 297
 298 yf::Zoom::Backend::~Backend()
 299 {
 300     if (xsp)
 301         xsltFreeStylesheet(xsp);
 302     if (explain_doc)
 303         xmlFreeDoc(explain_doc);
 304     cql_transform_close(cqlt);
 305     ZOOM_connection_destroy(m_connection);
 306     ZOOM_resultset_destroy(m_resultset);
 307 }
 308
 309
 310 void yf::Zoom::Backend::get_zoom_error(int *error, char **addinfo,
 311                                        ODR odr)
 312 {
 313     const char *msg = 0;
 314     const char *zoom_addinfo = 0;
 315     const char *dset = 0;
 316     int error0 = ZOOM_connection_error_x(m_connection, &msg,
 317                                          &zoom_addinfo, &dset);
 318     if (error0)
 319     {
 320         if (!dset)
 321             dset = "Unknown";
 322
 323         if (!strcmp(dset, "info:srw/diagnostic/1"))
 324             *error = yaz_diag_srw_to_bib1(error0);
 325         else if (!strcmp(dset, "Bib-1"))
 326             *error = error0;
 327         else if (!strcmp(dset, "ZOOM"))
 328         {
 329             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
 330             if (error0 == ZOOM_ERROR_INIT)
 331                 *error = YAZ_BIB1_INIT_AC_AUTHENTICATION_SYSTEM_ERROR;
 332             else if (error0 == ZOOM_ERROR_DECODE)
 333             {
 334                 if (zoom_addinfo)
 335                 {
 336                     if (strstr(zoom_addinfo, "Authentication") ||
 337                         strstr(zoom_addinfo, "authentication"))
 338                         *error = YAZ_BIB1_INIT_AC_AUTHENTICATION_SYSTEM_ERROR;
 339                 }
 340             }
 341         }
 342         else
 343             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
 344
 345         *addinfo = (char *) odr_malloc(
 346             odr, 30 + strlen(dset) + strlen(msg) +
 347             (zoom_addinfo ? strlen(zoom_addinfo) : 0));
 348         **addinfo = '\0';
 349         if (zoom_addinfo && *zoom_addinfo)
 350         {
 351             strcpy(*addinfo, zoom_addinfo);
 352             strcat(*addinfo, " ");
 353         }
 354         sprintf(*addinfo + strlen(*addinfo), "(%s %d %s)", dset, error0, msg);
 355     }
 356 }
 357
 358 void yf::Zoom::Backend::connect(std::string zurl,
 359                                 int *error, char **addinfo,
 360                                 ODR odr)
 361 {
 362     ZOOM_connection_connect(m_connection, zurl.length() ? zurl.c_str() : 0, 0);
 363     get_zoom_error(error, addinfo, odr);
 364 }
 365
 366 void yf::Zoom::Backend::search(ZOOM_query q, Odr_int *hits,
 367                                int *error, char **addinfo, ODR odr)
 368 {
 369     ZOOM_resultset_destroy(m_resultset);
 370     m_resultset = ZOOM_connection_search(m_connection, q);
 371     get_zoom_error(error, addinfo, odr);
 372     if (*error == 0)
 373         *hits = ZOOM_resultset_size(m_resultset);
 374     else
 375         *hits = 0;
 376 }
 377
 378 void yf::Zoom::Backend::present(Odr_int start, Odr_int number,
 379                                 ZOOM_record *recs,
 380                                 int *error, char **addinfo, ODR odr)
 381 {
 382     ZOOM_resultset_records(m_resultset, recs, start, number);
 383     get_zoom_error(error, addinfo, odr);
 384 }
 385
 386 void yf::Zoom::Backend::set_option(const char *name, const char *value)
 387 {
 388     ZOOM_connection_option_set(m_connection, name, value);
 389     if (m_resultset)
 390         ZOOM_resultset_option_set(m_resultset, name, value);
 391 }
 392
 393 void yf::Zoom::Backend::set_option(const char *name, std::string value)
 394 {
 395     set_option(name, value.c_str());
 396 }
 397
 398 const char *yf::Zoom::Backend::get_option(const char *name)
 399 {
 400     return ZOOM_connection_option_get(m_connection, name);
 401 }
 402
 403 yf::Zoom::Searchable::Searchable(CCL_bibset base)
 404 {
 405     piggyback = true;
 406     use_turbomarc = true;
 407     sortStrategy = "embed";
 408     ccl_bibset = ccl_qual_dup(base);
 409 }
 410
 411 yf::Zoom::Searchable::~Searchable()
 412 {
 413     ccl_qual_rm(&ccl_bibset);
 414 }
 415
 416 yf::Zoom::Frontend::Frontend(Impl *impl) :
 417     m_p(impl), m_is_virtual(false), m_in_use(true)
 418 {
 419 }
 420
 421 yf::Zoom::Frontend::~Frontend()
 422 {
 423 }
 424
 425 yf::Zoom::FrontendPtr yf::Zoom::Impl::get_frontend(mp::Package &package)
 426 {
 427     boost::mutex::scoped_lock lock(m_mutex);
 428
 429     std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
 430
 431     while(true)
 432     {
 433         it = m_clients.find(package.session());
 434         if (it == m_clients.end())
 435             break;
 436
 437         if (!it->second->m_in_use)
 438         {
 439             it->second->m_in_use = true;
 440             return it->second;
 441         }
 442         m_cond_session_ready.wait(lock);
 443     }
 444     FrontendPtr f(new Frontend(this));
 445     m_clients[package.session()] = f;
 446     f->m_in_use = true;
 447     return f;
 448 }
 449
 450 void yf::Zoom::Impl::release_frontend(mp::Package &package)
 451 {
 452     boost::mutex::scoped_lock lock(m_mutex);
 453     std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
 454
 455     it = m_clients.find(package.session());
 456     if (it != m_clients.end())
 457     {
 458         if (package.session().is_closed())
 459         {
 460             m_clients.erase(it);
 461         }
 462         else
 463         {
 464             it->second->m_in_use = false;
 465         }
 466         m_cond_session_ready.notify_all();
 467     }
 468 }
 469
 470 yf::Zoom::Impl::Impl() :
 471     apdu_log(false), element_transform("pz2") , element_raw("raw"),
 472     zoom_timeout("40"), proxy_timeout(1)
 473 {
 474     bibset = ccl_qual_mk();
 475
 476     explain_xsp = 0;
 477     record_xsp = 0;
 478     srand((unsigned int) time(0));
 479 }
 480
 481 yf::Zoom::Impl::~Impl()
 482 {
 483     if (explain_xsp)
 484         xsltFreeStylesheet(explain_xsp);
 485     ccl_qual_rm(&bibset);
 486 }
 487
 488 yf::Zoom::SearchablePtr yf::Zoom::Impl::parse_torus_record(const xmlNode *ptr)
 489 {
 490     Zoom::SearchablePtr s(new Searchable(bibset));
 491
 492     for (ptr = ptr->children; ptr; ptr = ptr->next)
 493     {
 494         if (ptr->type != XML_ELEMENT_NODE)
 495             continue;
 496         if (!strcmp((const char *) ptr->name, "layer"))
 497             ptr = ptr->children;
 498         else if (!strcmp((const char *) ptr->name,
 499                          "authentication"))
 500         {
 501             s->authentication = mp::xml::get_text(ptr);
 502         }
 503         else if (!strcmp((const char *) ptr->name,
 504                          "cfAuth"))
 505         {
 506             s->cfAuth = mp::xml::get_text(ptr);
 507         }
 508         else if (!strcmp((const char *) ptr->name,
 509                          "cfProxy"))
 510         {
 511             s->cfProxy = mp::xml::get_text(ptr);
 512         }
 513         else if (!strcmp((const char *) ptr->name,
 514                          "cfSubDB"))
 515         {
 516             s->cfSubDB = mp::xml::get_text(ptr);
 517         }
 518         else if (!strcmp((const char *) ptr->name,
 519                          "contentConnector"))
 520         {
 521             s->contentConnector = mp::xml::get_text(ptr);
 522         }
 523         else if (!strcmp((const char *) ptr->name, "udb"))
 524         {
 525             s->udb = mp::xml::get_text(ptr);
 526         }
 527         else if (!strcmp((const char *) ptr->name, "zurl"))
 528         {
 529             s->target = mp::xml::get_text(ptr);
 530         }
 531         else if (!strcmp((const char *) ptr->name, "sru"))
 532         {
 533             s->sru = mp::xml::get_text(ptr);
 534         }
 535         else if (!strcmp((const char *) ptr->name, "SRUVersion") ||
 536                  !strcmp((const char *) ptr->name, "sruVersion"))
 537         {
 538             s->sru_version = mp::xml::get_text(ptr);
 539         }
 540         else if (!strcmp((const char *) ptr->name,
 541                          "queryEncoding"))
 542         {
 543             s->query_encoding = mp::xml::get_text(ptr);
 544         }
 545         else if (!strcmp((const char *) ptr->name,
 546                          "piggyback"))
 547         {
 548             s->piggyback = mp::xml::get_bool(ptr, true);
 549         }
 550         else if (!strcmp((const char *) ptr->name,
 551                          "requestSyntax"))
 552         {
 553             s->request_syntax = mp::xml::get_text(ptr);
 554         }
 555         else if (!strcmp((const char *) ptr->name,
 556                          "elementSet"))
 557         {
 558             s->element_set = mp::xml::get_text(ptr);
 559         }
 560         else if (!strcmp((const char *) ptr->name,
 561                          "recordEncoding"))
 562         {
 563             s->record_encoding = mp::xml::get_text(ptr);
 564         }
 565         else if (!strcmp((const char *) ptr->name,
 566                          "transform"))
 567         {
 568             s->transform_xsl_fname = mp::xml::get_text(ptr);
 569         }
 570         else if (!strcmp((const char *) ptr->name,
 571                          "literalTransform"))
 572         {
 573             s->transform_xsl_content = mp::xml::get_text(ptr);
 574         }
 575         else if (!strcmp((const char *) ptr->name,
 576                          "urlRecipe"))
 577         {
 578             s->urlRecipe = mp::xml::get_text(ptr);
 579         }
 580         else if (!strcmp((const char *) ptr->name,
 581                          "useTurboMarc"))
 582         {
 583             ; // useTurboMarc is ignored
 584         }
 585         else if (!strncmp((const char *) ptr->name,
 586                           "cclmap_", 7))
 587         {
 588             std::string value = mp::xml::get_text(ptr);
 589             if (value.length() > 0)
 590             {
 591                 ccl_qual_fitem(s->ccl_bibset, value.c_str(),
 592                                (const char *) ptr->name + 7);
 593             }
 594         }
 595         else if (!strncmp((const char *) ptr->name,
 596                           "sortmap_", 8))
 597         {
 598             std::string value = mp::xml::get_text(ptr);
 599             s->sortmap[(const char *) ptr->name + 8] = value;
 600         }
 601         else if (!strcmp((const char *) ptr->name,
 602                           "sortStrategy"))
 603         {
 604             s->sortStrategy = mp::xml::get_text(ptr);
 605         }
 606         else if (!strcmp((const char *) ptr->name, "rpn2cql"))
 607             s->rpn2cql_fname = mp::xml::get_text(ptr);
 608     }
 609     return s;
 610 }
 611
 612 void yf::Zoom::Impl::configure_local_records(const xmlNode *ptr, bool test_only)
 613 {
 614     while (ptr && ptr->type != XML_ELEMENT_NODE)
 615         ptr = ptr->next;
 616
 617     if (ptr)
 618     {
 619         if (!strcmp((const char *) ptr->name, "records"))
 620         {
 621             for (ptr = ptr->children; ptr; ptr = ptr->next)
 622             {
 623                 if (ptr->type != XML_ELEMENT_NODE)
 624                     continue;
 625                 if (!strcmp((const char *) ptr->name, "record"))
 626                 {
 627                     SearchablePtr s = parse_torus_record(ptr);
 628                     if (s)
 629                     {
 630                         std::string udb = s->udb;
 631                         if (udb.length())
 632                             s_map[s->udb] = s;
 633                         else
 634                         {
 635                             throw mp::filter::FilterException
 636                                 ("No udb for local torus record");
 637                         }
 638                     }
 639                 }
 640                 else
 641                 {
 642                     throw mp::filter::FilterException
 643                         ("Bad element "
 644                          + std::string((const char *) ptr->name)
 645                          + " in zoom filter inside element "
 646                          "<torus><records>");
 647                 }
 648             }
 649         }
 650         else
 651         {
 652             throw mp::filter::FilterException
 653                 ("Bad element "
 654                  + std::string((const char *) ptr->name)
 655                  + " in zoom filter inside element <torus>");
 656         }
 657     }
 658 }
 659
 660 void yf::Zoom::Impl::configure(const xmlNode *ptr, bool test_only,
 661                                const char *path)
 662 {
 663     std::string explain_xslt_fname;
 664     std::string record_xslt_fname;
 665
 666     if (path && *path)
 667     {
 668         file_path = path;
 669     }
 670     for (ptr = ptr->children; ptr; ptr = ptr->next)
 671     {
 672         if (ptr->type != XML_ELEMENT_NODE)
 673             continue;
 674         else if (!strcmp((const char *) ptr->name, "torus"))
 675         {
 676             const struct _xmlAttr *attr;
 677             for (attr = ptr->properties; attr; attr = attr->next)
 678             {
 679                 if (!strcmp((const char *) attr->name, "url"))
 680                     torus_searchable_url = mp::xml::get_text(attr->children);
 681                 else if (!strcmp((const char *) attr->name, "content_url"))
 682                     torus_content_url = mp::xml::get_text(attr->children);
 683                 else if (!strcmp((const char *) attr->name, "auth_url"))
 684                     torus_auth_url = mp::xml::get_text(attr->children);
 685                 else if (!strcmp((const char *) attr->name, "realm"))
 686                     default_realm = mp::xml::get_text(attr->children);
 687                 else if (!strcmp((const char *) attr->name, "xsldir"))
 688                     xsldir = mp::xml::get_text(attr->children);
 689                 else if (!strcmp((const char *) attr->name, "element_transform"))
 690                     element_transform = mp::xml::get_text(attr->children);
 691                 else if (!strcmp((const char *) attr->name, "element_raw"))
 692                     element_raw = mp::xml::get_text(attr->children);
 693                 else if (!strcmp((const char *) attr->name, "proxy"))
 694                     proxy = mp::xml::get_text(attr->children);
 695                 else if (!strcmp((const char *) attr->name, "explain_xsl"))
 696                     explain_xslt_fname = mp::xml::get_text(attr->children);
 697                 else if (!strcmp((const char *) attr->name, "record_xsl"))
 698                     record_xslt_fname = mp::xml::get_text(attr->children);
 699                 else
 700                     throw mp::filter::FilterException(
 701                         "Bad attribute " + std::string((const char *)
 702                                                        attr->name));
 703             }
 704             // If content_url is not given, use value of searchable, to
 705             // ensure backwards compatibility
 706             if (!torus_content_url.length())
 707                 torus_content_url = torus_searchable_url;
 708             configure_local_records(ptr->children, test_only);
 709         }
 710         else if (!strcmp((const char *) ptr->name, "cclmap"))
 711         {
 712             const char *addinfo = 0;
 713             ccl_xml_config(bibset, ptr, &addinfo);
 714         }
 715         else if (!strcmp((const char *) ptr->name, "fieldmap"))
 716         {
 717             const struct _xmlAttr *attr;
 718             std::string ccl_field;
 719             std::string cql_field;
 720             for (attr = ptr->properties; attr; attr = attr->next)
 721             {
 722                 if (!strcmp((const char *) attr->name, "ccl"))
 723                     ccl_field = mp::xml::get_text(attr->children);
 724                 else if (!strcmp((const char *) attr->name, "cql"))
 725                     cql_field = mp::xml::get_text(attr->children);
 726                 else
 727                     throw mp::filter::FilterException(
 728                         "Bad attribute " + std::string((const char *)
 729                                                        attr->name));
 730             }
 731             if (cql_field.length())
 732                 fieldmap[cql_field] = ccl_field;
 733         }
 734         else if (!strcmp((const char *) ptr->name, "contentProxy"))
 735         {
 736             const struct _xmlAttr *attr;
 737             for (attr = ptr->properties; attr; attr = attr->next)
 738             {
 739                 if (!strcmp((const char *) attr->name, "server"))
 740                 {
 741                     yaz_log(YLOG_WARN,
 742                             "contentProxy's server attribute is deprecated");
 743                     yaz_log(YLOG_LOG,
 744                             "Specify config_file instead. For example:");
 745                     yaz_log(YLOG_LOG,
 746                             " content_file=\"/etc/cf-proxy/cproxy.cfg\"");
 747                     content_proxy_server = mp::xml::get_text(attr->children);
 748                 }
 749                 else if (!strcmp((const char *) attr->name, "tmp_file"))
 750                     content_tmp_file = mp::xml::get_text(attr->children);
 751                 else if (!strcmp((const char *) attr->name, "config_file"))
 752                     content_config_file = mp::xml::get_text(attr->children);
 753                 else
 754                     throw mp::filter::FilterException(
 755                         "Bad attribute " + std::string((const char *)
 756                                                        attr->name));
 757             }
 758         }
 759         else if (!strcmp((const char *) ptr->name, "log"))
 760         {
 761             const struct _xmlAttr *attr;
 762             for (attr = ptr->properties; attr; attr = attr->next)
 763             {
 764                 if (!strcmp((const char *) attr->name, "apdu"))
 765                     apdu_log = mp::xml::get_bool(attr->children, false);
 766                 else
 767                     throw mp::filter::FilterException(
 768                         "Bad attribute " + std::string((const char *)
 769                                                        attr->name));
 770             }
 771         }
 772         else if (!strcmp((const char *) ptr->name, "zoom"))
 773         {
 774             const struct _xmlAttr *attr;
 775             for (attr = ptr->properties; attr; attr = attr->next)
 776             {
 777                 if (!strcmp((const char *) attr->name, "timeout"))
 778                     zoom_timeout = mp::xml::get_text(attr->children);
 779                 else if (!strcmp((const char *) attr->name, "proxy_timeout"))
 780                     proxy_timeout = mp::xml::get_int(attr->children, 1);
 781                 else
 782                     throw mp::filter::FilterException(
 783                         "Bad attribute " + std::string((const char *)
 784                                                        attr->name));
 785             }
 786         }
 787         else
 788         {
 789             throw mp::filter::FilterException
 790                 ("Bad element "
 791                  + std::string((const char *) ptr->name)
 792                  + " in zoom filter");
 793         }
 794     }
 795
 796     if (explain_xslt_fname.length())
 797     {
 798         const char *path = 0;
 799
 800         if (xsldir.length())
 801             path = xsldir.c_str();
 802         else
 803             path = file_path.c_str();
 804
 805         char fullpath[1024];
 806         char *cp = yaz_filepath_resolve(explain_xslt_fname.c_str(),
 807                                         path, 0, fullpath);
 808         if (!cp)
 809         {
 810             throw mp::filter::FilterException
 811                 ("Cannot read XSLT " + explain_xslt_fname);
 812         }
 813
 814         xmlDoc *xsp_doc = xmlParseFile(cp);
 815         if (!xsp_doc)
 816         {
 817             throw mp::filter::FilterException
 818                 ("Cannot parse XSLT " + explain_xslt_fname);
 819         }
 820
 821         explain_xsp = xsltParseStylesheetDoc(xsp_doc);
 822         if (!explain_xsp)
 823         {
 824             xmlFreeDoc(xsp_doc);
 825             throw mp::filter::FilterException
 826                 ("Cannot parse XSLT " + explain_xslt_fname);
 827
 828         }
 829     }
 830
 831     if (record_xslt_fname.length())
 832     {
 833         const char *path = 0;
 834
 835         if (xsldir.length())
 836             path = xsldir.c_str();
 837         else
 838             path = file_path.c_str();
 839
 840         char fullpath[1024];
 841         char *cp = yaz_filepath_resolve(record_xslt_fname.c_str(),
 842                                         path, 0, fullpath);
 843         if (!cp)
 844         {
 845             throw mp::filter::FilterException
 846                 ("Cannot read XSLT " + record_xslt_fname);
 847         }
 848
 849         xmlDoc *xsp_doc = xmlParseFile(cp);
 850         if (!xsp_doc)
 851         {
 852             throw mp::filter::FilterException
 853                 ("Cannot parse XSLT " + record_xslt_fname);
 854         }
 855
 856         record_xsp = xsltParseStylesheetDoc(xsp_doc);
 857         if (!record_xsp)
 858         {
 859             xmlFreeDoc(xsp_doc);
 860             throw mp::filter::FilterException
 861                 ("Cannot parse XSLT " + record_xslt_fname);
 862
 863         }
 864     }
 865 }
 866
 867 bool yf::Zoom::Frontend::create_content_session(mp::Package &package,
 868                                                 BackendPtr b,
 869                                                 int *error, char **addinfo,
 870                                                 ODR odr,
 871                                                 std::string authentication,
 872                                                 std::string proxy,
 873                                                 std::string realm)
 874 {
 875     if (b->sptr->contentConnector.length())
 876     {
 877         std::string proxyhostname;
 878         std::string tmp_file;
 879         bool legacy_format = false;
 880
 881         if (m_p->content_proxy_server.length())
 882         {
 883             proxyhostname = m_p->content_proxy_server;
 884             legacy_format = true;
 885         }
 886
 887         if (m_p->content_tmp_file.length())
 888             tmp_file = m_p->content_tmp_file;
 889
 890         if (m_p->content_config_file.length())
 891         {
 892             FILE *inf = fopen(m_p->content_config_file.c_str(), "r");
 893             if (inf)
 894             {
 895                 char buf[1024];
 896                 while (fgets(buf, sizeof(buf)-1, inf))
 897                 {
 898                     char *cp;
 899                     cp = strchr(buf, '#');
 900                     if (cp)
 901                         *cp = '\0';
 902                     cp = strchr(buf, '\n');
 903                     if (cp)
 904                         *cp = '\0';
 905                     cp = strchr(buf, ':');
 906                     if (cp)
 907                     {
 908                         char *cp1 = cp;
 909                         while (cp1 != buf && cp1[-1] == ' ')
 910                             cp1--;
 911                         *cp1 = '\0';
 912                         cp++;
 913                         while (*cp == ' ')
 914                             cp++;
 915                         if (!strcmp(buf, "proxyhostname"))
 916                             proxyhostname = cp;
 917                         if (!strcmp(buf, "sessiondir") && *cp)
 918                         {
 919                             if (cp[strlen(cp)-1] == '/')
 920                                 cp[strlen(cp)-1] = '\0';
 921                             tmp_file = std::string(cp) + std::string("/cf.XXXXXX.p");
 922                         }
 923                     }
 924                 }
 925                 fclose(inf);
 926             }
 927             else
 928             {
 929                 package.log("zoom", YLOG_WARN|YLOG_ERRNO,
 930                             "unable to open content config %s",
 931                             m_p->content_config_file.c_str());
 932                 *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
 933                 *addinfo = (char *)  odr_malloc(odr, 60 + tmp_file.length());
 934                 sprintf(*addinfo, "unable to open content config %s",
 935                         m_p->content_config_file.c_str());
 936                 return false;
 937             }
 938         }
 939
 940         if (proxyhostname.length() == 0)
 941         {
 942             package.log("zoom", YLOG_WARN, "no proxyhostname");
 943             return true;
 944         }
 945         if (tmp_file.length() == 0)
 946         {
 947             package.log("zoom", YLOG_WARN, "no tmp_file");
 948             return true;
 949         }
 950
 951         char *fname = xstrdup(tmp_file.c_str());
 952         char *xx = strstr(fname, "XXXXXX");
 953         if (!xx)
 954         {
 955             package.log("zoom", YLOG_WARN, "bad tmp_file %s", tmp_file.c_str());
 956             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
 957             *addinfo = (char *)  odr_malloc(odr, 60 + tmp_file.length());
 958             sprintf(*addinfo, "bad format of content tmp_file: %s",
 959                     tmp_file.c_str());
 960             xfree(fname);
 961             return false;
 962         }
 963         char tmp_char = xx[6];
 964         sprintf(xx, "%06d", ((unsigned) rand()) % 1000000);
 965         if (legacy_format)
 966             b->cproxy_host = std::string(xx) + "." + proxyhostname;
 967         else
 968             b->cproxy_host = proxyhostname + "/" + xx;
 969         xx[6] = tmp_char;
 970
 971         FILE *file = fopen(fname, "w");
 972         if (!file)
 973         {
 974             package.log("zoom", YLOG_WARN|YLOG_ERRNO, "create %s", fname);
 975             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
 976             *addinfo = (char *)  odr_malloc(odr, 40 + strlen(fname));
 977             sprintf(*addinfo, "Could not create %s", fname);
 978             xfree(fname);
 979             return false;
 980         }
 981         mp::wrbuf w;
 982         wrbuf_puts(w, "#content_proxy\n");
 983         wrbuf_printf(w, "connector: %s\n", b->sptr->contentConnector.c_str());
 984         if (authentication.length())
 985             wrbuf_printf(w, "auth: %s\n", authentication.c_str());
 986         if (proxy.length())
 987             wrbuf_printf(w, "proxy: %s\n", proxy.c_str());
 988         if (realm.length())
 989             wrbuf_printf(w, "realm: %s\n", realm.c_str());
 990
 991         fwrite(w.buf(), 1, w.len(), file);
 992         fclose(file);
 993         package.log("zoom", YLOG_LOG, "content file: %s", fname);
 994         xfree(fname);
 995     }
 996     return true;
 997 }
 998
 999 yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
1000     mp::Package &package,
1001     std::string &database, int *error, char **addinfo, mp::odr &odr,
1002     int *proxy_step)
1003 {
1004     bool connection_reuse = false;
1005     std::string proxy;
1006
1007     std::list<BackendPtr>::const_iterator map_it;
1008     if (m_backend && !m_backend->enable_explain &&
1009         m_backend->m_frontend_database == database)
1010     {
1011         connection_reuse = true;
1012         proxy = m_backend->m_proxy;
1013     }
1014
1015     std::string input_args;
1016     std::string torus_db;
1017     size_t db_arg_pos = database.find(',');
1018     if (db_arg_pos != std::string::npos)
1019     {
1020         torus_db = database.substr(0, db_arg_pos);
1021         input_args = database.substr(db_arg_pos + 1);
1022     }
1023     else
1024         torus_db = database;
1025
1026     std::string authentication;
1027     std::string content_authentication;
1028     std::string content_proxy;
1029     std::string realm = session_realm;
1030     if (realm.length() == 0)
1031         realm = m_p->default_realm;
1032
1033     const char *param_user = 0;
1034     const char *param_password = 0;
1035     const char *param_content_user = 0;
1036     const char *param_content_password = 0;
1037     const char *param_nocproxy = 0;
1038     int no_parms = 0;
1039
1040     char **names;
1041     char **values;
1042     int no_out_args = 0;
1043     if (input_args.length())
1044         no_parms = yaz_uri_to_array(input_args.c_str(),
1045                                     odr, &names, &values);
1046     // adding 10 because we'll be adding other URL args
1047     const char **out_names = (const char **)
1048         odr_malloc(odr, (10 + no_parms) * sizeof(*out_names));
1049     const char **out_values = (const char **)
1050         odr_malloc(odr, (10 + no_parms) * sizeof(*out_values));
1051
1052     // may be changed if it's a content connection
1053     std::string torus_url = m_p->torus_searchable_url;
1054     int i;
1055     for (i = 0; i < no_parms; i++)
1056     {
1057         const char *name = names[i];
1058         const char *value = values[i];
1059         assert(name);
1060         assert(value);
1061         if (!strcmp(name, "user"))
1062             param_user = value;
1063         else if (!strcmp(name, "password"))
1064             param_password = value;
1065         else if (!strcmp(name, "content-user"))
1066             param_content_user = value;
1067         else if (!strcmp(name, "content-password"))
1068             param_content_password = value;
1069         else if (!strcmp(name, "content-proxy"))
1070             content_proxy = value;
1071         else if (!strcmp(name, "nocproxy"))
1072             param_nocproxy = value;
1073         else if (!strcmp(name, "proxy"))
1074         {
1075             char **dstr;
1076             int dnum = 0;
1077             nmem_strsplit(((ODR) odr)->mem, ",", value, &dstr, &dnum);
1078             if (connection_reuse)
1079             {
1080                 // find the step after our current proxy
1081                 int i;
1082                 for (i = 0; i < dnum; i++)
1083                     if (!strcmp(proxy.c_str(), dstr[i]))
1084                         break;
1085                 if (i >= dnum - 1)
1086                     *proxy_step = 0;
1087                 else
1088                     *proxy_step = i + 1;
1089             }
1090             else
1091             {
1092                 // step is known.. Guess our proxy from it
1093                 if (*proxy_step >= dnum)
1094                     *proxy_step = 0;
1095                 else
1096                 {
1097                     proxy = dstr[*proxy_step];
1098
1099                     (*proxy_step)++;
1100                     if (*proxy_step == dnum)
1101                         *proxy_step = 0;
1102                 }
1103             }
1104         }
1105         else if (!strcmp(name, "cproxysession"))
1106         {
1107             out_names[no_out_args] = name;
1108             out_values[no_out_args++] = value;
1109             torus_url = m_p->torus_content_url;
1110         }
1111         else if (!strcmp(name, "realm") && session_realm.length() == 0)
1112             realm = value;
1113         else if (!strcmp(name, "torus_url") && session_realm.length() == 0)
1114             torus_url = value;
1115         else if (name[0] == 'x' && name[1] == '-')
1116         {
1117             out_names[no_out_args] = name;
1118             out_values[no_out_args++] = value;
1119         }
1120         else
1121         {
1122             BackendPtr notfound;
1123             char *msg = (char*) odr_malloc(odr, strlen(name) + 30);
1124             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
1125             sprintf(msg, "Bad database argument: %s", name);
1126             *addinfo = msg;
1127             return notfound;
1128         }
1129     }
1130     if (proxy.length())
1131         package.log("zoom", YLOG_LOG, "proxy: %s", proxy.c_str());
1132
1133     if (connection_reuse)
1134     {
1135         m_backend->connect("", error, addinfo, odr);
1136         return m_backend;
1137     }
1138
1139     if (param_user)
1140     {
1141         authentication = std::string(param_user);
1142         if (param_password)
1143             authentication += "/" + std::string(param_password);
1144     }
1145     if (param_content_user)
1146     {
1147         content_authentication = std::string(param_content_user);
1148         if (param_content_password)
1149             content_authentication += "/" + std::string(param_content_password);
1150     }
1151
1152     if (torus_db.compare("IR-Explain---1") == 0)
1153         return explain_search(package, database, error, addinfo, odr, torus_url,
1154                               torus_db, realm);
1155
1156     SearchablePtr sptr;
1157
1158     std::map<std::string,SearchablePtr>::iterator it;
1159     it = m_p->s_map.find(torus_db);
1160     if (it != m_p->s_map.end())
1161         sptr = it->second;
1162     else if (torus_url.length() > 0)
1163     {
1164         std::string torus_query = "udb==" + torus_db;
1165         xmlDoc *doc = mp::get_searchable(package,torus_url, torus_db,
1166                                          torus_query,
1167                                          realm, m_p->proxy);
1168         if (!doc)
1169         {
1170             *error = YAZ_BIB1_UNSPECIFIED_ERROR;
1171             *addinfo = odr_strdup(odr, "Torus server unavailable or "
1172                                   "incorrectly configured");
1173             BackendPtr b;
1174             return b;
1175         }
1176         const xmlNode *ptr = xmlDocGetRootElement(doc);
1177         if (ptr && ptr->type == XML_ELEMENT_NODE)
1178         {
1179             if (!strcmp((const char *) ptr->name, "record"))
1180             {
1181                 sptr = m_p->parse_torus_record(ptr);
1182             }
1183             else if (!strcmp((const char *) ptr->name, "records"))
1184             {
1185                 for (ptr = ptr->children; ptr; ptr = ptr->next)
1186                 {
1187                     if (ptr->type == XML_ELEMENT_NODE
1188                         && !strcmp((const char *) ptr->name, "record"))
1189                     {
1190                         if (sptr)
1191                         {
1192                             *error = YAZ_BIB1_UNSPECIFIED_ERROR;
1193                             *addinfo = (char*)
1194                                 odr_malloc(odr, 40 + torus_db.length());
1195                             sprintf(*addinfo, "multiple records for udb=%s",
1196                                     database.c_str());
1197                             xmlFreeDoc(doc);
1198                             BackendPtr b;
1199                             return b;
1200                         }
1201                         sptr = m_p->parse_torus_record(ptr);
1202                     }
1203                 }
1204             }
1205             else
1206             {
1207                 *error = YAZ_BIB1_UNSPECIFIED_ERROR;
1208                 *addinfo = (char*) odr_malloc(
1209                     odr, 40 + strlen((const char *) ptr->name));
1210                 sprintf(*addinfo, "bad root element for torus: %s", ptr->name);
1211                 xmlFreeDoc(doc);
1212                 BackendPtr b;
1213                 return b;
1214             }
1215         }
1216         xmlFreeDoc(doc);
1217     }
1218
1219     if (!sptr)
1220     {
1221         *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
1222         *addinfo = odr_strdup(odr, torus_db.c_str());
1223         BackendPtr b;
1224         return b;
1225     }
1226
1227     xsltStylesheetPtr xsp = 0;
1228     if (sptr->transform_xsl_content.length())
1229     {
1230         xmlDoc *xsp_doc = xmlParseMemory(sptr->transform_xsl_content.c_str(),
1231                                          sptr->transform_xsl_content.length());
1232         if (!xsp_doc)
1233         {
1234             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
1235             *addinfo = (char *) odr_malloc(odr, 40);
1236             sprintf(*addinfo, "xmlParseMemory failed");
1237             BackendPtr b;
1238             return b;
1239         }
1240         xsp = xsltParseStylesheetDoc(xsp_doc);
1241         if (!xsp)
1242         {
1243             *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
1244             *addinfo = odr_strdup(odr, "xsltParseStylesheetDoc failed");
1245             BackendPtr b;
1246             xmlFreeDoc(xsp_doc);
1247             return b;
1248         }
1249     }
1250     else if (sptr->transform_xsl_fname.length())
1251     {
1252         const char *path = 0;
1253
1254         if (m_p->xsldir.length())
1255             path = m_p->xsldir.c_str();
1256         else
1257             path = m_p->file_path.c_str();
1258         std::string fname;
1259
1260         char fullpath[1024];
1261         char *cp = yaz_filepath_resolve(sptr->transform_xsl_fname.c_str(),
1262                                         path, 0, fullpath);
1263         if (cp)
1264             fname.assign(cp);
1265         else
1266         {
1267             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
1268             *addinfo = (char *)
1269                 odr_malloc(odr, 40 + sptr->transform_xsl_fname.length());
1270             sprintf(*addinfo, "File could not be read: %s",
1271                     sptr->transform_xsl_fname.c_str());
1272             BackendPtr b;
1273             return b;
1274         }
1275         xmlDoc *xsp_doc = xmlParseFile(fname.c_str());
1276         if (!xsp_doc)
1277         {
1278             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
1279             *addinfo = (char *) odr_malloc(odr, 40 + fname.length());
1280             sprintf(*addinfo, "xmlParseFile failed. File: %s", fname.c_str());
1281             BackendPtr b;
1282             return b;
1283         }
1284         xsp = xsltParseStylesheetDoc(xsp_doc);
1285         if (!xsp)
1286         {
1287             *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
1288             *addinfo = odr_strdup(odr, "xsltParseStylesheetDoc failed");
1289             BackendPtr b;
1290             xmlFreeDoc(xsp_doc);
1291             return b;
1292         }
1293     }
1294
1295     cql_transform_t cqlt = 0;
1296     if (sptr->rpn2cql_fname.length())
1297     {
1298         char fullpath[1024];
1299         char *cp = yaz_filepath_resolve(sptr->rpn2cql_fname.c_str(),
1300                                         m_p->file_path.c_str(), 0, fullpath);
1301         if (cp)
1302             cqlt = cql_transform_open_fname(fullpath);
1303     }
1304     else
1305         cqlt = cql_transform_create();
1306
1307     if (!cqlt)
1308     {
1309         *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
1310         *addinfo = odr_strdup(odr, "Missing/invalid cql2rpn file");
1311         BackendPtr b;
1312         xsltFreeStylesheet(xsp);
1313         return b;
1314     }
1315
1316     m_backend.reset();
1317
1318     BackendPtr b(new Backend);
1319
1320     b->cqlt = cqlt;
1321     b->sptr = sptr;
1322     b->xsp = xsp;
1323     b->m_frontend_database = database;
1324     b->enable_cproxy = param_nocproxy ? false : true;
1325
1326     if (sptr->query_encoding.length())
1327         b->set_option("rpnCharset", sptr->query_encoding);
1328
1329     b->set_option("timeout", m_p->zoom_timeout.c_str());
1330
1331     if (m_p->apdu_log)
1332         b->set_option("apdulog", "1");
1333
1334     if (sptr->piggyback && sptr->sru.length())
1335         b->set_option("count", "1"); /* some SRU servers INSIST on getting
1336                                         maximumRecords > 0 */
1337     b->set_option("piggyback", sptr->piggyback ? "1" : "0");
1338
1339     if (authentication.length() == 0)
1340         authentication = sptr->authentication;
1341
1342     if (proxy.length() == 0)
1343         proxy = sptr->cfProxy;
1344     b->m_proxy = proxy;
1345
1346     if (sptr->cfAuth.length())
1347     {
1348         // A CF target
1349         b->set_option("user", sptr->cfAuth);
1350         if (authentication.length())
1351         {
1352             size_t found = authentication.find('/');
1353             if (found != std::string::npos)
1354             {
1355                 out_names[no_out_args] = "user";
1356                 out_values[no_out_args++] =
1357                     odr_strdup(odr, authentication.substr(0, found).c_str());
1358
1359                 out_names[no_out_args] = "password";
1360                 out_values[no_out_args++] =
1361                     odr_strdup(odr, authentication.substr(found+1).c_str());
1362             }
1363             else
1364             {
1365                 out_names[no_out_args] = "user";
1366                 out_values[no_out_args++] =
1367                     odr_strdup(odr, authentication.c_str());
1368             }
1369         }
1370         if (proxy.length())
1371         {
1372             out_names[no_out_args] = "proxy";
1373             out_values[no_out_args++] = odr_strdup(odr, proxy.c_str());
1374         }
1375         if (sptr->cfSubDB.length())
1376         {
1377             out_names[no_out_args] = "subdatabase";
1378             out_values[no_out_args++] = odr_strdup(odr, sptr->cfSubDB.c_str());
1379         }
1380         if (param_nocproxy)
1381         {
1382             out_names[no_out_args] = "nocproxy";
1383             out_values[no_out_args++] = odr_strdup(odr, param_nocproxy);
1384         }
1385     }
1386     else
1387     {
1388         size_t found = authentication.find('/');
1389
1390         if (sptr->sru.length() && found != std::string::npos)
1391         {
1392             b->set_option("user", authentication.substr(0, found));
1393             b->set_option("password", authentication.substr(found+1));
1394         }
1395         else
1396             b->set_option("user", authentication);
1397
1398         if (proxy.length())
1399             b->set_option("proxy", proxy);
1400     }
1401     std::string url;
1402     if (sptr->sru.length())
1403     {
1404         url = "http://" + sptr->target;
1405         b->set_option("sru", sptr->sru);
1406
1407         if (sptr->sru_version.length())
1408             b->set_option("sru_version", sptr->sru_version);
1409     }
1410     else
1411     {
1412         url = sptr->target;
1413     }
1414     if (no_out_args)
1415     {
1416         char *x_args = 0;
1417         out_names[no_out_args] = 0; // terminate list
1418
1419         yaz_array_to_uri(&x_args, odr, (char **) out_names,
1420                          (char **) out_values);
1421         url += "," + std::string(x_args);
1422     }
1423     package.log("zoom", YLOG_LOG, "url: %s", url.c_str());
1424     b->connect(url, error, addinfo, odr);
1425     if (*error == 0 && b->enable_cproxy)
1426         create_content_session(package, b, error, addinfo, odr,
1427                                content_authentication.length() ?
1428                                content_authentication : authentication,
1429                                content_proxy.length() ? content_proxy : proxy,
1430                                realm);
1431     if (*error == 0)
1432         m_backend = b;
1433     return b;
1434 }
1435
1436 void yf::Zoom::Frontend::prepare_elements(BackendPtr b,
1437                                           Odr_oid *preferredRecordSyntax,
1438                                           const char *element_set_name,
1439                                           bool &enable_pz2_retrieval,
1440                                           bool &enable_pz2_transform,
1441                                           bool &enable_record_transform,
1442                                           bool &assume_marc8_charset)
1443 {
1444     char oid_name_str[OID_STR_MAX];
1445     const char *syntax_name = 0;
1446
1447     if (preferredRecordSyntax &&
1448         !oid_oidcmp(preferredRecordSyntax, yaz_oid_recsyn_xml))
1449     {
1450         if (element_set_name &&
1451             !strcmp(element_set_name, m_p->element_transform.c_str()))
1452         {
1453             enable_pz2_retrieval = true;
1454             enable_pz2_transform = true;
1455         }
1456         else if (element_set_name &&
1457                  !strcmp(element_set_name, m_p->element_raw.c_str()))
1458         {
1459             enable_pz2_retrieval = true;
1460         }
1461         else if (m_p->record_xsp)
1462         {
1463             enable_pz2_retrieval = true;
1464             enable_pz2_transform = true;
1465             enable_record_transform = true;
1466         }
1467     }
1468
1469     if (enable_pz2_retrieval)
1470     {
1471         std::string configured_request_syntax = b->sptr->request_syntax;
1472         if (configured_request_syntax.length())
1473         {
1474             syntax_name = configured_request_syntax.c_str();
1475             const Odr_oid *syntax_oid =
1476                 yaz_string_to_oid(yaz_oid_std(), CLASS_RECSYN, syntax_name);
1477             if (!oid_oidcmp(syntax_oid, yaz_oid_recsyn_usmarc)
1478                 || !oid_oidcmp(syntax_oid, yaz_oid_recsyn_opac))
1479                 assume_marc8_charset = true;
1480         }
1481     }
1482     else if (preferredRecordSyntax)
1483         syntax_name =
1484             yaz_oid_to_string_buf(preferredRecordSyntax, 0, oid_name_str);
1485
1486     if (b->sptr->sru.length())
1487         syntax_name = "XML";
1488
1489     b->set_option("preferredRecordSyntax", syntax_name);
1490
1491     if (enable_pz2_retrieval)
1492     {
1493         element_set_name = 0;
1494         if (b->sptr->element_set.length())
1495             element_set_name = b->sptr->element_set.c_str();
1496     }
1497
1498     b->set_option("elementSetName", element_set_name);
1499     if (b->sptr->sru.length() && element_set_name)
1500         b->set_option("schema", element_set_name);
1501 }
1502
1503 Z_Records *yf::Zoom::Frontend::get_explain_records(
1504     mp::Package &package,
1505     Odr_int start,
1506     Odr_int number_to_present,
1507     int *error,
1508     char **addinfo,
1509     Odr_int *number_of_records_returned,
1510     ODR odr,
1511     BackendPtr b,
1512     Odr_oid *preferredRecordSyntax,
1513     const char *element_set_name)
1514 {
1515     Odr_int i;
1516     Z_Records *records = 0;
1517
1518     if (!b->explain_doc)
1519     {
1520         return records;
1521     }
1522     if (number_to_present > 10000)
1523         number_to_present = 10000;
1524
1525     xmlNode *ptr = xmlDocGetRootElement(b->explain_doc);
1526
1527     Z_NamePlusRecordList *npl = (Z_NamePlusRecordList *)
1528         odr_malloc(odr, sizeof(*npl));
1529     npl->records = (Z_NamePlusRecord **)
1530         odr_malloc(odr, number_to_present * sizeof(*npl->records));
1531
1532     for (i = 0; i < number_to_present; i++)
1533     {
1534         int num = 0;
1535         xmlNode *res = xml_node_search(ptr, &num, start + i + 1);
1536         if (!res)
1537             break;
1538         xmlBufferPtr xml_buf = xmlBufferCreate();
1539         xmlNode *tmp_node = xmlCopyNode(res->children, 1);
1540         xmlNodeDump(xml_buf, tmp_node->doc, tmp_node, 0, 0);
1541
1542         Z_NamePlusRecord *npr =
1543             (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
1544         npr->databaseName = odr_strdup(odr, b->m_frontend_database.c_str());
1545         npr->which = Z_NamePlusRecord_databaseRecord;
1546         npr->u.databaseRecord =
1547             z_ext_record_xml(odr,
1548                              (const char *) xml_buf->content, xml_buf->use);
1549         npl->records[i] = npr;
1550         xmlFreeNode(tmp_node);
1551         xmlBufferFree(xml_buf);
1552     }
1553     records = (Z_Records*) odr_malloc(odr, sizeof(*records));
1554     records->which = Z_Records_DBOSD;
1555     records->u.databaseOrSurDiagnostics = npl;
1556
1557     npl->num_records = i;
1558     *number_of_records_returned = i;
1559     return records;
1560 }
1561
1562
1563 Z_Records *yf::Zoom::Frontend::get_records(mp::Package &package,
1564                                            Odr_int start,
1565                                            Odr_int number_to_present,
1566                                            int *error,
1567                                            char **addinfo,
1568                                            Odr_int *number_of_records_returned,
1569                                            ODR odr,
1570                                            BackendPtr b,
1571                                            Odr_oid *preferredRecordSyntax,
1572                                            const char *element_set_name)
1573 {
1574     *number_of_records_returned = 0;
1575     Z_Records *records = 0;
1576     bool enable_pz2_retrieval = false; // whether target profile is used
1577     bool enable_pz2_transform = false; // whether XSLT is used as well
1578     bool assume_marc8_charset = false;
1579     bool enable_record_transform = false;
1580
1581     prepare_elements(b, preferredRecordSyntax,
1582                      element_set_name,
1583                      enable_pz2_retrieval,
1584                      enable_pz2_transform,
1585                      enable_record_transform,
1586                      assume_marc8_charset);
1587
1588     package.log("zoom", YLOG_LOG, "pz2_retrieval: %s . pz2_transform: %s",
1589                 enable_pz2_retrieval ? "yes" : "no",
1590                 enable_pz2_transform ? "yes" : "no");
1591
1592     if (start < 0 || number_to_present <=0)
1593         return records;
1594
1595     if (number_to_present > 10000)
1596         number_to_present = 10000;
1597
1598     ZOOM_record *recs = (ZOOM_record *)
1599         odr_malloc(odr, (size_t) number_to_present * sizeof(*recs));
1600
1601     b->present(start, number_to_present, recs, error, addinfo, odr);
1602
1603     int i = 0;
1604     if (!*error)
1605     {
1606         for (i = 0; i < number_to_present; i++)
1607             if (!recs[i])
1608                 break;
1609     }
1610     if (i > 0)
1611     {  // only return records if no error and at least one record
1612
1613         const char *xsl_parms[3];
1614         mp::wrbuf cproxy_host;
1615
1616         if (b->enable_cproxy && b->cproxy_host.length())
1617         {
1618             wrbuf_puts(cproxy_host, "\"");
1619             wrbuf_puts(cproxy_host, b->cproxy_host.c_str());
1620             wrbuf_puts(cproxy_host, "/\"");
1621
1622             xsl_parms[0] = "cproxyhost";
1623             xsl_parms[1] = wrbuf_cstr(cproxy_host);
1624             xsl_parms[2] = 0;
1625         }
1626         else
1627         {
1628             xsl_parms[0] = 0;
1629         }
1630
1631         char *odr_database = odr_strdup(odr,
1632                                         b->m_frontend_database.c_str());
1633         Z_NamePlusRecordList *npl = (Z_NamePlusRecordList *)
1634             odr_malloc(odr, sizeof(*npl));
1635         *number_of_records_returned = i;
1636         npl->num_records = i;
1637         npl->records = (Z_NamePlusRecord **)
1638             odr_malloc(odr, i * sizeof(*npl->records));
1639         for (i = 0; i < number_to_present; i++)
1640         {
1641             Z_NamePlusRecord *npr = 0;
1642             const char *addinfo;
1643
1644             int sur_error = ZOOM_record_error(recs[i], 0 /* msg */,
1645                                               &addinfo, 0 /* diagset */);
1646
1647             if (sur_error)
1648             {
1649                 log_diagnostic(package, sur_error, addinfo);
1650                 npr = zget_surrogateDiagRec(odr, odr_database, sur_error,
1651                                             addinfo);
1652             }
1653             else if (enable_pz2_retrieval)
1654             {
1655                 char rec_type_str[100];
1656                 const char *record_encoding = 0;
1657
1658                 if (b->sptr->record_encoding.length())
1659                     record_encoding = b->sptr->record_encoding.c_str();
1660                 else if (assume_marc8_charset)
1661                     record_encoding = "marc8";
1662
1663                 strcpy(rec_type_str, b->sptr->use_turbomarc ? "txml" : "xml");
1664                 if (record_encoding)
1665                 {
1666                     strcat(rec_type_str, "; charset=");
1667                     strcat(rec_type_str, record_encoding);
1668                 }
1669
1670                 package.log("zoom", YLOG_LOG, "Getting record of type %s",
1671                             rec_type_str);
1672                 int rec_len;
1673                 xmlChar *xmlrec_buf = 0;
1674                 const char *rec_buf = ZOOM_record_get(recs[i], rec_type_str,
1675                                                       &rec_len);
1676                 if (!rec_buf && !npr)
1677                 {
1678                     std::string addinfo("ZOOM_record_get failed for type ");
1679
1680                     int error = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1681                     addinfo += rec_type_str;
1682                     log_diagnostic(package, error, addinfo.c_str());
1683                     npr = zget_surrogateDiagRec(odr, odr_database,
1684                                                 error, addinfo.c_str());
1685                 }
1686                 else
1687                 {
1688                     package.log_write(rec_buf, rec_len);
1689                     package.log_write("\r\n", 2);
1690                 }
1691
1692                 if (rec_buf && b->xsp && enable_pz2_transform)
1693                 {
1694                     xmlDoc *rec_doc = xmlParseMemory(rec_buf, rec_len);
1695                     if (!rec_doc)
1696                     {
1697                         const char *addinfo = "xml parse failed for record";
1698                         int error = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1699                         log_diagnostic(package, error, addinfo);
1700                         npr = zget_surrogateDiagRec(
1701                             odr, odr_database, error, addinfo);
1702                     }
1703                     else
1704                     {
1705                         // first stage XSLT - per target
1706                         xsltStylesheetPtr xsp = b->xsp;
1707                         xmlDoc *rec_res = xsltApplyStylesheet(xsp, rec_doc,
1708                                                               xsl_parms);
1709                         // insert generated-url
1710                         if (rec_res)
1711                         {
1712                             std::string res =
1713                                 mp::xml::url_recipe_handle(rec_res,
1714                                                            b->sptr->urlRecipe);
1715                             if (res.length())
1716                             {
1717                                 xmlNode *ptr = xmlDocGetRootElement(rec_res);
1718                                 while (ptr && ptr->type != XML_ELEMENT_NODE)
1719                                     ptr = ptr->next;
1720                                 xmlNode *c =
1721                                     xmlNewChild(ptr, 0, BAD_CAST "metadata", 0);
1722                                 xmlNewProp(c, BAD_CAST "type", BAD_CAST
1723                                            "generated-url");
1724                                 xmlNode * t = xmlNewText(BAD_CAST res.c_str());
1725                                 xmlAddChild(c, t);
1726                             }
1727                         }
1728                         // second stage XSLT - common
1729                         if (rec_res && m_p->record_xsp &&
1730                             enable_record_transform)
1731                         {
1732                             xmlDoc *tmp_doc = rec_res;
1733
1734                             xsp = m_p->record_xsp;
1735                             rec_res = xsltApplyStylesheet(xsp, tmp_doc,
1736                                                           xsl_parms);
1737                             xmlFreeDoc(tmp_doc);
1738                         }
1739                         // get result out of it
1740                         if (rec_res)
1741                         {
1742                             xsltSaveResultToString(&xmlrec_buf, &rec_len,
1743                                                    rec_res, xsp);
1744                             rec_buf = (const char *) xmlrec_buf;
1745                             package.log_write(rec_buf, rec_len);
1746
1747                             xmlFreeDoc(rec_res);
1748                         }
1749                         if (!rec_buf)
1750                         {
1751                             std::string addinfo;
1752                             int error =
1753                                 YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1754
1755                             addinfo = "xslt apply failed for "
1756                                 + b->sptr->transform_xsl_fname;
1757                             log_diagnostic(package, error, addinfo.c_str());
1758                             npr = zget_surrogateDiagRec(
1759                                 odr, odr_database, error, addinfo.c_str());
1760                         }
1761                         xmlFreeDoc(rec_doc);
1762                     }
1763                 }
1764
1765                 if (!npr)
1766                 {
1767                     if (!rec_buf)
1768                         npr = zget_surrogateDiagRec(
1769                             odr, odr_database,
1770                             YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
1771                             rec_type_str);
1772                     else
1773                     {
1774                         npr = (Z_NamePlusRecord *)
1775                             odr_malloc(odr, sizeof(*npr));
1776                         npr->databaseName = odr_database;
1777                         npr->which = Z_NamePlusRecord_databaseRecord;
1778                         npr->u.databaseRecord =
1779                             z_ext_record_xml(odr, rec_buf, rec_len);
1780                     }
1781                 }
1782                 if (xmlrec_buf)
1783                     xmlFree(xmlrec_buf);
1784             }
1785             else
1786             {
1787                 Z_External *ext =
1788                     (Z_External *) ZOOM_record_get(recs[i], "ext", 0);
1789                 if (ext)
1790                 {
1791                     npr = (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
1792                     npr->databaseName = odr_database;
1793                     npr->which = Z_NamePlusRecord_databaseRecord;
1794                     npr->u.databaseRecord = ext;
1795                 }
1796                 else
1797                 {
1798                     npr = zget_surrogateDiagRec(
1799                         odr, odr_database,
1800                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
1801                         "ZOOM_record, type ext");
1802                 }
1803             }
1804             npl->records[i] = npr;
1805         }
1806         records = (Z_Records*) odr_malloc(odr, sizeof(*records));
1807         records->which = Z_Records_DBOSD;
1808         records->u.databaseOrSurDiagnostics = npl;
1809     }
1810     return records;
1811 }
1812
1813 struct cql_node *yf::Zoom::Impl::convert_cql_fields(struct cql_node *cn,
1814                                                     ODR odr)
1815 {
1816     struct cql_node *r = 0;
1817     if (!cn)
1818         return 0;
1819     switch (cn->which)
1820     {
1821     case CQL_NODE_ST:
1822         if (cn->u.st.index)
1823         {
1824             std::map<std::string,std::string>::const_iterator it;
1825             it = fieldmap.find(cn->u.st.index);
1826             if (it == fieldmap.end())
1827                 return cn;
1828             if (it->second.length())
1829                 cn->u.st.index = odr_strdup(odr, it->second.c_str());
1830             else
1831                 cn->u.st.index = 0;
1832         }
1833         break;
1834     case CQL_NODE_BOOL:
1835         r = convert_cql_fields(cn->u.boolean.left, odr);
1836         if (!r)
1837             r = convert_cql_fields(cn->u.boolean.right, odr);
1838         break;
1839     case CQL_NODE_SORT:
1840         r = convert_cql_fields(cn->u.sort.search, odr);
1841         break;
1842     }
1843     return r;
1844 }
1845
1846 void yf::Zoom::Frontend::log_diagnostic(mp::Package &package,
1847                                         int error, const char *addinfo)
1848 {
1849     const char *err_msg = yaz_diag_bib1_str(error);
1850     if (addinfo)
1851         package.log("zoom", YLOG_WARN, "Diagnostic %d %s: %s",
1852                     error, err_msg, addinfo);
1853     else
1854         package.log("zoom", YLOG_WARN, "Diagnostic %d %s:",
1855                     error, err_msg);
1856 }
1857
1858 yf::Zoom::BackendPtr yf::Zoom::Frontend::explain_search(mp::Package &package,
1859                                                         std::string &database,
1860                                                         int *error,
1861                                                         char **addinfo,
1862                                                         mp::odr &odr,
1863                                                         std::string torus_url,
1864                                                         std::string &torus_db,
1865                                                         std::string &realm)
1866 {
1867     m_backend.reset();
1868
1869     BackendPtr b(new Backend);
1870
1871     b->m_frontend_database = database;
1872     b->enable_explain = true;
1873
1874     Z_GDU *gdu = package.request().get();
1875     Z_APDU *apdu_req = gdu->u.z3950;
1876     Z_SearchRequest *sr = apdu_req->u.searchRequest;
1877     Z_Query *query = sr->query;
1878
1879     if (!m_p->explain_xsp)
1880     {
1881         *error = YAZ_BIB1_UNSPECIFIED_ERROR;
1882         *addinfo =
1883             odr_strdup(odr, "IR-Explain---1 unsupported. "
1884                        "Torus explain_xsl not defined");
1885         return m_backend;
1886     }
1887     else if (query->which == Z_Query_type_104 &&
1888         query->u.type_104->which == Z_External_CQL)
1889     {
1890         std::string torus_query(query->u.type_104->u.cql);
1891         xmlDoc *doc = mp::get_searchable(package, torus_url, "",
1892                                          torus_query,
1893                                          realm, m_p->proxy);
1894         if (m_p->explain_xsp)
1895         {
1896             xmlDoc *rec_res =  xsltApplyStylesheet(m_p->explain_xsp, doc, 0);
1897
1898             xmlFreeDoc(doc);
1899             doc = rec_res;
1900         }
1901         if (!doc)
1902         {
1903             *error = YAZ_BIB1_UNSPECIFIED_ERROR;
1904             *addinfo = odr_strdup(odr, "Torus server unavailable or "
1905                                   "incorrectly configured");
1906         }
1907         else
1908         {
1909             xmlNode *ptr = xmlDocGetRootElement(doc);
1910             int hits = 0;
1911
1912             xml_node_search(ptr, &hits, 0);
1913
1914             Z_APDU *apdu_res = odr.create_searchResponse(apdu_req, 0, 0);
1915             apdu_res->u.searchResponse->resultCount = odr_intdup(odr, hits);
1916             package.response() = apdu_res;
1917             m_backend = b;
1918         }
1919         if (b->explain_doc)
1920             xmlFreeDoc(b->explain_doc);
1921         b->explain_doc = doc;
1922         return m_backend;
1923     }
1924     else
1925     {
1926         *error = YAZ_BIB1_QUERY_TYPE_UNSUPP;
1927         *addinfo = odr_strdup(odr, "IR-Explain---1 only supports CQL");
1928         return m_backend;
1929     }
1930 }
1931
1932 static bool wait_conn(COMSTACK cs, int secs)
1933 {
1934     struct yaz_poll_fd pfd;
1935
1936     yaz_poll_add(pfd.input_mask, yaz_poll_except);
1937     if (cs->io_pending && CS_WANT_WRITE)
1938         yaz_poll_add(pfd.input_mask, yaz_poll_write);
1939     if (cs->io_pending & CS_WANT_READ)
1940         yaz_poll_add(pfd.input_mask, yaz_poll_read);
1941
1942     pfd.fd = cs_fileno(cs);
1943     pfd.client_data = 0;
1944
1945     int ret = yaz_poll(&pfd, 1, secs, 0);
1946     return ret > 0;
1947 }
1948
1949 bool yf::Zoom::Impl::check_proxy(const char *proxy)
1950 {
1951     COMSTACK conn = 0;
1952     const char *uri = "http://localhost/";
1953     void *add;
1954     mp::odr odr;
1955     bool outcome = false;
1956     conn = cs_create_host_proxy(uri, 0, &add, proxy);
1957
1958     if (!conn)
1959         return false;
1960
1961     Z_GDU *gdu = z_get_HTTP_Request_uri(odr, uri, 0, 1);
1962     gdu->u.HTTP_Request->method = odr_strdup(odr, "GET");
1963
1964     if (z_GDU(odr, &gdu, 0, 0))
1965     {
1966         int len;
1967         char *buf = odr_getbuf(odr, &len, 0);
1968
1969         int ret = cs_connect(conn, add);
1970         if (ret > 0 || (ret == 0 && wait_conn(conn, 1)))
1971         {
1972             while (1)
1973             {
1974                 ret = cs_put(conn, buf, len);
1975                 if (ret != 1)
1976                     break;
1977                 if (!wait_conn(conn, proxy_timeout))
1978                     break;
1979             }
1980             if (ret == 0)
1981                 outcome = true;
1982         }
1983     }
1984     cs_close(conn);
1985     return outcome;
1986 }
1987
1988 bool yf::Zoom::Frontend::retry(mp::Package &package,
1989                                mp::odr &odr,
1990                                BackendPtr b,
1991                                int &error, char **addinfo,
1992                                int &proxy_step, int &same_retries,
1993                                int &proxy_retries)
1994 {
1995     if (b && b->m_proxy.length() && !m_p->check_proxy(b->m_proxy.c_str()))
1996     {
1997         log_diagnostic(package, error, *addinfo);
1998         package.log("zoom", YLOG_LOG, "proxy %s fails", b->m_proxy.c_str());
1999         m_backend.reset();
2000         if (proxy_step) // there is a failover
2001         {
2002             proxy_retries++;
2003             package.log("zoom", YLOG_WARN, "search failed: trying next proxy");
2004             return true;
2005         }
2006         error = YAZ_BIB1_PROXY_FAILURE;
2007         *addinfo = odr_strdup(odr, b->m_proxy.c_str());
2008     }
2009     else if (same_retries == 0 && proxy_retries == 0)
2010     {
2011         log_diagnostic(package, error, *addinfo);
2012         same_retries++;
2013         package.log("zoom", YLOG_WARN, "search failed: retry");
2014         m_backend.reset();
2015         proxy_step = 0;
2016         return true;
2017     }
2018     return false;
2019 }
2020
2021 void yf::Zoom::Frontend::handle_search(mp::Package &package)
2022 {
2023     Z_GDU *gdu = package.request().get();
2024     Z_APDU *apdu_req = gdu->u.z3950;
2025     Z_APDU *apdu_res = 0;
2026     mp::odr odr;
2027     Z_SearchRequest *sr = apdu_req->u.searchRequest;
2028     if (sr->num_databaseNames != 1)
2029     {
2030         int error = YAZ_BIB1_TOO_MANY_DATABASES_SPECIFIED;
2031         log_diagnostic(package, error, 0);
2032         apdu_res = odr.create_searchResponse(apdu_req, error, 0);
2033         package.response() = apdu_res;
2034         return;
2035     }
2036     int proxy_step = 0;
2037     int same_retries = 0;
2038     int proxy_retries = 0;
2039
2040 next_proxy:
2041
2042     int error = 0;
2043     char *addinfo = 0;
2044     std::string db(sr->databaseNames[0]);
2045
2046     BackendPtr b = get_backend_from_databases(package, db, &error,
2047                                               &addinfo, odr, &proxy_step);
2048     if (error)
2049     {
2050         if (retry(package, odr, b, error, &addinfo, proxy_step,
2051                   same_retries, proxy_retries))
2052             goto next_proxy;
2053     }
2054     if (error)
2055     {
2056         log_diagnostic(package, error, addinfo);
2057         apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
2058         package.response() = apdu_res;
2059         return;
2060     }
2061     if (!b || b->enable_explain)
2062         return;
2063
2064     b->set_option("setname", "default");
2065
2066     bool enable_pz2_retrieval = false;
2067     bool enable_pz2_transform = false;
2068     bool enable_record_transform = false;
2069     bool assume_marc8_charset = false;
2070     prepare_elements(b, sr->preferredRecordSyntax, 0 /*element_set_name */,
2071                      enable_pz2_retrieval,
2072                      enable_pz2_transform,
2073                      enable_record_transform,
2074                      assume_marc8_charset);
2075
2076     Odr_int hits = 0;
2077     Z_Query *query = sr->query;
2078     mp::wrbuf ccl_wrbuf;
2079     mp::wrbuf pqf_wrbuf;
2080     std::string sortkeys;
2081
2082     if (query->which == Z_Query_type_1 || query->which == Z_Query_type_101)
2083     {
2084         // RPN
2085         yaz_rpnquery_to_wrbuf(pqf_wrbuf, query->u.type_1);
2086     }
2087     else if (query->which == Z_Query_type_2)
2088     {
2089         // CCL
2090         wrbuf_write(ccl_wrbuf, (const char *) query->u.type_2->buf,
2091                     query->u.type_2->len);
2092     }
2093     else if (query->which == Z_Query_type_104 &&
2094              query->u.type_104->which == Z_External_CQL)
2095     {
2096         // CQL
2097         const char *cql = query->u.type_104->u.cql;
2098         CQL_parser cp = cql_parser_create();
2099         int r = cql_parser_string(cp, cql);
2100         package.log("zoom", YLOG_LOG, "CQL: %s", cql);
2101         if (r)
2102         {
2103             cql_parser_destroy(cp);
2104             error = YAZ_BIB1_MALFORMED_QUERY;
2105             const char *addinfo = "CQL syntax error";
2106             log_diagnostic(package, error, addinfo);
2107             apdu_res =
2108                 odr.create_searchResponse(apdu_req, error, addinfo);
2109             package.response() = apdu_res;
2110             return;
2111         }
2112         struct cql_node *cn = cql_parser_result(cp);
2113         struct cql_node *cn_error = m_p->convert_cql_fields(cn, odr);
2114         if (cn_error)
2115         {
2116             // hopefully we are getting a ptr to a index+relation+term node
2117             error = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
2118             addinfo = 0;
2119             if (cn_error->which == CQL_NODE_ST)
2120                 addinfo = cn_error->u.st.index;
2121
2122             log_diagnostic(package, error, addinfo);
2123             apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
2124             package.response() = apdu_res;
2125             cql_parser_destroy(cp);
2126             return;
2127         }
2128         r = cql_to_ccl(cn, wrbuf_vp_puts,  ccl_wrbuf);
2129         if (r)
2130         {
2131             error = YAZ_BIB1_MALFORMED_QUERY;
2132             const char *addinfo = "CQL to CCL conversion error";
2133
2134             log_diagnostic(package, error, addinfo);
2135             apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
2136             package.response() = apdu_res;
2137             cql_parser_destroy(cp);
2138             return;
2139         }
2140
2141         mp::wrbuf sru_sortkeys_wrbuf;
2142         if (cql_sortby_to_sortkeys(cn, wrbuf_vp_puts, sru_sortkeys_wrbuf))
2143         {
2144             error = YAZ_BIB1_ILLEGAL_SORT_RELATION;
2145             const char *addinfo = "CQL to CCL sortby conversion";
2146
2147             log_diagnostic(package, error, addinfo);
2148             apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
2149             package.response() = apdu_res;
2150             cql_parser_destroy(cp);
2151             return;
2152         }
2153         mp::wrbuf sort_spec_wrbuf;
2154         yaz_srw_sortkeys_to_sort_spec(wrbuf_cstr(sru_sortkeys_wrbuf),
2155                                       sort_spec_wrbuf);
2156         yaz_tok_cfg_t tc = yaz_tok_cfg_create();
2157         yaz_tok_parse_t tp =
2158             yaz_tok_parse_buf(tc, wrbuf_cstr(sort_spec_wrbuf));
2159         yaz_tok_cfg_destroy(tc);
2160
2161         /* go through sortspec and map fields */
2162         int token = yaz_tok_move(tp);
2163         while (token != YAZ_TOK_EOF)
2164         {
2165             if (token == YAZ_TOK_STRING)
2166             {
2167                 const char *field = yaz_tok_parse_string(tp);
2168                 std::map<std::string,std::string>::iterator it;
2169                 it = b->sptr->sortmap.find(field);
2170                 if (it != b->sptr->sortmap.end())
2171                     sortkeys += it->second;
2172                 else
2173                     sortkeys += field;
2174             }
2175             sortkeys += " ";
2176             token = yaz_tok_move(tp);
2177             if (token == YAZ_TOK_STRING)
2178             {
2179                 sortkeys += yaz_tok_parse_string(tp);
2180             }
2181             if (token != YAZ_TOK_EOF)
2182             {
2183                 sortkeys += " ";
2184                 token = yaz_tok_move(tp);
2185             }
2186         }
2187         yaz_tok_parse_destroy(tp);
2188         cql_parser_destroy(cp);
2189     }
2190     else
2191     {
2192         error = YAZ_BIB1_QUERY_TYPE_UNSUPP;
2193         const char *addinfo = 0;
2194         log_diagnostic(package, error, addinfo);
2195         apdu_res =  odr.create_searchResponse(apdu_req, error, addinfo);
2196         package.response() = apdu_res;
2197         return;
2198     }
2199
2200     if (ccl_wrbuf.len())
2201     {
2202         // CCL to PQF
2203         assert(pqf_wrbuf.len() == 0);
2204         int cerror, cpos;
2205         struct ccl_rpn_node *cn;
2206         package.log("zoom", YLOG_LOG, "CCL: %s", wrbuf_cstr(ccl_wrbuf));
2207         cn = ccl_find_str(b->sptr->ccl_bibset, wrbuf_cstr(ccl_wrbuf),
2208                           &cerror, &cpos);
2209         if (!cn)
2210         {
2211             char *addinfo = odr_strdup_null(odr, ccl_err_msg(cerror));
2212             error = YAZ_BIB1_MALFORMED_QUERY;
2213
2214             switch (cerror)
2215             {
2216             case CCL_ERR_UNKNOWN_QUAL:
2217             case CCL_ERR_TRUNC_NOT_LEFT:
2218             case CCL_ERR_TRUNC_NOT_RIGHT:
2219             case CCL_ERR_TRUNC_NOT_BOTH:
2220 #ifdef CCL_ERR_TRUNC_NOT_EMBED
2221             case CCL_ERR_TRUNC_NOT_EMBED:
2222 #endif
2223 #ifdef CCL_ERR_TRUNC_NOT_SINGLE
2224             case CCL_ERR_TRUNC_NOT_SINGLE:
2225 #endif
2226                 error = YAZ_BIB1_UNSUPP_SEARCH;
2227                 break;
2228             }
2229             log_diagnostic(package, error, addinfo);
2230             apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
2231             package.response() = apdu_res;
2232             return;
2233         }
2234         ccl_pquery(pqf_wrbuf, cn);
2235         package.log("zoom", YLOG_LOG, "RPN: %s", wrbuf_cstr(pqf_wrbuf));
2236         ccl_rpn_delete(cn);
2237     }
2238
2239     assert(pqf_wrbuf.len());
2240
2241     ZOOM_query q = ZOOM_query_create();
2242     ZOOM_query_sortby2(q, b->sptr->sortStrategy.c_str(), sortkeys.c_str());
2243
2244     if (b->get_option("sru"))
2245     {
2246         int status = 0;
2247         Z_RPNQuery *zquery;
2248         zquery = p_query_rpn(odr, wrbuf_cstr(pqf_wrbuf));
2249         mp::wrbuf wrb;
2250
2251         if (!strcmp(b->get_option("sru"), "solr"))
2252         {
2253             solr_transform_t cqlt = solr_transform_create();
2254
2255             status = solr_transform_rpn2solr_wrbuf(cqlt, wrb, zquery);
2256
2257             solr_transform_close(cqlt);
2258         }
2259         else
2260         {
2261             status = cql_transform_rpn2cql_wrbuf(b->cqlt, wrb, zquery);
2262         }
2263         if (status == 0)
2264         {
2265             ZOOM_query_cql(q, wrbuf_cstr(wrb));
2266             package.log("zoom", YLOG_LOG, "CQL: %s", wrbuf_cstr(wrb));
2267             b->search(q, &hits, &error, &addinfo, odr);
2268         }
2269         ZOOM_query_destroy(q);
2270
2271         if (status)
2272         {
2273             error = YAZ_BIB1_MALFORMED_QUERY;
2274             const char *addinfo = "can not convert from RPN to CQL/SOLR";
2275             log_diagnostic(package, error, addinfo);
2276             apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
2277             package.response() = apdu_res;
2278             return;
2279         }
2280     }
2281     else
2282     {
2283         ZOOM_query_prefix(q, wrbuf_cstr(pqf_wrbuf));
2284         package.log("zoom", YLOG_LOG, "search PQF: %s", wrbuf_cstr(pqf_wrbuf));
2285         b->search(q, &hits, &error, &addinfo, odr);
2286         ZOOM_query_destroy(q);
2287     }
2288
2289     if (error)
2290     {
2291         if (retry(package, odr, b, error, &addinfo, proxy_step,
2292                   same_retries, proxy_retries))
2293             goto next_proxy;
2294     }
2295
2296     const char *element_set_name = 0;
2297     Odr_int number_to_present = 0;
2298     if (!error)
2299         mp::util::piggyback_sr(sr, hits, number_to_present, &element_set_name);
2300
2301     Odr_int number_of_records_returned = 0;
2302     Z_Records *records = get_records(
2303         package,
2304         0, number_to_present, &error, &addinfo,
2305         &number_of_records_returned, odr, b, sr->preferredRecordSyntax,
2306         element_set_name);
2307     if (error)
2308         log_diagnostic(package, error, addinfo);
2309     apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
2310     if (records)
2311     {
2312         apdu_res->u.searchResponse->records = records;
2313         apdu_res->u.searchResponse->numberOfRecordsReturned =
2314             odr_intdup(odr, number_of_records_returned);
2315     }
2316     apdu_res->u.searchResponse->resultCount = odr_intdup(odr, hits);
2317     package.response() = apdu_res;
2318 }
2319
2320 void yf::Zoom::Frontend::handle_present(mp::Package &package)
2321 {
2322     Z_GDU *gdu = package.request().get();
2323     Z_APDU *apdu_req = gdu->u.z3950;
2324     Z_APDU *apdu_res = 0;
2325     Z_PresentRequest *pr = apdu_req->u.presentRequest;
2326
2327     mp::odr odr;
2328     if (!m_backend)
2329     {
2330         package.response() = odr.create_presentResponse(
2331             apdu_req, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST, 0);
2332         return;
2333     }
2334     const char *element_set_name = 0;
2335     Z_RecordComposition *comp = pr->recordComposition;
2336     if (comp && comp->which != Z_RecordComp_simple)
2337     {
2338         package.response() = odr.create_presentResponse(
2339             apdu_req,
2340             YAZ_BIB1_PRESENT_COMP_SPEC_PARAMETER_UNSUPP, 0);
2341         return;
2342     }
2343     if (comp && comp->u.simple->which == Z_ElementSetNames_generic)
2344         element_set_name = comp->u.simple->u.generic;
2345     Odr_int number_of_records_returned = 0;
2346     int error = 0;
2347     char *addinfo = 0;
2348
2349     if (m_backend->enable_explain)
2350     {
2351         Z_Records *records =
2352             get_explain_records(
2353                 package,
2354                 *pr->resultSetStartPoint - 1, *pr->numberOfRecordsRequested,
2355                 &error, &addinfo, &number_of_records_returned, odr, m_backend,
2356                 pr->preferredRecordSyntax, element_set_name);
2357
2358         apdu_res = odr.create_presentResponse(apdu_req, error, addinfo);
2359         if (records)
2360         {
2361             apdu_res->u.presentResponse->records = records;
2362             apdu_res->u.presentResponse->numberOfRecordsReturned =
2363                 odr_intdup(odr, number_of_records_returned);
2364         }
2365         package.response() = apdu_res;
2366     }
2367     else
2368     {
2369         Z_Records *records =
2370             get_records(package,
2371                         *pr->resultSetStartPoint - 1, *pr->numberOfRecordsRequested,
2372                         &error, &addinfo, &number_of_records_returned, odr, m_backend,
2373                         pr->preferredRecordSyntax, element_set_name);
2374
2375         apdu_res = odr.create_presentResponse(apdu_req, error, addinfo);
2376         if (records)
2377         {
2378             apdu_res->u.presentResponse->records = records;
2379             apdu_res->u.presentResponse->numberOfRecordsReturned =
2380                 odr_intdup(odr, number_of_records_returned);
2381         }
2382         package.response() = apdu_res;
2383     }
2384 }
2385
2386 void yf::Zoom::Frontend::handle_package(mp::Package &package)
2387 {
2388     Z_GDU *gdu = package.request().get();
2389     if (!gdu)
2390         ;
2391     else if (gdu->which == Z_GDU_Z3950)
2392     {
2393         Z_APDU *apdu_req = gdu->u.z3950;
2394
2395         if (m_backend)
2396             wrbuf_rewind(m_backend->m_apdu_wrbuf);
2397         if (apdu_req->which == Z_APDU_initRequest)
2398         {
2399             mp::odr odr;
2400             package.response() = odr.create_close(
2401                 apdu_req,
2402                 Z_Close_protocolError,
2403                 "double init");
2404         }
2405         else if (apdu_req->which == Z_APDU_searchRequest)
2406         {
2407             handle_search(package);
2408         }
2409         else if (apdu_req->which == Z_APDU_presentRequest)
2410         {
2411             handle_present(package);
2412         }
2413         else
2414         {
2415             mp::odr odr;
2416             package.response() = odr.create_close(
2417                 apdu_req,
2418                 Z_Close_protocolError,
2419                 "zoom filter cannot handle this APDU");
2420             package.session().close();
2421         }
2422         if (m_backend)
2423         {
2424             WRBUF w = m_backend->m_apdu_wrbuf;
2425             package.log_write(wrbuf_buf(w), wrbuf_len(w));
2426         }
2427     }
2428     else
2429     {
2430         package.session().close();
2431     }
2432 }
2433
2434 std::string escape_cql_term(std::string inp)
2435 {
2436     std::string res;
2437     size_t l = inp.length();
2438     size_t i;
2439     for (i = 0; i < l; i++)
2440     {
2441         if (strchr("*?^\"", inp[i]))
2442             res += "\\";
2443         res += inp[i];
2444     }
2445     return res;
2446 }
2447
2448 void yf::Zoom::Frontend::auth(mp::Package &package, Z_InitRequest *req,
2449                               int *error, char **addinfo, ODR odr)
2450 {
2451     if (m_p->torus_auth_url.length() == 0)
2452         return;
2453
2454     std::string user;
2455     std::string password;
2456     if (req->idAuthentication)
2457     {
2458         Z_IdAuthentication *auth = req->idAuthentication;
2459         switch (auth->which)
2460         {
2461         case Z_IdAuthentication_open:
2462             if (auth->u.open)
2463             {
2464                 const char *cp = strchr(auth->u.open, '/');
2465                 if (cp)
2466                 {
2467                     user.assign(auth->u.open, cp - auth->u.open);
2468                     password.assign(cp + 1);
2469                 }
2470             }
2471             break;
2472         case Z_IdAuthentication_idPass:
2473             if (auth->u.idPass->userId)
2474                 user.assign(auth->u.idPass->userId);
2475             if (auth->u.idPass->password)
2476                 password.assign(auth->u.idPass->password);
2477             break;
2478         }
2479     }
2480
2481     std::string ip = package.origin().get_address();
2482     yaz_log(YLOG_LOG, "IP=%s", ip.c_str());
2483
2484     std::string torus_query;
2485     int failure_code;
2486
2487     if (user.length() && password.length())
2488     {
2489         torus_query = "userName==\"" + escape_cql_term(user) +
2490             "\" and password==\"" + escape_cql_term(password) + "\"";
2491         failure_code = YAZ_BIB1_INIT_AC_BAD_USERID_AND_OR_PASSWORD;
2492     }
2493     else
2494     {
2495         const char *ip_cstr = ip.c_str();
2496         const char *cp = strchr(ip_cstr, ':');
2497         if (cp)
2498             ip_cstr = cp + 1;
2499
2500         torus_query = "ip encloses/net.ipaddress \"";
2501         torus_query += escape_cql_term(std::string(ip_cstr));
2502         torus_query += "\"";
2503         failure_code = YAZ_BIB1_INIT_AC_BLOCKED_NETWORK_ADDRESS;
2504     }
2505
2506     std::string dummy_db;
2507     std::string dummy_realm;
2508     xmlDoc *doc = mp::get_searchable(package, m_p->torus_auth_url, dummy_db,
2509                                      torus_query, dummy_realm, m_p->proxy);
2510     if (!doc)
2511     {
2512         // something fundamental broken in lookup.
2513         *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
2514         *addinfo = odr_strdup(odr, "Torus server unavailable or "
2515                               "incorrectly configured");
2516         return;
2517     }
2518     const xmlNode *ptr = xmlDocGetRootElement(doc);
2519     if (ptr && ptr->type == XML_ELEMENT_NODE)
2520     {
2521         if (strcmp((const char *) ptr->name, "records") == 0)
2522         {
2523             ptr = ptr->children;
2524             while (ptr && ptr->type != XML_ELEMENT_NODE)
2525                 ptr = ptr->next;
2526         }
2527         if (ptr && strcmp((const char *) ptr->name, "record") == 0)
2528         {
2529             ptr = ptr->children;
2530             while (ptr && ptr->type != XML_ELEMENT_NODE)
2531                 ptr = ptr->next;
2532         }
2533         if (ptr && strcmp((const char *) ptr->name, "layer") == 0)
2534         {
2535             ptr = ptr->children;
2536             while (ptr && ptr->type != XML_ELEMENT_NODE)
2537                 ptr = ptr->next;
2538         }
2539         while (ptr)
2540         {
2541             if (ptr && ptr->type == XML_ELEMENT_NODE &&
2542                 !strcmp((const char *) ptr->name, "identityId"))
2543                 break;
2544             ptr = ptr->next;
2545         }
2546     }
2547     if (!ptr)
2548     {
2549         *error = failure_code;
2550         return;
2551     }
2552     session_realm = mp::xml::get_text(ptr);
2553 }
2554
2555 void yf::Zoom::Impl::process(mp::Package &package)
2556 {
2557     FrontendPtr f = get_frontend(package);
2558     Z_GDU *gdu = package.request().get();
2559
2560     if (f->m_is_virtual)
2561     {
2562         f->handle_package(package);
2563     }
2564     else if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
2565              Z_APDU_initRequest)
2566     {
2567         Z_InitRequest *req = gdu->u.z3950->u.initRequest;
2568         f->m_init_gdu = gdu;
2569
2570         mp::odr odr;
2571         Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
2572         Z_InitResponse *resp = apdu->u.initResponse;
2573
2574         int i;
2575         static const int masks[] = {
2576             Z_Options_search,
2577             Z_Options_present,
2578             -1
2579         };
2580         for (i = 0; masks[i] != -1; i++)
2581             if (ODR_MASK_GET(req->options, masks[i]))
2582                 ODR_MASK_SET(resp->options, masks[i]);
2583
2584         static const int versions[] = {
2585             Z_ProtocolVersion_1,
2586             Z_ProtocolVersion_2,
2587             Z_ProtocolVersion_3,
2588             -1
2589         };
2590         for (i = 0; versions[i] != -1; i++)
2591             if (ODR_MASK_GET(req->protocolVersion, versions[i]))
2592                 ODR_MASK_SET(resp->protocolVersion, versions[i]);
2593             else
2594                 break;
2595
2596         *resp->preferredMessageSize = *req->preferredMessageSize;
2597         *resp->maximumRecordSize = *req->maximumRecordSize;
2598
2599         int error = 0;
2600         char *addinfo = 0;
2601         f->auth(package, req, &error, &addinfo, odr);
2602         if (error)
2603         {
2604             resp->userInformationField =
2605                 zget_init_diagnostics(odr, error, addinfo);
2606             *resp->result = 0;
2607             package.session().close();
2608         }
2609         else
2610             f->m_is_virtual = true;
2611         package.response() = apdu;
2612     }
2613     else
2614         package.move();
2615
2616     release_frontend(package);
2617 }
2618
2619
2620 static mp::filter::Base* filter_creator()
2621 {
2622     return new mp::filter::Zoom;
2623 }
2624
2625 extern "C" {
2626     struct metaproxy_1_filter_struct metaproxy_1_filter_zoom = {
2627         0,
2628         "zoom",
2629         filter_creator
2630     };
2631 }
2632
2633
2634 /*
2635  * Local variables:
2636  * c-basic-offset: 4
2637  * c-file-style: "Stroustrup"
2638  * indent-tabs-mode: nil
2639  * End:
2640  * vim: shiftwidth=4 tabstop=8 expandtab
2641  */
2642