src/filter_zoom.cpp

   1 /* This file is part of Metaproxy.
   2    Copyright (C) 2005-2011 Index Data
   3
   4 Metaproxy is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17 */
  18
  19 #include "config.hpp"
  20 #include "filter_zoom.hpp"
  21 #include <yaz/zoom.h>
  22 #include <yaz/yaz-version.h>
  23 #include <yaz/srw.h>
  24 #include <metaproxy/package.hpp>
  25 #include <metaproxy/util.hpp>
  26 #include "torus.hpp"
  27
  28 #include <libxslt/xsltutils.h>
  29 #include <libxslt/transform.h>
  30
  31 #include <boost/thread/mutex.hpp>
  32 #include <boost/thread/condition.hpp>
  33 #include <yaz/ccl_xml.h>
  34 #include <yaz/ccl.h>
  35 #include <yaz/rpn2cql.h>
  36 #include <yaz/rpn2solr.h>
  37 #include <yaz/pquery.h>
  38 #include <yaz/cql.h>
  39 #include <yaz/oid_db.h>
  40 #include <yaz/diagbib1.h>
  41 #include <yaz/log.h>
  42 #include <yaz/zgdu.h>
  43 #include <yaz/querytowrbuf.h>
  44
  45 namespace mp = metaproxy_1;
  46 namespace yf = mp::filter;
  47
  48 namespace metaproxy_1 {
  49     namespace filter {
  50         struct Zoom::Searchable : boost::noncopyable {
  51             std::string authentication;
  52             std::string cfAuth;
  53             std::string cfProxy;
  54             std::string cfSubDb;
  55             std::string udb;
  56             std::string target;
  57             std::string query_encoding;
  58             std::string sru;
  59             std::string request_syntax;
  60             std::string element_set;
  61             std::string record_encoding;
  62             std::string transform_xsl_fname;
  63             bool use_turbomarc;
  64             bool piggyback;
  65             CCL_bibset ccl_bibset;
  66             Searchable(CCL_bibset base);
  67             ~Searchable();
  68         };
  69         class Zoom::Backend : boost::noncopyable {
  70             friend class Impl;
  71             friend class Frontend;
  72             std::string zurl;
  73             ZOOM_connection m_connection;
  74             ZOOM_resultset m_resultset;
  75             std::string m_frontend_database;
  76             SearchablePtr sptr;
  77             xsltStylesheetPtr xsp;
  78         public:
  79             Backend(SearchablePtr sptr);
  80             ~Backend();
  81             void connect(std::string zurl, int *error, const char **addinfo);
  82             void search_pqf(const char *pqf, Odr_int *hits,
  83                             int *error, const char **addinfo);
  84             void search_cql(const char *cql, Odr_int *hits,
  85                             int *error, const char **addinfo);
  86             void present(Odr_int start, Odr_int number, ZOOM_record *recs,
  87                          int *error, const char **addinfo);
  88             void set_option(const char *name, const char *value);
  89             const char *get_option(const char *name);
  90             void get_zoom_error(int *error, const char **addinfo);
  91         };
  92         class Zoom::Frontend : boost::noncopyable {
  93             friend class Impl;
  94             Impl *m_p;
  95             bool m_is_virtual;
  96             bool m_in_use;
  97             yazpp_1::GDU m_init_gdu;
  98             BackendPtr m_backend;
  99             void handle_package(mp::Package &package);
 100             void handle_search(mp::Package &package);
 101             void handle_present(mp::Package &package);
 102             BackendPtr get_backend_from_databases(std::string &database,
 103                                                   int *error,
 104                                                   const char **addinfo);
 105             Z_Records *get_records(Odr_int start,
 106                                    Odr_int number_to_present,
 107                                    int *error,
 108                                    const char **addinfo,
 109                                    Odr_int *number_of_records_returned,
 110                                    ODR odr, BackendPtr b,
 111                                    Odr_oid *preferredRecordSyntax,
 112                                    const char *element_set_name);
 113         public:
 114             Frontend(Impl *impl);
 115             ~Frontend();
 116         };
 117         class Zoom::Impl {
 118             friend class Frontend;
 119         public:
 120             Impl();
 121             ~Impl();
 122             void process(metaproxy_1::Package & package);
 123             void configure(const xmlNode * ptr, bool test_only);
 124         private:
 125             void configure_local_records(const xmlNode * ptr, bool test_only);
 126             FrontendPtr get_frontend(mp::Package &package);
 127             void release_frontend(mp::Package &package);
 128             SearchablePtr parse_torus_record(const xmlNode *ptr);
 129             struct cql_node *convert_cql_fields(struct cql_node *cn, ODR odr);
 130             std::map<mp::Session, FrontendPtr> m_clients;
 131             boost::mutex m_mutex;
 132             boost::condition m_cond_session_ready;
 133             std::string torus_url;
 134             std::map<std::string,std::string> fieldmap;
 135             std::string xsldir;
 136             CCL_bibset bibset;
 137             std::map<std::string,SearchablePtr> s_map;
 138         };
 139     }
 140 }
 141
 142 // define Pimpl wrapper forwarding to Impl
 143
 144 yf::Zoom::Zoom() : m_p(new Impl)
 145 {
 146 }
 147
 148 yf::Zoom::~Zoom()
 149 {  // must have a destructor because of boost::scoped_ptr
 150 }
 151
 152 void yf::Zoom::configure(const xmlNode *xmlnode, bool test_only)
 153 {
 154     m_p->configure(xmlnode, test_only);
 155 }
 156
 157 void yf::Zoom::process(mp::Package &package) const
 158 {
 159     m_p->process(package);
 160 }
 161
 162
 163 // define Implementation stuff
 164
 165 yf::Zoom::Backend::Backend(SearchablePtr ptr) : sptr(ptr)
 166 {
 167     m_connection = ZOOM_connection_create(0);
 168     m_resultset = 0;
 169     xsp = 0;
 170 }
 171
 172 yf::Zoom::Backend::~Backend()
 173 {
 174     if (xsp)
 175         xsltFreeStylesheet(xsp);
 176     ZOOM_connection_destroy(m_connection);
 177     ZOOM_resultset_destroy(m_resultset);
 178 }
 179
 180
 181 void yf::Zoom::Backend::get_zoom_error(int *error, const char **addinfo)
 182 {
 183     const char *msg = 0;
 184     *error = ZOOM_connection_error(m_connection, &msg, addinfo);
 185     if (*error)
 186     {
 187         if (*error >= ZOOM_ERROR_CONNECT)
 188         {
 189             // turn ZOOM diagnostic into a Bib-1 2: with addinfo=zoom err msg
 190             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
 191             if (addinfo)
 192                 *addinfo = msg;
 193         }
 194     }
 195 }
 196
 197 void yf::Zoom::Backend::connect(std::string zurl,
 198                                 int *error, const char **addinfo)
 199 {
 200     ZOOM_connection_connect(m_connection, zurl.c_str(), 0);
 201     get_zoom_error(error, addinfo);
 202 }
 203
 204 void yf::Zoom::Backend::search_pqf(const char *pqf, Odr_int *hits,
 205                                    int *error, const char **addinfo)
 206 {
 207     m_resultset = ZOOM_connection_search_pqf(m_connection, pqf);
 208     get_zoom_error(error, addinfo);
 209     if (*error == 0)
 210         *hits = ZOOM_resultset_size(m_resultset);
 211     else
 212         *hits = 0;
 213 }
 214
 215 void yf::Zoom::Backend::search_cql(const char *cql, Odr_int *hits,
 216                                    int *error, const char **addinfo)
 217 {
 218     ZOOM_query q = ZOOM_query_create();
 219
 220     ZOOM_query_cql(q, cql);
 221
 222     m_resultset = ZOOM_connection_search(m_connection, q);
 223     ZOOM_query_destroy(q);
 224     get_zoom_error(error, addinfo);
 225     if (*error == 0)
 226         *hits = ZOOM_resultset_size(m_resultset);
 227     else
 228         *hits = 0;
 229 }
 230
 231 void yf::Zoom::Backend::present(Odr_int start, Odr_int number,
 232                                 ZOOM_record *recs,
 233                                 int *error, const char **addinfo)
 234 {
 235     ZOOM_resultset_records(m_resultset, recs, start, number);
 236     get_zoom_error(error, addinfo);
 237 }
 238
 239 void yf::Zoom::Backend::set_option(const char *name, const char *value)
 240 {
 241     ZOOM_connection_option_set(m_connection, name, value);
 242     if (m_resultset)
 243         ZOOM_resultset_option_set(m_resultset, name, value);
 244 }
 245
 246 const char *yf::Zoom::Backend::get_option(const char *name)
 247 {
 248     return ZOOM_connection_option_get(m_connection, name);
 249 }
 250
 251 yf::Zoom::Searchable::Searchable(CCL_bibset base)
 252 {
 253     piggyback = true;
 254     use_turbomarc = true;
 255     ccl_bibset = ccl_qual_dup(base);
 256 }
 257
 258 yf::Zoom::Searchable::~Searchable()
 259 {
 260     ccl_qual_rm(&ccl_bibset);
 261 }
 262
 263 yf::Zoom::Frontend::Frontend(Impl *impl) :
 264     m_p(impl), m_is_virtual(false), m_in_use(true)
 265 {
 266 }
 267
 268 yf::Zoom::Frontend::~Frontend()
 269 {
 270 }
 271
 272 yf::Zoom::FrontendPtr yf::Zoom::Impl::get_frontend(mp::Package &package)
 273 {
 274     boost::mutex::scoped_lock lock(m_mutex);
 275
 276     std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
 277
 278     while(true)
 279     {
 280         it = m_clients.find(package.session());
 281         if (it == m_clients.end())
 282             break;
 283
 284         if (!it->second->m_in_use)
 285         {
 286             it->second->m_in_use = true;
 287             return it->second;
 288         }
 289         m_cond_session_ready.wait(lock);
 290     }
 291     FrontendPtr f(new Frontend(this));
 292     m_clients[package.session()] = f;
 293     f->m_in_use = true;
 294     return f;
 295 }
 296
 297 void yf::Zoom::Impl::release_frontend(mp::Package &package)
 298 {
 299     boost::mutex::scoped_lock lock(m_mutex);
 300     std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
 301
 302     it = m_clients.find(package.session());
 303     if (it != m_clients.end())
 304     {
 305         if (package.session().is_closed())
 306         {
 307             m_clients.erase(it);
 308         }
 309         else
 310         {
 311             it->second->m_in_use = false;
 312         }
 313         m_cond_session_ready.notify_all();
 314     }
 315 }
 316
 317 yf::Zoom::Impl::Impl()
 318 {
 319     bibset = ccl_qual_mk();
 320 }
 321
 322 yf::Zoom::Impl::~Impl()
 323 {
 324     ccl_qual_rm(&bibset);
 325 }
 326
 327 yf::Zoom::SearchablePtr yf::Zoom::Impl::parse_torus_record(const xmlNode *ptr)
 328 {
 329     Zoom::SearchablePtr s(new Searchable(bibset));
 330
 331     for (ptr = ptr->children; ptr; ptr = ptr->next)
 332     {
 333         if (ptr->type != XML_ELEMENT_NODE)
 334             continue;
 335         if (!strcmp((const char *) ptr->name, "layer"))
 336             ptr = ptr->children;
 337         else if (!strcmp((const char *) ptr->name,
 338                          "authentication"))
 339         {
 340             s->authentication = mp::xml::get_text(ptr);
 341         }
 342         else if (!strcmp((const char *) ptr->name,
 343                          "cfAuth"))
 344         {
 345             s->cfAuth = mp::xml::get_text(ptr);
 346         }
 347         else if (!strcmp((const char *) ptr->name,
 348                          "cfProxy"))
 349         {
 350             s->cfProxy = mp::xml::get_text(ptr);
 351         }
 352         else if (!strcmp((const char *) ptr->name,
 353                          "cfSubDb"))
 354         {
 355             s->cfSubDb = mp::xml::get_text(ptr);
 356         }
 357         else if (!strcmp((const char *) ptr->name, "udb"))
 358         {
 359             s->udb = mp::xml::get_text(ptr);
 360         }
 361         else if (!strcmp((const char *) ptr->name, "zurl"))
 362         {
 363             s->target = mp::xml::get_text(ptr);
 364         }
 365         else if (!strcmp((const char *) ptr->name, "sru"))
 366         {
 367             s->sru = mp::xml::get_text(ptr);
 368         }
 369         else if (!strcmp((const char *) ptr->name,
 370                          "queryEncoding"))
 371         {
 372             s->query_encoding = mp::xml::get_text(ptr);
 373         }
 374         else if (!strcmp((const char *) ptr->name,
 375                          "piggyback"))
 376         {
 377             s->piggyback = mp::xml::get_bool(ptr, true);
 378         }
 379         else if (!strcmp((const char *) ptr->name,
 380                          "requestSyntax"))
 381         {
 382             s->request_syntax = mp::xml::get_text(ptr);
 383         }
 384         else if (!strcmp((const char *) ptr->name,
 385                          "elementSet"))
 386         {
 387             s->element_set = mp::xml::get_text(ptr);
 388         }
 389         else if (!strcmp((const char *) ptr->name,
 390                          "recordEncoding"))
 391         {
 392             s->record_encoding = mp::xml::get_text(ptr);
 393         }
 394         else if (!strcmp((const char *) ptr->name,
 395                          "transform"))
 396         {
 397             s->transform_xsl_fname = mp::xml::get_text(ptr);
 398         }
 399         else if (!strcmp((const char *) ptr->name,
 400                          "useTurboMarc"))
 401         {
 402             ; // useTurboMarc is ignored
 403         }
 404         else if (!strncmp((const char *) ptr->name,
 405                           "cclmap_", 7))
 406         {
 407             std::string value = mp::xml::get_text(ptr);
 408             ccl_qual_fitem(s->ccl_bibset, value.c_str(),
 409                            (const char *) ptr->name + 7);
 410         }
 411     }
 412     return s;
 413 }
 414
 415 void yf::Zoom::Impl::configure_local_records(const xmlNode *ptr, bool test_only)
 416 {
 417     while (ptr && ptr->type != XML_ELEMENT_NODE)
 418         ptr = ptr->next;
 419
 420     if (ptr)
 421     {
 422         if (!strcmp((const char *) ptr->name, "records"))
 423         {
 424             for (ptr = ptr->children; ptr; ptr = ptr->next)
 425             {
 426                 if (ptr->type != XML_ELEMENT_NODE)
 427                     continue;
 428                 if (!strcmp((const char *) ptr->name, "record"))
 429                 {
 430                     SearchablePtr s = parse_torus_record(ptr);
 431                     if (s)
 432                     {
 433                         std::string udb = s->udb;
 434                         if (udb.length())
 435                             s_map[s->udb] = s;
 436                         else
 437                         {
 438                             throw mp::filter::FilterException
 439                                 ("No udb for local torus record");
 440                         }
 441                     }
 442                 }
 443                 else
 444                 {
 445                     throw mp::filter::FilterException
 446                         ("Bad element "
 447                          + std::string((const char *) ptr->name)
 448                          + " in zoom filter inside element "
 449                          "<torus><records>");
 450                 }
 451             }
 452         }
 453         else
 454         {
 455             throw mp::filter::FilterException
 456                 ("Bad element "
 457                  + std::string((const char *) ptr->name)
 458                  + " in zoom filter inside element <torus>");
 459         }
 460     }
 461 }
 462
 463 void yf::Zoom::Impl::configure(const xmlNode *ptr, bool test_only)
 464 {
 465     for (ptr = ptr->children; ptr; ptr = ptr->next)
 466     {
 467         if (ptr->type != XML_ELEMENT_NODE)
 468             continue;
 469         else if (!strcmp((const char *) ptr->name, "torus"))
 470         {
 471             const struct _xmlAttr *attr;
 472             for (attr = ptr->properties; attr; attr = attr->next)
 473             {
 474                 if (!strcmp((const char *) attr->name, "url"))
 475                     torus_url = mp::xml::get_text(attr->children);
 476                 else if (!strcmp((const char *) attr->name, "xsldir"))
 477                     xsldir = mp::xml::get_text(attr->children);
 478                 else
 479                     throw mp::filter::FilterException(
 480                         "Bad attribute " + std::string((const char *)
 481                                                        attr->name));
 482             }
 483             configure_local_records(ptr->children, test_only);
 484         }
 485         else if (!strcmp((const char *) ptr->name, "cclmap"))
 486         {
 487             const char *addinfo = 0;
 488             ccl_xml_config(bibset, ptr, &addinfo);
 489         }
 490         else if (!strcmp((const char *) ptr->name, "fieldmap"))
 491         {
 492             const struct _xmlAttr *attr;
 493             std::string ccl_field;
 494             std::string cql_field;
 495             for (attr = ptr->properties; attr; attr = attr->next)
 496             {
 497                 if (!strcmp((const char *) attr->name, "ccl"))
 498                     ccl_field = mp::xml::get_text(attr->children);
 499                 else if (!strcmp((const char *) attr->name, "cql"))
 500                     cql_field = mp::xml::get_text(attr->children);
 501                 else
 502                     throw mp::filter::FilterException(
 503                         "Bad attribute " + std::string((const char *)
 504                                                        attr->name));
 505             }
 506             if (cql_field.length())
 507                 fieldmap[cql_field] = ccl_field;
 508         }
 509         else
 510         {
 511             throw mp::filter::FilterException
 512                 ("Bad element "
 513                  + std::string((const char *) ptr->name)
 514                  + " in zoom filter");
 515         }
 516     }
 517 }
 518
 519 yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
 520     std::string &database, int *error, const char **addinfo)
 521 {
 522     std::list<BackendPtr>::const_iterator map_it;
 523     if (m_backend && m_backend->m_frontend_database == database)
 524         return m_backend;
 525
 526     std::string db_args;
 527     std::string cf_parm;
 528     std::string torus_db;
 529     size_t db_arg_pos = database.find(',');
 530     if (db_arg_pos != std::string::npos)
 531     {
 532         torus_db = database.substr(0, db_arg_pos);
 533         db_args = database.substr(db_arg_pos+1);
 534     }
 535     else
 536         torus_db = database;
 537
 538     SearchablePtr sptr;
 539
 540     std::map<std::string,SearchablePtr>::iterator it;
 541     it = m_p->s_map.find(torus_db);
 542     if (it != m_p->s_map.end())
 543         sptr = it->second;
 544     else
 545     {
 546         xmlDoc *doc = mp::get_searchable(m_p->torus_url, torus_db);
 547         if (!doc)
 548         {
 549             *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
 550             *addinfo = database.c_str();
 551             BackendPtr b;
 552             return b;
 553         }
 554         const xmlNode *ptr = xmlDocGetRootElement(doc);
 555         if (ptr)
 556         {   // presumably ptr is a records element node
 557             // parse first record in document
 558             for (ptr = ptr->children; ptr; ptr = ptr->next)
 559             {
 560                 if (ptr->type == XML_ELEMENT_NODE
 561                     && !strcmp((const char *) ptr->name, "record"))
 562                 {
 563                     sptr = m_p->parse_torus_record(ptr);
 564                     break;
 565                 }
 566             }
 567         }
 568         xmlFreeDoc(doc);
 569     }
 570
 571     if (!sptr)
 572     {
 573         *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
 574         *addinfo = database.c_str();
 575         BackendPtr b;
 576         return b;
 577     }
 578
 579     xsltStylesheetPtr xsp = 0;
 580     if (sptr->transform_xsl_fname.length())
 581     {
 582         std::string fname;
 583
 584         if (m_p->xsldir.length())
 585             fname = m_p->xsldir + "/" + sptr->transform_xsl_fname;
 586         else
 587             fname = sptr->transform_xsl_fname;
 588         xmlDoc *xsp_doc = xmlParseFile(fname.c_str());
 589         if (!xsp_doc)
 590         {
 591             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
 592             *addinfo = "xmlParseFile failed";
 593             BackendPtr b;
 594             return b;
 595         }
 596         xsp = xsltParseStylesheetDoc(xsp_doc);
 597         if (!xsp)
 598         {
 599             *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
 600             *addinfo = "xsltParseStylesheetDoc failed";
 601             BackendPtr b;
 602             xmlFreeDoc(xsp_doc);
 603             return b;
 604         }
 605     }
 606
 607     m_backend.reset();
 608
 609     BackendPtr b(new Backend(sptr));
 610
 611     b->xsp = xsp;
 612     b->m_frontend_database = database;
 613     std::string authentication = sptr->authentication;
 614
 615     b->set_option("timeout", "40");
 616
 617     if (sptr->query_encoding.length())
 618         b->set_option("rpnCharset", sptr->query_encoding.c_str());
 619
 620     if (sptr->cfAuth.length())
 621     {
 622         b->set_option("user", sptr->cfAuth.c_str());
 623         if (authentication.length())
 624         {
 625             size_t found = authentication.find('/');
 626             if (found != std::string::npos)
 627             {
 628                 cf_parm += "user=" + mp::util::uri_encode(authentication.substr(0, found))
 629                     + "&password=" + mp::util::uri_encode(authentication.substr(found+1));
 630             }
 631             else
 632                 cf_parm += "user=" + mp::util::uri_encode(authentication);
 633         }
 634     }
 635     else if (authentication.length())
 636         b->set_option("user", authentication.c_str());
 637
 638     if (sptr->cfProxy.length())
 639     {
 640         if (cf_parm.length())
 641             cf_parm += "&";
 642         cf_parm += "proxy=" + mp::util::uri_encode(sptr->cfProxy);
 643     }
 644     if (sptr->cfSubDb.length())
 645     {
 646         if (cf_parm.length())
 647             cf_parm += "&";
 648         cf_parm += "subdatabase=" + mp::util::uri_encode(sptr->cfSubDb);
 649     }
 650
 651     std::string url;
 652     if (sptr->sru.length())
 653     {
 654         url = "http://" + sptr->target;
 655         b->set_option("sru", sptr->sru.c_str());
 656     }
 657     else
 658     {
 659         url = sptr->target;
 660     }
 661     if (db_args.length())
 662         url += "," + db_args;
 663     else if (cf_parm.length())
 664         url += "," + cf_parm;
 665     yaz_log(YLOG_LOG, "url=%s", url.c_str());
 666     b->connect(url, error, addinfo);
 667     if (*error == 0)
 668     {
 669         m_backend = b;
 670     }
 671     return b;
 672 }
 673
 674 Z_Records *yf::Zoom::Frontend::get_records(Odr_int start,
 675                                            Odr_int number_to_present,
 676                                            int *error,
 677                                            const char **addinfo,
 678                                            Odr_int *number_of_records_returned,
 679                                            ODR odr,
 680                                            BackendPtr b,
 681                                            Odr_oid *preferredRecordSyntax,
 682                                            const char *element_set_name)
 683 {
 684     *number_of_records_returned = 0;
 685     Z_Records *records = 0;
 686     bool enable_pz2_transform = false;
 687
 688     if (start < 0 || number_to_present <= 0)
 689         return records;
 690
 691     if (number_to_present > 10000)
 692         number_to_present = 10000;
 693
 694     ZOOM_record *recs = (ZOOM_record *)
 695         odr_malloc(odr, number_to_present * sizeof(*recs));
 696
 697     char oid_name_str[OID_STR_MAX];
 698     const char *syntax_name = 0;
 699
 700     if (preferredRecordSyntax)
 701     {
 702         if (!oid_oidcmp(preferredRecordSyntax, yaz_oid_recsyn_xml)
 703             && element_set_name &&
 704             !strcmp(element_set_name, "pz2"))
 705         {
 706             if (b->sptr->request_syntax.length())
 707                 syntax_name = b->sptr->request_syntax.c_str();
 708             enable_pz2_transform = true;
 709         }
 710         else
 711         {
 712             syntax_name =
 713                 yaz_oid_to_string_buf(preferredRecordSyntax, 0, oid_name_str);
 714         }
 715     }
 716
 717     b->set_option("preferredRecordSyntax", syntax_name);
 718
 719     if (enable_pz2_transform)
 720     {
 721         element_set_name = "F";
 722         if (b->sptr->element_set.length())
 723             element_set_name = b->sptr->element_set.c_str();
 724     }
 725
 726     b->set_option("elementSetName", element_set_name);
 727
 728     b->present(start, number_to_present, recs, error, addinfo);
 729
 730     Odr_int i = 0;
 731     if (!*error)
 732     {
 733         for (i = 0; i < number_to_present; i++)
 734             if (!recs[i])
 735                 break;
 736     }
 737     if (i > 0)
 738     {  // only return records if no error and at least one record
 739         char *odr_database = odr_strdup(odr,
 740                                         b->m_frontend_database.c_str());
 741         Z_NamePlusRecordList *npl = (Z_NamePlusRecordList *)
 742             odr_malloc(odr, sizeof(*npl));
 743         *number_of_records_returned = i;
 744         npl->num_records = i;
 745         npl->records = (Z_NamePlusRecord **)
 746             odr_malloc(odr, i * sizeof(*npl->records));
 747         for (i = 0; i < number_to_present; i++)
 748         {
 749             Z_NamePlusRecord *npr = 0;
 750             const char *addinfo;
 751             int sur_error = ZOOM_record_error(recs[i], 0 /* msg */,
 752                                               &addinfo, 0 /* diagset */);
 753
 754             if (sur_error)
 755             {
 756                 npr = zget_surrogateDiagRec(odr, odr_database, sur_error,
 757                                             addinfo);
 758             }
 759             else if (enable_pz2_transform)
 760             {
 761                 char rec_type_str[100];
 762
 763                 strcpy(rec_type_str, b->sptr->use_turbomarc ?
 764                        "txml" : "xml");
 765                 // prevent buffer overflow ...
 766                 if (b->sptr->record_encoding.length() > 0 &&
 767                     b->sptr->record_encoding.length() <
 768                     (sizeof(rec_type_str)-20))
 769                 {
 770                     strcat(rec_type_str, "; charset=");
 771                     strcat(rec_type_str, b->sptr->record_encoding.c_str());
 772                 }
 773
 774                 int rec_len;
 775                 const char *rec_buf = ZOOM_record_get(recs[i], rec_type_str,
 776                                                       &rec_len);
 777                 if (rec_buf && b->xsp)
 778                 {
 779                     xmlDoc *rec_doc = xmlParseMemory(rec_buf, rec_len);
 780                     if (rec_doc)
 781                     {
 782                         xmlDoc *rec_res;
 783                         rec_res = xsltApplyStylesheet(b->xsp, rec_doc, 0);
 784
 785                         if (rec_res)
 786                             xsltSaveResultToString((xmlChar **) &rec_buf, &rec_len,
 787                                                    rec_res, b->xsp);
 788                     }
 789                 }
 790
 791                 if (rec_buf)
 792                 {
 793                     npr = (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
 794                     npr->databaseName = odr_database;
 795                     npr->which = Z_NamePlusRecord_databaseRecord;
 796                     npr->u.databaseRecord =
 797                         z_ext_record_xml(odr, rec_buf, rec_len);
 798                 }
 799                 else
 800                 {
 801                     npr = zget_surrogateDiagRec(
 802                         odr, odr_database,
 803                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
 804                         rec_type_str);
 805                 }
 806             }
 807             else
 808             {
 809                 Z_External *ext =
 810                     (Z_External *) ZOOM_record_get(recs[i], "ext", 0);
 811                 if (ext)
 812                 {
 813                     npr = (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
 814                     npr->databaseName = odr_database;
 815                     npr->which = Z_NamePlusRecord_databaseRecord;
 816                     npr->u.databaseRecord = ext;
 817                 }
 818                 else
 819                 {
 820                     npr = zget_surrogateDiagRec(
 821                         odr, odr_database,
 822                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
 823                         "ZOOM_record, type ext");
 824                 }
 825             }
 826             npl->records[i] = npr;
 827         }
 828         records = (Z_Records*) odr_malloc(odr, sizeof(*records));
 829         records->which = Z_Records_DBOSD;
 830         records->u.databaseOrSurDiagnostics = npl;
 831     }
 832     return records;
 833 }
 834
 835 struct cql_node *yf::Zoom::Impl::convert_cql_fields(struct cql_node *cn,
 836                                                     ODR odr)
 837 {
 838     struct cql_node *r = 0;
 839     if (!cn)
 840         return 0;
 841     switch (cn->which)
 842     {
 843     case CQL_NODE_ST:
 844         if (cn->u.st.index)
 845         {
 846             std::map<std::string,std::string>::const_iterator it;
 847             it = fieldmap.find(cn->u.st.index);
 848             if (it == fieldmap.end())
 849                 return cn;
 850             if (it->second.length())
 851                 cn->u.st.index = odr_strdup(odr, it->second.c_str());
 852             else
 853                 cn->u.st.index = 0;
 854         }
 855         break;
 856     case CQL_NODE_BOOL:
 857         r = convert_cql_fields(cn->u.boolean.left, odr);
 858         if (!r)
 859             r = convert_cql_fields(cn->u.boolean.right, odr);
 860         break;
 861     case CQL_NODE_SORT:
 862         r = convert_cql_fields(cn->u.sort.search, odr);
 863         break;
 864     }
 865     return r;
 866 }
 867
 868 void yf::Zoom::Frontend::handle_search(mp::Package &package)
 869 {
 870     Z_GDU *gdu = package.request().get();
 871     Z_APDU *apdu_req = gdu->u.z3950;
 872     Z_APDU *apdu_res = 0;
 873     mp::odr odr;
 874     Z_SearchRequest *sr = apdu_req->u.searchRequest;
 875     if (sr->num_databaseNames != 1)
 876     {
 877         apdu_res = odr.create_searchResponse(
 878             apdu_req, YAZ_BIB1_TOO_MANY_DATABASES_SPECIFIED, 0);
 879         package.response() = apdu_res;
 880         return;
 881     }
 882
 883     int error = 0;
 884     const char *addinfo = 0;
 885     std::string db(sr->databaseNames[0]);
 886     BackendPtr b = get_backend_from_databases(db, &error, &addinfo);
 887     if (error)
 888     {
 889         apdu_res =
 890             odr.create_searchResponse(
 891                 apdu_req, error, addinfo);
 892         package.response() = apdu_res;
 893         return;
 894     }
 895
 896     b->set_option("setname", "default");
 897
 898     Odr_int hits = 0;
 899     Z_Query *query = sr->query;
 900     WRBUF ccl_wrbuf = 0;
 901     WRBUF pqf_wrbuf = 0;
 902
 903     if (query->which == Z_Query_type_1 || query->which == Z_Query_type_101)
 904     {
 905         // RPN
 906         pqf_wrbuf = wrbuf_alloc();
 907         yaz_rpnquery_to_wrbuf(pqf_wrbuf, query->u.type_1);
 908     }
 909     else if (query->which == Z_Query_type_2)
 910     {
 911         // CCL
 912         ccl_wrbuf = wrbuf_alloc();
 913         wrbuf_write(ccl_wrbuf, (const char *) query->u.type_2->buf,
 914                     query->u.type_2->len);
 915     }
 916     else if (query->which == Z_Query_type_104 &&
 917              query->u.type_104->which == Z_External_CQL)
 918     {
 919         // CQL
 920         const char *cql = query->u.type_104->u.cql;
 921         CQL_parser cp = cql_parser_create();
 922         int r = cql_parser_string(cp, cql);
 923         if (r)
 924         {
 925             cql_parser_destroy(cp);
 926             apdu_res =
 927                 odr.create_searchResponse(apdu_req,
 928                                           YAZ_BIB1_MALFORMED_QUERY,
 929                                           "CQL syntax error");
 930             package.response() = apdu_res;
 931             return;
 932         }
 933         struct cql_node *cn = cql_parser_result(cp);
 934         struct cql_node *cn_error = m_p->convert_cql_fields(cn, odr);
 935         if (cn_error)
 936         {
 937             // hopefully we are getting a ptr to a index+relation+term node
 938             addinfo = 0;
 939             if (cn_error->which == CQL_NODE_ST)
 940                 addinfo = cn_error->u.st.index;
 941
 942             apdu_res =
 943                 odr.create_searchResponse(apdu_req,
 944                                           YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
 945                                           addinfo);
 946             package.response() = apdu_res;
 947             return;
 948         }
 949         char ccl_buf[1024];
 950
 951         r = cql_to_ccl_buf(cn, ccl_buf, sizeof(ccl_buf));
 952         if (r == 0)
 953         {
 954             ccl_wrbuf = wrbuf_alloc();
 955             wrbuf_puts(ccl_wrbuf, ccl_buf);
 956         }
 957         cql_parser_destroy(cp);
 958         if (r)
 959         {
 960             apdu_res =
 961                 odr.create_searchResponse(apdu_req,
 962                                           YAZ_BIB1_MALFORMED_QUERY,
 963                                           "CQL to CCL conversion error");
 964             package.response() = apdu_res;
 965             return;
 966         }
 967     }
 968     else
 969     {
 970         apdu_res =
 971             odr.create_searchResponse(apdu_req, YAZ_BIB1_QUERY_TYPE_UNSUPP, 0);
 972         package.response() = apdu_res;
 973         return;
 974     }
 975
 976     if (ccl_wrbuf)
 977     {
 978         // CCL to PQF
 979         assert(pqf_wrbuf == 0);
 980         int cerror, cpos;
 981         struct ccl_rpn_node *cn;
 982         yaz_log(YLOG_LOG, "CCL: %s", wrbuf_cstr(ccl_wrbuf));
 983         cn = ccl_find_str(b->sptr->ccl_bibset, wrbuf_cstr(ccl_wrbuf),
 984                           &cerror, &cpos);
 985         wrbuf_destroy(ccl_wrbuf);
 986         if (!cn)
 987         {
 988             char *addinfo = odr_strdup(odr, ccl_err_msg(cerror));
 989             int z3950_diag = YAZ_BIB1_MALFORMED_QUERY;
 990
 991             switch (cerror)
 992             {
 993             case CCL_ERR_UNKNOWN_QUAL:
 994                 z3950_diag = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
 995                 break;
 996             case CCL_ERR_TRUNC_NOT_LEFT:
 997             case CCL_ERR_TRUNC_NOT_RIGHT:
 998             case CCL_ERR_TRUNC_NOT_BOTH:
 999                 z3950_diag = YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE;
1000                 break;
1001             }
1002             apdu_res =
1003                 odr.create_searchResponse(apdu_req, z3950_diag, addinfo);
1004             package.response() = apdu_res;
1005             return;
1006         }
1007         pqf_wrbuf = wrbuf_alloc();
1008         ccl_pquery(pqf_wrbuf, cn);
1009         ccl_rpn_delete(cn);
1010     }
1011
1012     assert(pqf_wrbuf);
1013     if (b->get_option("sru"))
1014     {
1015         int status = 0;
1016         Z_RPNQuery *zquery;
1017         zquery = p_query_rpn(odr, wrbuf_cstr(pqf_wrbuf));
1018         WRBUF wrb = wrbuf_alloc();
1019
1020         if (!strcmp(b->get_option("sru"), "solr"))
1021         {
1022             solr_transform_t cqlt = solr_transform_create();
1023
1024             status = solr_transform_rpn2solr_wrbuf(cqlt, wrb, zquery);
1025
1026             solr_transform_close(cqlt);
1027         }
1028         else
1029         {
1030             cql_transform_t cqlt = cql_transform_create();
1031
1032             status = cql_transform_rpn2cql_wrbuf(cqlt, wrb, zquery);
1033
1034             cql_transform_close(cqlt);
1035         }
1036         if (status == 0)
1037         {
1038             yaz_log(YLOG_LOG, "search CQL: %s", wrbuf_cstr(wrb));
1039             b->search_cql(wrbuf_cstr(wrb), &hits, &error, &addinfo);
1040         }
1041
1042         wrbuf_destroy(wrb);
1043         wrbuf_destroy(pqf_wrbuf);
1044         if (status)
1045         {
1046             apdu_res =
1047                 odr.create_searchResponse(apdu_req, YAZ_BIB1_MALFORMED_QUERY,
1048                                           "can not convert from RPN to CQL/SOLR");
1049             package.response() = apdu_res;
1050             return;
1051         }
1052     }
1053     else
1054     {
1055         yaz_log(YLOG_LOG, "search PQF: %s", wrbuf_cstr(pqf_wrbuf));
1056         b->search_pqf(wrbuf_cstr(pqf_wrbuf), &hits, &error, &addinfo);
1057         wrbuf_destroy(pqf_wrbuf);
1058     }
1059
1060
1061     const char *element_set_name = 0;
1062     Odr_int number_to_present = 0;
1063     if (!error)
1064         mp::util::piggyback_sr(sr, hits, number_to_present, &element_set_name);
1065
1066     Odr_int number_of_records_returned = 0;
1067     Z_Records *records = get_records(
1068         0, number_to_present, &error, &addinfo,
1069         &number_of_records_returned, odr, b, sr->preferredRecordSyntax,
1070         element_set_name);
1071     apdu_res = odr.create_searchResponse(apdu_req, error, addinfo);
1072     if (records)
1073     {
1074         apdu_res->u.searchResponse->records = records;
1075         apdu_res->u.searchResponse->numberOfRecordsReturned =
1076             odr_intdup(odr, number_of_records_returned);
1077     }
1078     apdu_res->u.searchResponse->resultCount = odr_intdup(odr, hits);
1079     package.response() = apdu_res;
1080 }
1081
1082 void yf::Zoom::Frontend::handle_present(mp::Package &package)
1083 {
1084     Z_GDU *gdu = package.request().get();
1085     Z_APDU *apdu_req = gdu->u.z3950;
1086     Z_APDU *apdu_res = 0;
1087     Z_PresentRequest *pr = apdu_req->u.presentRequest;
1088
1089     mp::odr odr;
1090     if (!m_backend)
1091     {
1092         package.response() = odr.create_presentResponse(
1093             apdu_req, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST, 0);
1094         return;
1095     }
1096     const char *element_set_name = 0;
1097     Z_RecordComposition *comp = pr->recordComposition;
1098     if (comp && comp->which != Z_RecordComp_simple)
1099     {
1100         package.response() = odr.create_presentResponse(
1101             apdu_req,
1102             YAZ_BIB1_PRESENT_COMP_SPEC_PARAMETER_UNSUPP, 0);
1103         return;
1104     }
1105     if (comp && comp->u.simple->which == Z_ElementSetNames_generic)
1106         element_set_name = comp->u.simple->u.generic;
1107     Odr_int number_of_records_returned = 0;
1108     int error = 0;
1109     const char *addinfo = 0;
1110     Z_Records *records = get_records(
1111         *pr->resultSetStartPoint - 1, *pr->numberOfRecordsRequested,
1112         &error, &addinfo, &number_of_records_returned, odr, m_backend,
1113         pr->preferredRecordSyntax, element_set_name);
1114
1115     apdu_res = odr.create_presentResponse(apdu_req, error, addinfo);
1116     if (records)
1117     {
1118         apdu_res->u.presentResponse->records = records;
1119         apdu_res->u.presentResponse->numberOfRecordsReturned =
1120             odr_intdup(odr, number_of_records_returned);
1121     }
1122     package.response() = apdu_res;
1123 }
1124
1125 void yf::Zoom::Frontend::handle_package(mp::Package &package)
1126 {
1127     Z_GDU *gdu = package.request().get();
1128     if (!gdu)
1129         ;
1130     else if (gdu->which == Z_GDU_Z3950)
1131     {
1132         Z_APDU *apdu_req = gdu->u.z3950;
1133         if (apdu_req->which == Z_APDU_initRequest)
1134         {
1135             mp::odr odr;
1136             package.response() = odr.create_close(
1137                 apdu_req,
1138                 Z_Close_protocolError,
1139                 "double init");
1140         }
1141         else if (apdu_req->which == Z_APDU_searchRequest)
1142         {
1143             handle_search(package);
1144         }
1145         else if (apdu_req->which == Z_APDU_presentRequest)
1146         {
1147             handle_present(package);
1148         }
1149         else
1150         {
1151             mp::odr odr;
1152             package.response() = odr.create_close(
1153                 apdu_req,
1154                 Z_Close_protocolError,
1155                 "zoom filter cannot handle this APDU");
1156             package.session().close();
1157         }
1158     }
1159     else
1160     {
1161         package.session().close();
1162     }
1163 }
1164
1165 void yf::Zoom::Impl::process(mp::Package &package)
1166 {
1167     FrontendPtr f = get_frontend(package);
1168     Z_GDU *gdu = package.request().get();
1169
1170     if (f->m_is_virtual)
1171     {
1172         f->handle_package(package);
1173     }
1174     else if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
1175              Z_APDU_initRequest)
1176     {
1177         Z_InitRequest *req = gdu->u.z3950->u.initRequest;
1178         f->m_init_gdu = gdu;
1179
1180         mp::odr odr;
1181         Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
1182         Z_InitResponse *resp = apdu->u.initResponse;
1183
1184         int i;
1185         static const int masks[] = {
1186             Z_Options_search,
1187             Z_Options_present,
1188             -1
1189         };
1190         for (i = 0; masks[i] != -1; i++)
1191             if (ODR_MASK_GET(req->options, masks[i]))
1192                 ODR_MASK_SET(resp->options, masks[i]);
1193
1194         static const int versions[] = {
1195             Z_ProtocolVersion_1,
1196             Z_ProtocolVersion_2,
1197             Z_ProtocolVersion_3,
1198             -1
1199         };
1200         for (i = 0; versions[i] != -1; i++)
1201             if (ODR_MASK_GET(req->protocolVersion, versions[i]))
1202                 ODR_MASK_SET(resp->protocolVersion, versions[i]);
1203             else
1204                 break;
1205
1206         *resp->preferredMessageSize = *req->preferredMessageSize;
1207         *resp->maximumRecordSize = *req->maximumRecordSize;
1208
1209         package.response() = apdu;
1210         f->m_is_virtual = true;
1211     }
1212     else
1213         package.move();
1214
1215     release_frontend(package);
1216 }
1217
1218
1219 static mp::filter::Base* filter_creator()
1220 {
1221     return new mp::filter::Zoom;
1222 }
1223
1224 extern "C" {
1225     struct metaproxy_1_filter_struct metaproxy_1_filter_zoom = {
1226         0,
1227         "zoom",
1228         filter_creator
1229     };
1230 }
1231
1232
1233 /*
1234  * Local variables:
1235  * c-basic-offset: 4
1236  * c-file-style: "Stroustrup"
1237  * indent-tabs-mode: nil
1238  * End:
1239  * vim: shiftwidth=4 tabstop=8 expandtab
1240  */
1241