src/filter_sparql.cpp

   1 /* This file is part of Metaproxy.
   2    Copyright (C) Index Data
   3
   4 Metaproxy is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17 */
  18
  19 #include <metaproxy/package.hpp>
  20 #include <metaproxy/util.hpp>
  21 #include <yaz/log.h>
  22 #include <yaz/srw.h>
  23 #include <yaz/diagbib1.h>
  24 #include <yaz/match_glob.h>
  25 #include <yaz/querytowrbuf.h>
  26 #include <boost/scoped_ptr.hpp>
  27 #include <boost/thread/mutex.hpp>
  28 #include <boost/thread/condition.hpp>
  29 #include <boost/algorithm/string.hpp>
  30 #include "sparql.h"
  31
  32 #include <yaz/zgdu.h>
  33
  34 namespace mp = metaproxy_1;
  35 namespace yf = mp::filter;
  36
  37 namespace metaproxy_1 {
  38     namespace filter {
  39         class SPARQL : public Base {
  40             class Session;
  41             class Rep;
  42             class Conf;
  43             class Result;
  44             class FrontendSet;
  45
  46             typedef boost::shared_ptr<Session> SessionPtr;
  47             typedef boost::shared_ptr<Conf> ConfPtr;
  48
  49             typedef boost::shared_ptr<FrontendSet> FrontendSetPtr;
  50             typedef std::map<std::string,FrontendSetPtr> FrontendSets;
  51         public:
  52             SPARQL();
  53             ~SPARQL();
  54             void process(metaproxy_1::Package & package) const;
  55             void configure(const xmlNode * ptr, bool test_only,
  56                            const char *path);
  57             SessionPtr get_session(Package &package, Z_APDU **apdu) const;
  58             void release_session(Package &package) const;
  59             boost::scoped_ptr<Rep> m_p;
  60             std::list<ConfPtr> db_conf;
  61         };
  62         class SPARQL::Conf {
  63         public:
  64             std::string db;
  65             std::string uri;
  66             std::string schema;
  67             yaz_sparql_t s;
  68             ~Conf();
  69         };
  70         class SPARQL::Rep {
  71             friend class SPARQL;
  72             boost::condition m_cond_session_ready;
  73             boost::mutex m_mutex;
  74             std::map<mp::Session,SessionPtr> m_clients;
  75         };
  76         class SPARQL::Result {
  77         public:
  78             Result();
  79             ~Result();
  80         private:
  81             friend class FrontendSet;
  82             friend class Session;
  83             ConfPtr conf;
  84             xmlDoc *doc;
  85         };
  86         class SPARQL::FrontendSet {
  87         private:
  88             friend class Session;
  89             Odr_int hits;
  90             std::string db;
  91             std::list<Result> results;
  92             std::vector<ConfPtr> explaindblist;
  93         };
  94         class SPARQL::Session {
  95         public:
  96             Session(const SPARQL *);
  97             ~Session();
  98             void handle_z(Package &package, Z_APDU *apdu);
  99             Z_APDU *search(mp::Package &package,
 100                            Z_APDU *apdu_req,
 101                            mp::odr &odr,
 102                            const char *sparql_query,
 103                            ConfPtr conf,
 104                            FrontendSetPtr fset);
 105             Z_APDU *explain_search(mp::Package &package,
 106                            Z_APDU *apdu_req,
 107                            mp::odr &odr,
 108                            const char *sparql_query,
 109                            FrontendSetPtr fset);
 110             int invoke_sparql(mp::Package &package,
 111                               const char *sparql_query,
 112                               ConfPtr conf,
 113                               WRBUF w);
 114             Z_Records *fetch(
 115                 Package &package,
 116                 FrontendSetPtr fset,
 117                 ODR odr, Odr_oid *preferredRecordSyntax,
 118                 Z_ElementSetNames *esn,
 119                 int start, int number, int &error_code, std::string &addinfo,
 120                 int *number_returned, int *next_position);
 121             Z_Records *explain_fetch(
 122                 Package &package,
 123                 FrontendSetPtr fset,
 124                 ODR odr, Odr_oid *preferredRecordSyntax,
 125                 Z_ElementSetNames *esn,
 126                 int start, int number, int &error_code, std::string &addinfo,
 127                 int *number_returned, int *next_position);
 128             bool m_in_use;
 129         private:
 130             bool m_support_named_result_sets;
 131             FrontendSets m_frontend_sets;
 132             const SPARQL *m_sparql;
 133         };
 134     }
 135 }
 136
 137 yf::SPARQL::Result::~Result()
 138 {
 139     if (doc)
 140         xmlFreeDoc(doc);
 141 }
 142
 143 yf::SPARQL::Result::Result()
 144 {
 145     doc = 0;
 146 }
 147
 148 yf::SPARQL::SPARQL() : m_p(new Rep)
 149 {
 150 }
 151
 152 yf::SPARQL::~SPARQL()
 153 {
 154 }
 155
 156 void yf::SPARQL::configure(const xmlNode *xmlnode, bool test_only,
 157                            const char *path)
 158 {
 159     const xmlNode *ptr = xmlnode->children;
 160     std::string uri;
 161
 162     for (; ptr; ptr = ptr->next)
 163     {
 164         if (ptr->type != XML_ELEMENT_NODE)
 165             continue;
 166         if (!strcmp((const char *) ptr->name, "defaults"))
 167         {
 168             const struct _xmlAttr *attr;
 169             for (attr = ptr->properties; attr; attr = attr->next)
 170             {
 171                 if (!strcmp((const char *) attr->name, "uri"))
 172                     uri = mp::xml::get_text(attr->children);
 173                 else
 174                     throw mp::filter::FilterException(
 175                         "Bad attribute " + std::string((const char *)
 176                                                        attr->name));
 177             }
 178         }
 179         else if (!strcmp((const char *) ptr->name, "db"))
 180         {
 181             yaz_sparql_t s = yaz_sparql_create();
 182             ConfPtr conf(new Conf);
 183             conf->s = s;
 184             conf->uri = uri;
 185
 186             const struct _xmlAttr *attr;
 187             for (attr = ptr->properties; attr; attr = attr->next)
 188             {
 189                 if (!strcmp((const char *) attr->name, "path"))
 190                     conf->db = mp::xml::get_text(attr->children);
 191                 else if (!strcmp((const char *) attr->name, "uri"))
 192                     conf->uri = mp::xml::get_text(attr->children);
 193                 else if (!strcmp((const char *) attr->name, "schema"))
 194                     conf->schema = mp::xml::get_text(attr->children);
 195                 else if (!strcmp((const char *) attr->name, "include"))
 196                 {
 197                     std::vector<std::string> dbs;
 198                     std::string db = mp::xml::get_text(attr->children);
 199                     boost::split(dbs, db, boost::is_any_of(" \t"));
 200                     size_t i;
 201                     for (i = 0; i < dbs.size(); i++)
 202                     {
 203                         if (dbs[i].length() == 0)
 204                             continue;
 205                         std::list<ConfPtr>::const_iterator it = db_conf.begin();
 206                         while (1)
 207                             if (it == db_conf.end())
 208                             {
 209                                 throw mp::filter::FilterException(
 210                                     "include db not found: " + dbs[i]);
 211                             }
 212                             else if (dbs[i].compare((*it)->db) == 0)
 213                             {
 214                                 yaz_sparql_include(s, (*it)->s);
 215                                 break;
 216                             }
 217                             else
 218                                 it++;
 219                     }
 220                 }
 221                 else
 222                     throw mp::filter::FilterException(
 223                         "Bad attribute " + std::string((const char *)
 224                                                        attr->name));
 225             }
 226             xmlNode *p = ptr->children;
 227             for (; p; p = p->next)
 228             {
 229                 if (p->type != XML_ELEMENT_NODE)
 230                     continue;
 231                 std::string name = (const char *) p->name;
 232                 const struct _xmlAttr *attr;
 233                 for (attr = p->properties; attr; attr = attr->next)
 234                 {
 235                     if (!strcmp((const char *) attr->name, "type"))
 236                     {
 237                         name.append(".");
 238                         name.append(mp::xml::get_text(attr->children));
 239                     }
 240                     else
 241                         throw mp::filter::FilterException(
 242                             "Bad attribute " + std::string((const char *)
 243                                                            attr->name));
 244                 }
 245                 std::string value = mp::xml::get_text(p);
 246                 if (yaz_sparql_add_pattern(s, name.c_str(), value.c_str()))
 247                 {
 248                     throw mp::filter::FilterException(
 249                         "Bad SPARQL config " + name);
 250                 }
 251             }
 252             if (!conf->uri.length())
 253             {
 254                 throw mp::filter::FilterException("Missing uri");
 255             }
 256             if (!conf->db.length())
 257             {
 258                 throw mp::filter::FilterException("Missing path");
 259             }
 260             db_conf.push_back(conf);
 261         }
 262         else
 263         {
 264             throw mp::filter::FilterException
 265                 ("Bad element "
 266                  + std::string((const char *) ptr->name)
 267                  + " in sparql filter");
 268         }
 269     }
 270 }
 271
 272 yf::SPARQL::Conf::~Conf()
 273 {
 274     yaz_sparql_destroy(s);
 275 }
 276
 277 yf::SPARQL::Session::Session(const SPARQL *sparql) :
 278     m_in_use(true),
 279     m_support_named_result_sets(false),
 280     m_sparql(sparql)
 281 {
 282 }
 283
 284 yf::SPARQL::Session::~Session()
 285 {
 286 }
 287
 288 yf::SPARQL::SessionPtr yf::SPARQL::get_session(Package & package,
 289                                                Z_APDU **apdu) const
 290 {
 291     SessionPtr ptr0;
 292
 293     Z_GDU *gdu = package.request().get();
 294
 295     boost::mutex::scoped_lock lock(m_p->m_mutex);
 296
 297     std::map<mp::Session,SPARQL::SessionPtr>::iterator it;
 298
 299     if (gdu && gdu->which == Z_GDU_Z3950)
 300         *apdu = gdu->u.z3950;
 301     else
 302         *apdu = 0;
 303
 304     while (true)
 305     {
 306         it = m_p->m_clients.find(package.session());
 307         if (it == m_p->m_clients.end())
 308             break;
 309         if (!it->second->m_in_use)
 310         {
 311             it->second->m_in_use = true;
 312             return it->second;
 313         }
 314         m_p->m_cond_session_ready.wait(lock);
 315     }
 316     if (!*apdu)
 317         return ptr0;
 318
 319     // new Z39.50 session ..
 320     SessionPtr p(new Session(this));
 321     m_p->m_clients[package.session()] = p;
 322     return p;
 323 }
 324
 325 void yf::SPARQL::release_session(Package &package) const
 326 {
 327     boost::mutex::scoped_lock lock(m_p->m_mutex);
 328     std::map<mp::Session,SessionPtr>::iterator it;
 329
 330     it = m_p->m_clients.find(package.session());
 331     if (it != m_p->m_clients.end())
 332     {
 333         it->second->m_in_use = false;
 334
 335         if (package.session().is_closed())
 336             m_p->m_clients.erase(it);
 337         m_p->m_cond_session_ready.notify_all();
 338     }
 339 }
 340
 341 static bool get_result(xmlDoc *doc, Odr_int *sz, Odr_int pos, xmlDoc **ndoc)
 342 {
 343     xmlNode *ptr = xmlDocGetRootElement(doc);
 344     xmlNode *q0;
 345     Odr_int cur = 0;
 346
 347     if (ndoc)
 348         *ndoc = xmlNewDoc(BAD_CAST "1.0");
 349
 350     if (ptr->type == XML_ELEMENT_NODE &&
 351         !strcmp((const char *) ptr->name, "RDF"))
 352     {
 353         if (ndoc)
 354         {
 355             q0 = xmlCopyNode(ptr, 2);
 356             xmlDocSetRootElement(*ndoc, q0);
 357         }
 358         ptr = ptr->children;
 359
 360         while (ptr && ptr->type != XML_ELEMENT_NODE)
 361             ptr = ptr->next;
 362         if (ptr && ptr->type == XML_ELEMENT_NODE &&
 363             !strcmp((const char *) ptr->name, "Description"))
 364         {
 365             xmlNode *p = ptr->children;
 366
 367             while (p && p->type != XML_ELEMENT_NODE)
 368                 p = p->next;
 369             if (p && p->type == XML_ELEMENT_NODE &&
 370                 !strcmp((const char *) p->name, "type"))
 371             { /* SELECT RESULT */
 372                 for (ptr = ptr->children; ptr; ptr = ptr->next)
 373                     if (ptr->type == XML_ELEMENT_NODE &&
 374                         !strcmp((const char *) ptr->name, "solution"))
 375                     {
 376                         if (cur++ == pos)
 377                         {
 378                             if (ndoc)
 379                             {
 380                                 xmlNode *q1 = xmlCopyNode(ptr, 1);
 381                                 xmlAddChild(q0, q1);
 382                             }
 383                             break;
 384                         }
 385                     }
 386             }
 387             else
 388             {   /* CONSTRUCT result */
 389                 for (; ptr; ptr = ptr->next)
 390                     if (ptr->type == XML_ELEMENT_NODE &&
 391                         !strcmp((const char *) ptr->name, "Description"))
 392                     {
 393                         if (cur++ == pos)
 394                         {
 395                             if (ndoc)
 396                             {
 397                                 xmlNode *q1 = xmlCopyNode(ptr, 1);
 398                                 xmlAddChild(q0, q1);
 399                             }
 400                             return true;
 401                         }
 402                     }
 403             }
 404         }
 405     }
 406     else
 407     {
 408         for (; ptr; ptr = ptr->next)
 409             if (ptr->type == XML_ELEMENT_NODE &&
 410                 !strcmp((const char *) ptr->name, "sparql"))
 411                 break;
 412         if (ptr)
 413         {
 414             if (ndoc)
 415             {
 416                 q0 = xmlCopyNode(ptr, 2);
 417                 xmlDocSetRootElement(*ndoc, q0);
 418             }
 419             for (ptr = ptr->children; ptr; ptr = ptr->next)
 420                 if (ptr->type == XML_ELEMENT_NODE &&
 421                     !strcmp((const char *) ptr->name, "results"))
 422                     break;
 423         }
 424         if (ptr)
 425         {
 426             xmlNode *q1 = 0;
 427             if (ndoc)
 428             {
 429                 q1 = xmlCopyNode(ptr, 0);
 430                 xmlAddChild(q0, q1);
 431             }
 432             for (ptr = ptr->children; ptr; ptr = ptr->next)
 433                 if (ptr->type == XML_ELEMENT_NODE &&
 434                     !strcmp((const char *) ptr->name, "result"))
 435                 {
 436                     if (cur++ == pos)
 437                     {
 438                         if (ndoc)
 439                         {
 440                             xmlNode *q2 = xmlCopyNode(ptr, 1);
 441                             xmlAddChild(q1, q2);
 442                         }
 443                         return true;
 444                     }
 445                 }
 446         }
 447     }
 448     if (sz)
 449         *sz = cur;
 450     return false;
 451 }
 452
 453 Z_Records *yf::SPARQL::Session::fetch(
 454     Package &package,
 455     FrontendSetPtr fset,
 456     ODR odr, Odr_oid *preferredRecordSyntax,
 457     Z_ElementSetNames *esn,
 458     int start, int number, int &error_code, std::string &addinfo,
 459     int *number_returned, int *next_position)
 460 {
 461     Z_Records *rec = (Z_Records *) odr_malloc(odr, sizeof(Z_Records));
 462     std::list<Result>::iterator it = fset->results.begin();
 463     const char *schema = 0;
 464     bool uri_lookup = false;
 465     bool fetch_logged = false;
 466     if (esn && esn->which == Z_ElementSetNames_generic)
 467         schema = esn->u.generic;
 468
 469     for (; it != fset->results.end(); it++)
 470     {
 471         if (yaz_sparql_lookup_schema(it->conf->s, schema))
 472         {
 473             uri_lookup = true;
 474             break;
 475         }
 476         if (!schema || !strcmp(esn->u.generic, it->conf->schema.c_str()))
 477             break;
 478     }
 479     if (it == fset->results.end())
 480     {
 481         rec->which = Z_Records_NSD;
 482         rec->u.nonSurrogateDiagnostic =
 483             zget_DefaultDiagFormat(
 484                 odr,
 485                 YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_,
 486                 schema);
 487         return rec;
 488     }
 489     rec->which = Z_Records_DBOSD;
 490     rec->u.databaseOrSurDiagnostics = (Z_NamePlusRecordList *)
 491         odr_malloc(odr, sizeof(Z_NamePlusRecordList));
 492     rec->u.databaseOrSurDiagnostics->records = (Z_NamePlusRecord **)
 493         odr_malloc(odr, sizeof(Z_NamePlusRecord *) * number);
 494     int i;
 495     for (i = 0; i < number; i++)
 496     {
 497         rec->u.databaseOrSurDiagnostics->records[i] = (Z_NamePlusRecord *)
 498             odr_malloc(odr, sizeof(Z_NamePlusRecord));
 499         Z_NamePlusRecord *npr = rec->u.databaseOrSurDiagnostics->records[i];
 500         npr->databaseName = odr_strdup(odr, fset->db.c_str());
 501         npr->which = Z_NamePlusRecord_databaseRecord;
 502         xmlDoc *ndoc = 0;
 503
 504         if (!get_result(it->doc, 0, start - 1 + i, &ndoc))
 505         {
 506             if (ndoc)
 507                 xmlFreeDoc(ndoc);
 508             break;
 509         }
 510         xmlNode *ndoc_root = xmlDocGetRootElement(ndoc);
 511         if (!ndoc_root)
 512         {
 513             xmlFreeDoc(ndoc);
 514             break;
 515         }
 516         if (uri_lookup)
 517         {
 518             std::string uri;
 519             xmlNode *n = ndoc_root;
 520             while (n)
 521             {
 522                 if (n->type == XML_ELEMENT_NODE)
 523                 {
 524                     if (!strcmp((const char *) n->name, "uri") ||
 525                         !strcmp((const char *) n->name, "bnode") )
 526                     {
 527                         uri = mp::xml::get_text(n->children);
 528
 529                     }
 530                     n = n->children;
 531                 }
 532                 else
 533                     n = n->next;
 534             }
 535             if (!uri.length())
 536             {
 537                 rec->which = Z_Records_NSD;
 538                 rec->u.nonSurrogateDiagnostic =
 539                     zget_DefaultDiagFormat(
 540                         odr,
 541                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS, 0);
 542                 xmlFreeDoc(ndoc);
 543                 return rec;
 544             }
 545             else
 546             {
 547                 mp::wrbuf addinfo, query, w;
 548                 int error = yaz_sparql_from_uri_wrbuf(it->conf->s,
 549                                                       addinfo, query,
 550                                                       uri.c_str(), schema);
 551                 if (!error)
 552                 {
 553                     if (!fetch_logged)
 554                     { // Log the fetch query only once
 555                         package.log("sparql", YLOG_LOG,
 556                             "fetch query: for %s \n%s",
 557                             uri.c_str(), query.c_str() );
 558                         fetch_logged = true;
 559                     }
 560                     else
 561                     {
 562                         package.log("sparql", YLOG_LOG,
 563                             "fetch uri:%s", uri.c_str() );
 564                     }
 565                     error = invoke_sparql(package, query.c_str(),
 566                                           it->conf, w);
 567                 }
 568                 if (error)
 569                 {
 570                     rec->which = Z_Records_NSD;
 571                     rec->u.nonSurrogateDiagnostic =
 572                         zget_DefaultDiagFormat(
 573                             odr,
 574                             error,
 575                             addinfo.len() ? addinfo.c_str() : 0);
 576                     xmlFreeDoc(ndoc);
 577                     return rec;
 578                 }
 579                 npr->u.databaseRecord =
 580                     z_ext_record_xml(odr, w.c_str(), w.len());
 581             }
 582         }
 583         else
 584         {
 585             xmlBufferPtr buf = xmlBufferCreate();
 586             xmlNodeDump(buf, ndoc, ndoc_root, 0, 0);
 587             yaz_log(YLOG_LOG, "record %s %.*s", uri_lookup ? "uri" : "normal",
 588                     (int) buf->use, (const char *) buf->content);
 589             npr->u.databaseRecord =
 590                 z_ext_record_xml(odr, (const char *) buf->content, buf->use);
 591             xmlBufferFree(buf);
 592         }
 593         xmlFreeDoc(ndoc);
 594     }
 595     rec->u.databaseOrSurDiagnostics->num_records = i;
 596     *number_returned = i;
 597     if (start + number > fset->hits)
 598         *next_position = 0;
 599     else
 600         *next_position = start + number;
 601     return rec;
 602 }
 603
 604 int yf::SPARQL::Session::invoke_sparql(mp::Package &package,
 605                                        const char *sparql_query,
 606                                        ConfPtr conf,
 607                                        WRBUF w)
 608 {
 609     Package http_package(package.session(), package.origin());
 610     mp::odr odr;
 611
 612     http_package.copy_filter(package);
 613     Z_GDU *gdu = z_get_HTTP_Request_uri(odr, conf->uri.c_str(), 0, 1);
 614
 615     z_HTTP_header_add(odr, &gdu->u.HTTP_Request->headers,
 616                       "Content-Type", "application/x-www-form-urlencoded");
 617     z_HTTP_header_add(odr, &gdu->u.HTTP_Request->headers,
 618                       "Accept", "application/sparql-results+xml,"
 619                       "application/rdf+xml");
 620     const char *names[2];
 621     names[0] = "query";
 622     names[1] = 0;
 623     const char *values[1];
 624     values[0] = sparql_query;
 625     char *path = 0;
 626     yaz_array_to_uri(&path, odr, (char **) names, (char **) values);
 627
 628     gdu->u.HTTP_Request->content_buf = path;
 629     gdu->u.HTTP_Request->content_len = strlen(path);
 630
 631     yaz_log(YLOG_DEBUG, "sparql: HTTP request\n%s", sparql_query);
 632
 633     http_package.request() = gdu;
 634     http_package.move();
 635
 636     Z_GDU *gdu_resp = http_package.response().get();
 637
 638     if (!gdu_resp || gdu_resp->which != Z_GDU_HTTP_Response)
 639     {
 640         wrbuf_puts(w, "no HTTP response from backend");
 641         return YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
 642     }
 643     else if (gdu_resp->u.HTTP_Response->code != 200)
 644     {
 645         Z_HTTP_Response *resp = gdu_resp->u.HTTP_Response;
 646         wrbuf_printf(w, "sparql: HTTP error %d from backend",
 647                      resp->code);
 648         package.log("sparql", YLOG_LOG,
 649             "HTTP error %d from backend ",
 650             resp->code );
 651         package.log("sparql", YLOG_LOG,
 652             "%.*s" , resp->content_len, resp->content_buf );
 653         return YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
 654     }
 655     Z_HTTP_Response *resp = gdu_resp->u.HTTP_Response;
 656     wrbuf_write(w, resp->content_buf, resp->content_len);
 657     return 0;
 658 }
 659
 660 Z_Records *yf::SPARQL::Session::explain_fetch(
 661     Package &package,
 662     FrontendSetPtr fset,
 663     ODR odr, Odr_oid *preferredRecordSyntax,
 664     Z_ElementSetNames *esn,
 665     int start, int number, int &error_code, std::string &addinfo,
 666     int *number_returned, int *next_position)
 667 {
 668     Z_Records *rec = (Z_Records *) odr_malloc(odr, sizeof(Z_Records));
 669     rec->which = Z_Records_DBOSD;
 670     rec->u.databaseOrSurDiagnostics = (Z_NamePlusRecordList *)
 671         odr_malloc(odr, sizeof(Z_NamePlusRecordList));
 672     rec->u.databaseOrSurDiagnostics->records = (Z_NamePlusRecord **)
 673         odr_malloc(odr, sizeof(Z_NamePlusRecord *) * number);
 674     int i;
 675     for (i = 0; i < number; i++)
 676     {
 677         unsigned int idx = start + i - 1;
 678         if ( idx >= fset->explaindblist.size() )
 679             break;
 680         ConfPtr cp = fset->explaindblist[idx];
 681         mp::wrbuf w;
 682         wrbuf_puts(w,"<info>\n");
 683         wrbuf_puts(w,"  <databaseInfo>\n");
 684         wrbuf_puts(w,"    <title>");
 685         wrbuf_xmlputs(w, cp->db.c_str());
 686         wrbuf_puts(w,"</title>\n");
 687         wrbuf_puts(w,"  </databaseInfo>\n");
 688         yaz_sparql_explain_indexes( cp->s, w, 2);
 689         wrbuf_puts(w,"</info>\n");
 690
 691         rec->u.databaseOrSurDiagnostics->records[i] = (Z_NamePlusRecord *)
 692             odr_malloc(odr, sizeof(Z_NamePlusRecord));
 693         Z_NamePlusRecord *npr = rec->u.databaseOrSurDiagnostics->records[i];
 694         npr->databaseName = odr_strdup(odr, fset->db.c_str());
 695         npr->which = Z_NamePlusRecord_databaseRecord;
 696         npr->u.databaseRecord =
 697             z_ext_record_xml(odr, w.buf(), w.len() );
 698     }
 699     rec->u.databaseOrSurDiagnostics->num_records = i;
 700     *number_returned = i;
 701     if (start + number > (int)fset->explaindblist.size())
 702         *next_position = 0;
 703     else
 704         *next_position = start + number;
 705     return rec;
 706 }
 707
 708
 709
 710 Z_APDU *yf::SPARQL::Session::explain_search(mp::Package &package,
 711                            Z_APDU *apdu_req,
 712                            mp::odr &odr,
 713                            const char *explain_query,
 714                            FrontendSetPtr fset)
 715 {
 716     Z_SearchRequest *req = apdu_req->u.searchRequest;
 717     Z_APDU *apdu_res = 0;
 718     //mp::wrbuf w;
 719
 720     package.log("sparql", YLOG_LOG, "Explain search '%s'", explain_query );
 721     const char *term = explain_query + strlen(explain_query);
 722     while ( term > explain_query && *term != ' ')
 723         term--;
 724     term++;
 725     if ( ! isalpha( *term) )
 726         term=""; // anything non-alpha is taken to mean all
 727                  // Empty string is seen here as two double quotes ""
 728                  // so it returns all bases as well
 729     int numbases = 0;
 730     std::list<ConfPtr>::const_iterator it = m_sparql->db_conf.begin();
 731     m_frontend_sets[req->resultSetName] = fset;
 732     fset->explaindblist.clear();
 733     fset->explaindblist.reserve(m_sparql->db_conf.size());
 734
 735     for (; it != m_sparql->db_conf.end(); it++)
 736         if ( (*it)->schema.length() > 0  &&  // searchable db
 737             (!*term || strcmp(term,(*it)->db.c_str())==0)  )
 738         { // and want all, or found the matching one
 739             numbases++;
 740             package.log("sparql", YLOG_LOG, "Explain %d: '%s'",
 741                         numbases, (*it)->db.c_str() );
 742             fset->explaindblist.push_back(*it);
 743         }
 744     int number_returned = 0;
 745     int next_position = 0;
 746     Z_Records *records = 0;
 747     int error_code = 0;
 748     std::string addinfo;
 749
 750     Odr_int number = 0;
 751     const char *element_set_name = 0;
 752     mp::util::piggyback_sr(req, numbases, number, &element_set_name);
 753     if (number)
 754     {
 755         Z_ElementSetNames *esn;
 756
 757         if (number > *req->smallSetUpperBound)
 758             esn = req->mediumSetElementSetNames;
 759         else
 760             esn = req->smallSetElementSetNames;
 761         records = explain_fetch(package, fset,
 762                         odr, req->preferredRecordSyntax, esn,
 763                         1, number,
 764                         error_code, addinfo,
 765                         &number_returned,
 766                         &next_position);
 767     }
 768
 769     if (error_code)
 770     {
 771         apdu_res = odr.create_searchResponse(
 772                 apdu_req, error_code, addinfo.c_str());
 773     }
 774     else
 775     {
 776         apdu_res = odr.create_searchResponse(apdu_req, 0, 0);
 777         Z_SearchResponse *resp = apdu_res->u.searchResponse;
 778         *resp->resultCount = numbases;
 779         *resp->numberOfRecordsReturned = number_returned;
 780         *resp->nextResultSetPosition = next_position;
 781         resp->records = records;
 782     }
 783
 784     return apdu_res;
 785 }
 786
 787
 788 Z_APDU *yf::SPARQL::Session::search(mp::Package &package,
 789                                     Z_APDU *apdu_req,
 790                                     mp::odr &odr,
 791                                     const char *sparql_query,
 792                                     ConfPtr conf, FrontendSetPtr fset)
 793 {
 794     Z_SearchRequest *req = apdu_req->u.searchRequest;
 795     Z_APDU *apdu_res = 0;
 796     mp::wrbuf w;
 797
 798     package.log("sparql", YLOG_LOG,
 799         "search query:\n%s", sparql_query );
 800
 801     int error = invoke_sparql(package, sparql_query, conf, w);
 802     if (error)
 803     {
 804         apdu_res = odr.create_searchResponse(apdu_req, error,
 805                                              w.len() ?
 806                                              w.c_str() : 0);
 807     }
 808     else
 809     {
 810         xmlDocPtr doc = xmlParseMemory(w.c_str(), w.len());
 811         if (!doc)
 812         {
 813             apdu_res = odr.create_searchResponse(
 814                 apdu_req,
 815                 YAZ_BIB1_TEMPORARY_SYSTEM_ERROR,
 816                 "invalid XML from backendbackend");
 817         }
 818         else
 819         {
 820             Result result;
 821             Z_Records *records = 0;
 822             int number_returned = 0;
 823             int next_position = 0;
 824             int error_code = 0;
 825             std::string addinfo;
 826
 827             result.doc = doc;
 828             result.conf = conf;
 829             fset->results.push_back(result);
 830             yaz_log(YLOG_DEBUG, "saving sparql result xmldoc=%p", doc);
 831
 832             get_result(result.doc, &fset->hits, -1, 0);
 833             m_frontend_sets[req->resultSetName] = fset;
 834
 835             result.doc = 0;
 836
 837             Odr_int number = 0;
 838             const char *element_set_name = 0;
 839             mp::util::piggyback_sr(req, fset->hits, number, &element_set_name);
 840             if (number)
 841             {
 842                 Z_ElementSetNames *esn;
 843
 844                 if (number > *req->smallSetUpperBound)
 845                     esn = req->mediumSetElementSetNames;
 846                 else
 847                     esn = req->smallSetElementSetNames;
 848                 records = fetch(package, fset,
 849                                 odr, req->preferredRecordSyntax, esn,
 850                                 1, number,
 851                                 error_code, addinfo,
 852                                 &number_returned,
 853                                 &next_position);
 854             }
 855             if (error_code)
 856             {
 857                 apdu_res =
 858                     odr.create_searchResponse(
 859                         apdu_req, error_code, addinfo.c_str());
 860             }
 861             else
 862             {
 863                 apdu_res =
 864                     odr.create_searchResponse(apdu_req, 0, 0);
 865                 Z_SearchResponse *resp = apdu_res->u.searchResponse;
 866                 *resp->resultCount = fset->hits;
 867                 *resp->numberOfRecordsReturned = number_returned;
 868                 *resp->nextResultSetPosition = next_position;
 869                 resp->records = records;
 870             }
 871         }
 872     }
 873     return apdu_res;
 874 }
 875
 876 void yf::SPARQL::Session::handle_z(mp::Package &package, Z_APDU *apdu_req)
 877 {
 878     mp::odr odr;
 879     Z_APDU *apdu_res = 0;
 880     if (apdu_req->which == Z_APDU_initRequest)
 881     {
 882         apdu_res = odr.create_initResponse(apdu_req, 0, 0);
 883         Z_InitRequest *req = apdu_req->u.initRequest;
 884         Z_InitResponse *resp = apdu_res->u.initResponse;
 885
 886         resp->implementationName = odr_strdup(odr, "sparql");
 887         if (ODR_MASK_GET(req->options, Z_Options_namedResultSets))
 888             m_support_named_result_sets = true;
 889         int i;
 890         static const int masks[] = {
 891             Z_Options_search, Z_Options_present,
 892             Z_Options_namedResultSets, -1
 893         };
 894         for (i = 0; masks[i] != -1; i++)
 895             if (ODR_MASK_GET(req->options, masks[i]))
 896                 ODR_MASK_SET(resp->options, masks[i]);
 897         static const int versions[] = {
 898             Z_ProtocolVersion_1,
 899             Z_ProtocolVersion_2,
 900             Z_ProtocolVersion_3,
 901             -1
 902         };
 903         for (i = 0; versions[i] != -1; i++)
 904             if (ODR_MASK_GET(req->protocolVersion, versions[i]))
 905                 ODR_MASK_SET(resp->protocolVersion, versions[i]);
 906             else
 907                 break;
 908         *resp->preferredMessageSize = *req->preferredMessageSize;
 909         *resp->maximumRecordSize = *req->maximumRecordSize;
 910     }
 911     else if (apdu_req->which == Z_APDU_close)
 912     {
 913         apdu_res = odr.create_close(apdu_req,
 914                                     Z_Close_finished, 0);
 915         package.session().close();
 916     }
 917     else if (apdu_req->which == Z_APDU_searchRequest)
 918     {
 919         Z_SearchRequest *req = apdu_req->u.searchRequest;
 920
 921         FrontendSets::iterator fset_it =
 922             m_frontend_sets.find(req->resultSetName);
 923         if (fset_it != m_frontend_sets.end())
 924         {
 925             // result set already exist
 926             // if replace indicator is off: we return diagnostic if
 927             // result set already exist.
 928             if (*req->replaceIndicator == 0)
 929             {
 930                 Z_APDU *apdu =
 931                     odr.create_searchResponse(
 932                         apdu_req,
 933                         YAZ_BIB1_RESULT_SET_EXISTS_AND_REPLACE_INDICATOR_OFF,
 934                         0);
 935                 package.response() = apdu;
 936             }
 937             m_frontend_sets.erase(fset_it);
 938         }
 939         if (req->query->which != Z_Query_type_1)
 940         {
 941             apdu_res = odr.create_searchResponse(
 942                 apdu_req, YAZ_BIB1_QUERY_TYPE_UNSUPP, 0);
 943         }
 944         else if (req->num_databaseNames != 1)
 945         {
 946             apdu_res = odr.create_searchResponse(
 947                 apdu_req,
 948                 YAZ_BIB1_ACCESS_TO_SPECIFIED_DATABASE_DENIED, 0);
 949         }
 950         else
 951         {
 952             std::string db = req->databaseNames[0];
 953             std::list<ConfPtr>::const_iterator it;
 954             FrontendSetPtr fset(new FrontendSet);
 955
 956             m_frontend_sets.erase(req->resultSetName);
 957             fset->db = db;
 958             if ( db != "info" )
 959             {
 960                 it = m_sparql->db_conf.begin();
 961                 for (; it != m_sparql->db_conf.end(); it++)
 962                     if ((*it)->schema.length() > 0
 963                         && yaz_match_glob((*it)->db.c_str(), db.c_str()))
 964                     {
 965                         mp::wrbuf addinfo_wr;
 966                         mp::wrbuf sparql_wr;
 967                         int error =
 968                             yaz_sparql_from_rpn_wrbuf((*it)->s,
 969                                                     addinfo_wr, sparql_wr,
 970                                                     req->query->u.type_1);
 971                         if (error)
 972                         {
 973                             apdu_res = odr.create_searchResponse(
 974                                 apdu_req, error,
 975                                 addinfo_wr.len() ? addinfo_wr.c_str() : 0);
 976                         }
 977                         else
 978                         {
 979                             Z_APDU *apdu_1 = search(package, apdu_req, odr,
 980                                                     sparql_wr.c_str(), *it,
 981                                                     fset);
 982                             if (!apdu_res)
 983                                 apdu_res = apdu_1;
 984                         }
 985                     }
 986                 if (apdu_res == 0)
 987                 {
 988                     apdu_res = odr.create_searchResponse(
 989                         apdu_req, YAZ_BIB1_DATABASE_DOES_NOT_EXIST, db.c_str());
 990                 }
 991             }
 992             else
 993             { // The magic "explain" base
 994                 yaz_log(YLOG_LOG,"About to call explain_search");
 995                 mp::wrbuf qry;
 996                 yaz_query_to_wrbuf(qry, req->query);
 997                 apdu_res = explain_search( package, apdu_req, odr,
 998                                            qry.c_str(), fset);
 999                   // TODO - Extract at least a term from the query, and
1000                   // do some filtering by that
1001                 yaz_log(YLOG_LOG,"Returned from explain_search");
1002             }
1003         }
1004     }
1005     else if (apdu_req->which == Z_APDU_presentRequest)
1006     {
1007         Z_PresentRequest *req = apdu_req->u.presentRequest;
1008         FrontendSets::iterator fset_it =
1009             m_frontend_sets.find(req->resultSetId);
1010         if (fset_it == m_frontend_sets.end())
1011         {
1012             apdu_res =
1013                 odr.create_presentResponse(
1014                     apdu_req, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
1015                     req->resultSetId);
1016             package.response() = apdu_res;
1017             return;
1018         }
1019         int number_returned = 0;
1020         int next_position = 0;
1021         int error_code = 0;
1022         std::string addinfo;
1023         Z_ElementSetNames *esn = 0;
1024         if (req->recordComposition)
1025         {
1026             if (req->recordComposition->which == Z_RecordComp_simple)
1027                 esn = req->recordComposition->u.simple;
1028             else
1029             {
1030                 apdu_res =
1031                     odr.create_presentResponse(
1032                         apdu_req,
1033                         YAZ_BIB1_ONLY_A_SINGLE_ELEMENT_SET_NAME_SUPPORTED,
1034                         0);
1035                 package.response() = apdu_res;
1036                 return;
1037             }
1038         }
1039         Z_Records *records;
1040         if ( fset_it->second->explaindblist.size() > 0 )
1041             records = explain_fetch(
1042                 package,
1043                 fset_it->second,
1044                 odr, req->preferredRecordSyntax, esn,
1045                 *req->resultSetStartPoint, *req->numberOfRecordsRequested,
1046                 error_code, addinfo,
1047                 &number_returned,
1048                 &next_position);
1049         else
1050             records = fetch(
1051                 package,
1052                 fset_it->second,
1053                 odr, req->preferredRecordSyntax, esn,
1054                 *req->resultSetStartPoint, *req->numberOfRecordsRequested,
1055                 error_code, addinfo,
1056                 &number_returned,
1057                 &next_position);
1058         if (error_code)
1059         {
1060             apdu_res =
1061                 odr.create_presentResponse(apdu_req, error_code,
1062                                            addinfo.c_str());
1063         }
1064         else
1065         {
1066             apdu_res =
1067                 odr.create_presentResponse(apdu_req, 0, 0);
1068             Z_PresentResponse *resp = apdu_res->u.presentResponse;
1069             resp->records = records;
1070             *resp->numberOfRecordsReturned = number_returned;
1071             *resp->nextResultSetPosition = next_position;
1072         }
1073     }
1074     else
1075     {
1076         apdu_res = odr.create_close(apdu_req,
1077                                     Z_Close_protocolError,
1078                                     "sparql: unhandled APDU");
1079         package.session().close();
1080     }
1081
1082     assert(apdu_res);
1083     package.response() = apdu_res;
1084 }
1085
1086 void yf::SPARQL::process(mp::Package &package) const
1087 {
1088     Z_APDU *apdu;
1089     SessionPtr p = get_session(package, &apdu);
1090     if (p && apdu)
1091     {
1092         p->handle_z(package, apdu);
1093     }
1094     else
1095         package.move();
1096     release_session(package);
1097 }
1098
1099 static mp::filter::Base* filter_creator()
1100 {
1101     return new mp::filter::SPARQL;
1102 }
1103
1104 extern "C" {
1105     struct metaproxy_1_filter_struct metaproxy_1_filter_sparql = {
1106         0,
1107         "sparql",
1108         filter_creator
1109     };
1110 }
1111
1112
1113 /*
1114  * Local variables:
1115  * c-basic-offset: 4
1116  * c-file-style: "Stroustrup"
1117  * indent-tabs-mode: nil
1118  * End:
1119  * vim: shiftwidth=4 tabstop=8 expandtab
1120  */
1121