src/filter_sparql.cpp

   1 /* This file is part of Metaproxy.
   2    Copyright (C) Index Data
   3
   4 Metaproxy is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17 */
  18
  19 #include <metaproxy/package.hpp>
  20 #include <metaproxy/util.hpp>
  21 #include <yaz/log.h>
  22 #include <yaz/srw.h>
  23 #include <yaz/diagbib1.h>
  24 #include <yaz/match_glob.h>
  25 #include <boost/scoped_ptr.hpp>
  26 #include <boost/thread/mutex.hpp>
  27 #include <boost/thread/condition.hpp>
  28 #include <boost/algorithm/string.hpp>
  29 #include "sparql.h"
  30
  31 #include <yaz/zgdu.h>
  32
  33 namespace mp = metaproxy_1;
  34 namespace yf = mp::filter;
  35
  36 namespace metaproxy_1 {
  37     namespace filter {
  38         class SPARQL : public Base {
  39             class Session;
  40             class Rep;
  41             class Conf;
  42             class Result;
  43             class FrontendSet;
  44
  45             typedef boost::shared_ptr<Session> SessionPtr;
  46             typedef boost::shared_ptr<Conf> ConfPtr;
  47
  48             typedef boost::shared_ptr<FrontendSet> FrontendSetPtr;
  49             typedef std::map<std::string,FrontendSetPtr> FrontendSets;
  50         public:
  51             SPARQL();
  52             ~SPARQL();
  53             void process(metaproxy_1::Package & package) const;
  54             void configure(const xmlNode * ptr, bool test_only,
  55                            const char *path);
  56             SessionPtr get_session(Package &package, Z_APDU **apdu) const;
  57             void release_session(Package &package) const;
  58             boost::scoped_ptr<Rep> m_p;
  59             std::list<ConfPtr> db_conf;
  60         };
  61         class SPARQL::Conf {
  62         public:
  63             std::string db;
  64             std::string uri;
  65             std::string schema;
  66             yaz_sparql_t s;
  67             ~Conf();
  68         };
  69         class SPARQL::Rep {
  70             friend class SPARQL;
  71             boost::condition m_cond_session_ready;
  72             boost::mutex m_mutex;
  73             std::map<mp::Session,SessionPtr> m_clients;
  74         };
  75         class SPARQL::Result {
  76         public:
  77             Result();
  78             ~Result();
  79         private:
  80             friend class FrontendSet;
  81             friend class Session;
  82             ConfPtr conf;
  83             xmlDoc *doc;
  84         };
  85         class SPARQL::FrontendSet {
  86         private:
  87             friend class Session;
  88             Odr_int hits;
  89             std::string db;
  90             std::list<Result> results;
  91             std::vector<ConfPtr> explaindblist;
  92         };
  93         class SPARQL::Session {
  94         public:
  95             Session(const SPARQL *);
  96             ~Session();
  97             void handle_z(Package &package, Z_APDU *apdu);
  98             Z_APDU *search(mp::Package &package,
  99                            Z_APDU *apdu_req,
 100                            mp::odr &odr,
 101                            const char *sparql_query,
 102                            ConfPtr conf,
 103                            FrontendSetPtr fset);
 104             Z_APDU *explain_search(mp::Package &package,
 105                            Z_APDU *apdu_req,
 106                            mp::odr &odr,
 107                            const char *sparql_query,
 108                            FrontendSetPtr fset);
 109             int invoke_sparql(mp::Package &package,
 110                               const char *sparql_query,
 111                               ConfPtr conf,
 112                               WRBUF w);
 113             Z_Records *fetch(
 114                 Package &package,
 115                 FrontendSetPtr fset,
 116                 ODR odr, Odr_oid *preferredRecordSyntax,
 117                 Z_ElementSetNames *esn,
 118                 int start, int number, int &error_code, std::string &addinfo,
 119                 int *number_returned, int *next_position);
 120             Z_Records *explain_fetch(
 121                 Package &package,
 122                 FrontendSetPtr fset,
 123                 ODR odr, Odr_oid *preferredRecordSyntax,
 124                 Z_ElementSetNames *esn,
 125                 int start, int number, int &error_code, std::string &addinfo,
 126                 int *number_returned, int *next_position);
 127             bool m_in_use;
 128         private:
 129             bool m_support_named_result_sets;
 130             FrontendSets m_frontend_sets;
 131             const SPARQL *m_sparql;
 132         };
 133     }
 134 }
 135
 136 yf::SPARQL::Result::~Result()
 137 {
 138     if (doc)
 139         xmlFreeDoc(doc);
 140 }
 141
 142 yf::SPARQL::Result::Result()
 143 {
 144     doc = 0;
 145 }
 146
 147 yf::SPARQL::SPARQL() : m_p(new Rep)
 148 {
 149 }
 150
 151 yf::SPARQL::~SPARQL()
 152 {
 153 }
 154
 155 void yf::SPARQL::configure(const xmlNode *xmlnode, bool test_only,
 156                            const char *path)
 157 {
 158     const xmlNode *ptr = xmlnode->children;
 159     std::string uri;
 160
 161     for (; ptr; ptr = ptr->next)
 162     {
 163         if (ptr->type != XML_ELEMENT_NODE)
 164             continue;
 165         if (!strcmp((const char *) ptr->name, "defaults"))
 166         {
 167             const struct _xmlAttr *attr;
 168             for (attr = ptr->properties; attr; attr = attr->next)
 169             {
 170                 if (!strcmp((const char *) attr->name, "uri"))
 171                     uri = mp::xml::get_text(attr->children);
 172                 else
 173                     throw mp::filter::FilterException(
 174                         "Bad attribute " + std::string((const char *)
 175                                                        attr->name));
 176             }
 177         }
 178         else if (!strcmp((const char *) ptr->name, "db"))
 179         {
 180             yaz_sparql_t s = yaz_sparql_create();
 181             ConfPtr conf(new Conf);
 182             conf->s = s;
 183             conf->uri = uri;
 184
 185             const struct _xmlAttr *attr;
 186             for (attr = ptr->properties; attr; attr = attr->next)
 187             {
 188                 if (!strcmp((const char *) attr->name, "path"))
 189                     conf->db = mp::xml::get_text(attr->children);
 190                 else if (!strcmp((const char *) attr->name, "uri"))
 191                     conf->uri = mp::xml::get_text(attr->children);
 192                 else if (!strcmp((const char *) attr->name, "schema"))
 193                     conf->schema = mp::xml::get_text(attr->children);
 194                 else if (!strcmp((const char *) attr->name, "include"))
 195                 {
 196                     std::vector<std::string> dbs;
 197                     std::string db = mp::xml::get_text(attr->children);
 198                     boost::split(dbs, db, boost::is_any_of(" \t"));
 199                     size_t i;
 200                     for (i = 0; i < dbs.size(); i++)
 201                     {
 202                         if (dbs[i].length() == 0)
 203                             continue;
 204                         std::list<ConfPtr>::const_iterator it = db_conf.begin();
 205                         while (1)
 206                             if (it == db_conf.end())
 207                             {
 208                                 throw mp::filter::FilterException(
 209                                     "include db not found: " + dbs[i]);
 210                             }
 211                             else if (dbs[i].compare((*it)->db) == 0)
 212                             {
 213                                 yaz_sparql_include(s, (*it)->s);
 214                                 break;
 215                             }
 216                             else
 217                                 it++;
 218                     }
 219                 }
 220                 else
 221                     throw mp::filter::FilterException(
 222                         "Bad attribute " + std::string((const char *)
 223                                                        attr->name));
 224             }
 225             xmlNode *p = ptr->children;
 226             for (; p; p = p->next)
 227             {
 228                 if (p->type != XML_ELEMENT_NODE)
 229                     continue;
 230                 std::string name = (const char *) p->name;
 231                 const struct _xmlAttr *attr;
 232                 for (attr = p->properties; attr; attr = attr->next)
 233                 {
 234                     if (!strcmp((const char *) attr->name, "type"))
 235                     {
 236                         name.append(".");
 237                         name.append(mp::xml::get_text(attr->children));
 238                     }
 239                     else
 240                         throw mp::filter::FilterException(
 241                             "Bad attribute " + std::string((const char *)
 242                                                            attr->name));
 243                 }
 244                 std::string value = mp::xml::get_text(p);
 245                 if (yaz_sparql_add_pattern(s, name.c_str(), value.c_str()))
 246                 {
 247                     throw mp::filter::FilterException(
 248                         "Bad SPARQL config " + name);
 249                 }
 250             }
 251             if (!conf->uri.length())
 252             {
 253                 throw mp::filter::FilterException("Missing uri");
 254             }
 255             if (!conf->db.length())
 256             {
 257                 throw mp::filter::FilterException("Missing path");
 258             }
 259             db_conf.push_back(conf);
 260         }
 261         else
 262         {
 263             throw mp::filter::FilterException
 264                 ("Bad element "
 265                  + std::string((const char *) ptr->name)
 266                  + " in sparql filter");
 267         }
 268     }
 269 }
 270
 271 yf::SPARQL::Conf::~Conf()
 272 {
 273     yaz_sparql_destroy(s);
 274 }
 275
 276 yf::SPARQL::Session::Session(const SPARQL *sparql) :
 277     m_in_use(true),
 278     m_support_named_result_sets(false),
 279     m_sparql(sparql)
 280 {
 281 }
 282
 283 yf::SPARQL::Session::~Session()
 284 {
 285 }
 286
 287 yf::SPARQL::SessionPtr yf::SPARQL::get_session(Package & package,
 288                                                Z_APDU **apdu) const
 289 {
 290     SessionPtr ptr0;
 291
 292     Z_GDU *gdu = package.request().get();
 293
 294     boost::mutex::scoped_lock lock(m_p->m_mutex);
 295
 296     std::map<mp::Session,SPARQL::SessionPtr>::iterator it;
 297
 298     if (gdu && gdu->which == Z_GDU_Z3950)
 299         *apdu = gdu->u.z3950;
 300     else
 301         *apdu = 0;
 302
 303     while (true)
 304     {
 305         it = m_p->m_clients.find(package.session());
 306         if (it == m_p->m_clients.end())
 307             break;
 308         if (!it->second->m_in_use)
 309         {
 310             it->second->m_in_use = true;
 311             return it->second;
 312         }
 313         m_p->m_cond_session_ready.wait(lock);
 314     }
 315     if (!*apdu)
 316         return ptr0;
 317
 318     // new Z39.50 session ..
 319     SessionPtr p(new Session(this));
 320     m_p->m_clients[package.session()] = p;
 321     return p;
 322 }
 323
 324 void yf::SPARQL::release_session(Package &package) const
 325 {
 326     boost::mutex::scoped_lock lock(m_p->m_mutex);
 327     std::map<mp::Session,SessionPtr>::iterator it;
 328
 329     it = m_p->m_clients.find(package.session());
 330     if (it != m_p->m_clients.end())
 331     {
 332         it->second->m_in_use = false;
 333
 334         if (package.session().is_closed())
 335             m_p->m_clients.erase(it);
 336         m_p->m_cond_session_ready.notify_all();
 337     }
 338 }
 339
 340 static bool get_result(xmlDoc *doc, Odr_int *sz, Odr_int pos, xmlDoc **ndoc)
 341 {
 342     xmlNode *ptr = xmlDocGetRootElement(doc);
 343     xmlNode *q0;
 344     Odr_int cur = 0;
 345
 346     if (ndoc)
 347         *ndoc = xmlNewDoc(BAD_CAST "1.0");
 348
 349     if (ptr->type == XML_ELEMENT_NODE &&
 350         !strcmp((const char *) ptr->name, "RDF"))
 351     {
 352         if (ndoc)
 353         {
 354             q0 = xmlCopyNode(ptr, 2);
 355             xmlDocSetRootElement(*ndoc, q0);
 356         }
 357         ptr = ptr->children;
 358
 359         while (ptr && ptr->type != XML_ELEMENT_NODE)
 360             ptr = ptr->next;
 361         if (ptr && ptr->type == XML_ELEMENT_NODE &&
 362             !strcmp((const char *) ptr->name, "Description"))
 363         {
 364             xmlNode *p = ptr->children;
 365
 366             while (p && p->type != XML_ELEMENT_NODE)
 367                 p = p->next;
 368             if (p && p->type == XML_ELEMENT_NODE &&
 369                 !strcmp((const char *) p->name, "type"))
 370             { /* SELECT RESULT */
 371                 for (ptr = ptr->children; ptr; ptr = ptr->next)
 372                     if (ptr->type == XML_ELEMENT_NODE &&
 373                         !strcmp((const char *) ptr->name, "solution"))
 374                     {
 375                         if (cur++ == pos)
 376                         {
 377                             if (ndoc)
 378                             {
 379                                 xmlNode *q1 = xmlCopyNode(ptr, 1);
 380                                 xmlAddChild(q0, q1);
 381                             }
 382                             break;
 383                         }
 384                     }
 385             }
 386             else
 387             {   /* CONSTRUCT result */
 388                 for (; ptr; ptr = ptr->next)
 389                     if (ptr->type == XML_ELEMENT_NODE &&
 390                         !strcmp((const char *) ptr->name, "Description"))
 391                     {
 392                         if (cur++ == pos)
 393                         {
 394                             if (ndoc)
 395                             {
 396                                 xmlNode *q1 = xmlCopyNode(ptr, 1);
 397                                 xmlAddChild(q0, q1);
 398                             }
 399                             return true;
 400                         }
 401                     }
 402             }
 403         }
 404     }
 405     else
 406     {
 407         for (; ptr; ptr = ptr->next)
 408             if (ptr->type == XML_ELEMENT_NODE &&
 409                 !strcmp((const char *) ptr->name, "sparql"))
 410                 break;
 411         if (ptr)
 412         {
 413             if (ndoc)
 414             {
 415                 q0 = xmlCopyNode(ptr, 2);
 416                 xmlDocSetRootElement(*ndoc, q0);
 417             }
 418             for (ptr = ptr->children; ptr; ptr = ptr->next)
 419                 if (ptr->type == XML_ELEMENT_NODE &&
 420                     !strcmp((const char *) ptr->name, "results"))
 421                     break;
 422         }
 423         if (ptr)
 424         {
 425             xmlNode *q1 = 0;
 426             if (ndoc)
 427             {
 428                 q1 = xmlCopyNode(ptr, 0);
 429                 xmlAddChild(q0, q1);
 430             }
 431             for (ptr = ptr->children; ptr; ptr = ptr->next)
 432                 if (ptr->type == XML_ELEMENT_NODE &&
 433                     !strcmp((const char *) ptr->name, "result"))
 434                 {
 435                     if (cur++ == pos)
 436                     {
 437                         if (ndoc)
 438                         {
 439                             xmlNode *q2 = xmlCopyNode(ptr, 1);
 440                             xmlAddChild(q1, q2);
 441                         }
 442                         return true;
 443                     }
 444                 }
 445         }
 446     }
 447     if (sz)
 448         *sz = cur;
 449     return false;
 450 }
 451
 452 Z_Records *yf::SPARQL::Session::fetch(
 453     Package &package,
 454     FrontendSetPtr fset,
 455     ODR odr, Odr_oid *preferredRecordSyntax,
 456     Z_ElementSetNames *esn,
 457     int start, int number, int &error_code, std::string &addinfo,
 458     int *number_returned, int *next_position)
 459 {
 460     Z_Records *rec = (Z_Records *) odr_malloc(odr, sizeof(Z_Records));
 461     std::list<Result>::iterator it = fset->results.begin();
 462     const char *schema = 0;
 463     bool uri_lookup = false;
 464     bool fetch_logged = false;
 465     if (esn && esn->which == Z_ElementSetNames_generic)
 466         schema = esn->u.generic;
 467
 468     for (; it != fset->results.end(); it++)
 469     {
 470         if (yaz_sparql_lookup_schema(it->conf->s, schema))
 471         {
 472             uri_lookup = true;
 473             break;
 474         }
 475         if (!schema || !strcmp(esn->u.generic, it->conf->schema.c_str()))
 476             break;
 477     }
 478     if (it == fset->results.end())
 479     {
 480         rec->which = Z_Records_NSD;
 481         rec->u.nonSurrogateDiagnostic =
 482             zget_DefaultDiagFormat(
 483                 odr,
 484                 YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_,
 485                 schema);
 486         return rec;
 487     }
 488     rec->which = Z_Records_DBOSD;
 489     rec->u.databaseOrSurDiagnostics = (Z_NamePlusRecordList *)
 490         odr_malloc(odr, sizeof(Z_NamePlusRecordList));
 491     rec->u.databaseOrSurDiagnostics->records = (Z_NamePlusRecord **)
 492         odr_malloc(odr, sizeof(Z_NamePlusRecord *) * number);
 493     int i;
 494     for (i = 0; i < number; i++)
 495     {
 496         rec->u.databaseOrSurDiagnostics->records[i] = (Z_NamePlusRecord *)
 497             odr_malloc(odr, sizeof(Z_NamePlusRecord));
 498         Z_NamePlusRecord *npr = rec->u.databaseOrSurDiagnostics->records[i];
 499         npr->databaseName = odr_strdup(odr, fset->db.c_str());
 500         npr->which = Z_NamePlusRecord_databaseRecord;
 501         xmlDoc *ndoc = 0;
 502
 503         if (!get_result(it->doc, 0, start - 1 + i, &ndoc))
 504         {
 505             if (ndoc)
 506                 xmlFreeDoc(ndoc);
 507             break;
 508         }
 509         xmlNode *ndoc_root = xmlDocGetRootElement(ndoc);
 510         if (!ndoc_root)
 511         {
 512             xmlFreeDoc(ndoc);
 513             break;
 514         }
 515         if (uri_lookup)
 516         {
 517             std::string uri;
 518             xmlNode *n = ndoc_root;
 519             while (n)
 520             {
 521                 if (n->type == XML_ELEMENT_NODE)
 522                 {
 523                     if (!strcmp((const char *) n->name, "uri") ||
 524                         !strcmp((const char *) n->name, "bnode") )
 525                     {
 526                         uri = mp::xml::get_text(n->children);
 527
 528                     }
 529                     n = n->children;
 530                 }
 531                 else
 532                     n = n->next;
 533             }
 534             if (!uri.length())
 535             {
 536                 rec->which = Z_Records_NSD;
 537                 rec->u.nonSurrogateDiagnostic =
 538                     zget_DefaultDiagFormat(
 539                         odr,
 540                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS, 0);
 541                 xmlFreeDoc(ndoc);
 542                 return rec;
 543             }
 544             else
 545             {
 546                 mp::wrbuf addinfo, query, w;
 547                 int error = yaz_sparql_from_uri_wrbuf(it->conf->s,
 548                                                       addinfo, query,
 549                                                       uri.c_str(), schema);
 550                 if (!error)
 551                 {
 552                     if (!fetch_logged)
 553                     { // Log the fetch query only once
 554                         package.log("sparql", YLOG_LOG,
 555                             "fetch query: for %s \n%s",
 556                             uri.c_str(), query.c_str() );
 557                         fetch_logged = true;
 558                     }
 559                     else
 560                     {
 561                         package.log("sparql", YLOG_LOG,
 562                             "fetch uri:%s", uri.c_str() );
 563                     }
 564                     error = invoke_sparql(package, query.c_str(),
 565                                           it->conf, w);
 566                 }
 567                 if (error)
 568                 {
 569                     rec->which = Z_Records_NSD;
 570                     rec->u.nonSurrogateDiagnostic =
 571                         zget_DefaultDiagFormat(
 572                             odr,
 573                             error,
 574                             addinfo.len() ? addinfo.c_str() : 0);
 575                     xmlFreeDoc(ndoc);
 576                     return rec;
 577                 }
 578                 npr->u.databaseRecord =
 579                     z_ext_record_xml(odr, w.c_str(), w.len());
 580             }
 581         }
 582         else
 583         {
 584             xmlBufferPtr buf = xmlBufferCreate();
 585             xmlNodeDump(buf, ndoc, ndoc_root, 0, 0);
 586             yaz_log(YLOG_LOG, "record %s %.*s", uri_lookup ? "uri" : "normal",
 587                     (int) buf->use, (const char *) buf->content);
 588             npr->u.databaseRecord =
 589                 z_ext_record_xml(odr, (const char *) buf->content, buf->use);
 590             xmlBufferFree(buf);
 591         }
 592         xmlFreeDoc(ndoc);
 593     }
 594     rec->u.databaseOrSurDiagnostics->num_records = i;
 595     *number_returned = i;
 596     if (start + number > fset->hits)
 597         *next_position = 0;
 598     else
 599         *next_position = start + number;
 600     return rec;
 601 }
 602
 603 int yf::SPARQL::Session::invoke_sparql(mp::Package &package,
 604                                        const char *sparql_query,
 605                                        ConfPtr conf,
 606                                        WRBUF w)
 607 {
 608     Package http_package(package.session(), package.origin());
 609     mp::odr odr;
 610
 611     http_package.copy_filter(package);
 612     Z_GDU *gdu = z_get_HTTP_Request_uri(odr, conf->uri.c_str(), 0, 1);
 613
 614     z_HTTP_header_add(odr, &gdu->u.HTTP_Request->headers,
 615                       "Content-Type", "application/x-www-form-urlencoded");
 616     z_HTTP_header_add(odr, &gdu->u.HTTP_Request->headers,
 617                       "Accept", "application/sparql-results+xml,"
 618                       "application/rdf+xml");
 619     const char *names[2];
 620     names[0] = "query";
 621     names[1] = 0;
 622     const char *values[1];
 623     values[0] = sparql_query;
 624     char *path = 0;
 625     yaz_array_to_uri(&path, odr, (char **) names, (char **) values);
 626
 627     gdu->u.HTTP_Request->content_buf = path;
 628     gdu->u.HTTP_Request->content_len = strlen(path);
 629
 630     yaz_log(YLOG_DEBUG, "sparql: HTTP request\n%s", sparql_query);
 631
 632     http_package.request() = gdu;
 633     http_package.move();
 634
 635     Z_GDU *gdu_resp = http_package.response().get();
 636
 637     if (!gdu_resp || gdu_resp->which != Z_GDU_HTTP_Response)
 638     {
 639         wrbuf_puts(w, "no HTTP response from backend");
 640         return YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
 641     }
 642     else if (gdu_resp->u.HTTP_Response->code != 200)
 643     {
 644         Z_HTTP_Response *resp = gdu_resp->u.HTTP_Response;
 645         wrbuf_printf(w, "sparql: HTTP error %d from backend",
 646                      resp->code);
 647         package.log("sparql", YLOG_LOG,
 648             "HTTP error %d from backend ",
 649             resp->code );
 650         package.log("sparql", YLOG_LOG,
 651             "%.*s" , resp->content_len, resp->content_buf );
 652         return YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
 653     }
 654     Z_HTTP_Response *resp = gdu_resp->u.HTTP_Response;
 655     wrbuf_write(w, resp->content_buf, resp->content_len);
 656     return 0;
 657 }
 658
 659 Z_Records *yf::SPARQL::Session::explain_fetch(
 660     Package &package,
 661     FrontendSetPtr fset,
 662     ODR odr, Odr_oid *preferredRecordSyntax,
 663     Z_ElementSetNames *esn,
 664     int start, int number, int &error_code, std::string &addinfo,
 665     int *number_returned, int *next_position)
 666 {
 667     Z_Records *rec = (Z_Records *) odr_malloc(odr, sizeof(Z_Records));
 668     rec->which = Z_Records_DBOSD;
 669     rec->u.databaseOrSurDiagnostics = (Z_NamePlusRecordList *)
 670         odr_malloc(odr, sizeof(Z_NamePlusRecordList));
 671     rec->u.databaseOrSurDiagnostics->records = (Z_NamePlusRecord **)
 672         odr_malloc(odr, sizeof(Z_NamePlusRecord *) * number);
 673     int i;
 674     for (i = 0; i < number; i++)
 675     {
 676         unsigned int idx = start + i - 1;
 677         if ( idx >= fset->explaindblist.size() )
 678             break;
 679         ConfPtr cp = fset->explaindblist[idx];
 680         package.log("sparql", YLOG_LOG, "fetch explain %d:%s", idx, cp->db.c_str() );
 681         mp::wrbuf w;
 682         wrbuf_puts(w,"<explain xmlns=\"http://explain.z3950.org/dtd/2.0/\">\n");
 683         wrbuf_puts(w,"  <databaseInfo>\n");
 684         wrbuf_puts(w,"    <title>");
 685         wrbuf_xmlputs(w, cp->db.c_str());
 686         wrbuf_puts(w,"</title>\n");
 687         wrbuf_puts(w,"  </databaseInfo>\n");
 688         yaz_sparql_explain_indexes( cp->s, w, 2);
 689         wrbuf_puts(w,"</explain>\n");
 690
 691         rec->u.databaseOrSurDiagnostics->records[i] = (Z_NamePlusRecord *)
 692             odr_malloc(odr, sizeof(Z_NamePlusRecord));
 693         Z_NamePlusRecord *npr = rec->u.databaseOrSurDiagnostics->records[i];
 694         npr->databaseName = odr_strdup(odr, fset->db.c_str());
 695         npr->which = Z_NamePlusRecord_databaseRecord;
 696         npr->u.databaseRecord =
 697             z_ext_record_xml(odr, w.buf(), w.len() );
 698     }
 699     rec->u.databaseOrSurDiagnostics->num_records = i;
 700     *number_returned = i;
 701     if (start + number > (int)fset->explaindblist.size())
 702         *next_position = 0;
 703     else
 704         *next_position = start + number;
 705     return rec;
 706 }
 707
 708
 709
 710 Z_APDU *yf::SPARQL::Session::explain_search(mp::Package &package,
 711                            Z_APDU *apdu_req,
 712                            mp::odr &odr,
 713                            const char *sparql_query,
 714                            FrontendSetPtr fset)
 715 {
 716     Z_SearchRequest *req = apdu_req->u.searchRequest;
 717     Z_APDU *apdu_res = 0;
 718     //mp::wrbuf w;
 719
 720     package.log("sparql", YLOG_LOG, "Explain search" );
 721     int numbases = 0;
 722     //std::list<std::string> dblist;
 723     std::list<ConfPtr>::const_iterator it = m_sparql->db_conf.begin();
 724     m_frontend_sets[req->resultSetName] = fset;
 725     fset->explaindblist.clear();
 726     fset->explaindblist.reserve(m_sparql->db_conf.size());
 727
 728     for (; it != m_sparql->db_conf.end(); it++)
 729         if ((*it)->schema.length() > 0 )  // searchable db
 730         {
 731             numbases++;
 732             package.log("sparql", YLOG_LOG, "Explain %d: '%s'",
 733                         numbases, (*it)->db.c_str() );
 734             fset->explaindblist.push_back(*it);
 735         }
 736     int number_returned = 0;
 737     int next_position = 0;
 738     Z_Records *records = 0;
 739     int error_code = 0;
 740     std::string addinfo;
 741
 742     Odr_int number = 0;
 743     const char *element_set_name = 0;
 744     mp::util::piggyback_sr(req, numbases, number, &element_set_name);
 745     if (number)
 746     {
 747         Z_ElementSetNames *esn;
 748
 749         if (number > *req->smallSetUpperBound)
 750             esn = req->mediumSetElementSetNames;
 751         else
 752             esn = req->smallSetElementSetNames;
 753         records = explain_fetch(package, fset,
 754                         odr, req->preferredRecordSyntax, esn,
 755                         1, number,
 756                         error_code, addinfo,
 757                         &number_returned,
 758                         &next_position);
 759     }
 760
 761     if (error_code)
 762     {
 763         apdu_res = odr.create_searchResponse(
 764                 apdu_req, error_code, addinfo.c_str());
 765     }
 766     else
 767     {
 768         apdu_res = odr.create_searchResponse(apdu_req, 0, 0);
 769         Z_SearchResponse *resp = apdu_res->u.searchResponse;
 770         *resp->resultCount = numbases;
 771         *resp->numberOfRecordsReturned = number_returned;
 772         *resp->nextResultSetPosition = next_position;
 773         resp->records = records;
 774     }
 775
 776     return apdu_res;
 777 }
 778
 779 Z_APDU *yf::SPARQL::Session::search(mp::Package &package,
 780                                     Z_APDU *apdu_req,
 781                                     mp::odr &odr,
 782                                     const char *sparql_query,
 783                                     ConfPtr conf, FrontendSetPtr fset)
 784 {
 785     Z_SearchRequest *req = apdu_req->u.searchRequest;
 786     Z_APDU *apdu_res = 0;
 787     mp::wrbuf w;
 788
 789     package.log("sparql", YLOG_LOG,
 790         "search query:\n%s", sparql_query );
 791
 792     int error = invoke_sparql(package, sparql_query, conf, w);
 793     if (error)
 794     {
 795         apdu_res = odr.create_searchResponse(apdu_req, error,
 796                                              w.len() ?
 797                                              w.c_str() : 0);
 798     }
 799     else
 800     {
 801         xmlDocPtr doc = xmlParseMemory(w.c_str(), w.len());
 802         if (!doc)
 803         {
 804             apdu_res = odr.create_searchResponse(
 805                 apdu_req,
 806                 YAZ_BIB1_TEMPORARY_SYSTEM_ERROR,
 807                 "invalid XML from backendbackend");
 808         }
 809         else
 810         {
 811             Result result;
 812             Z_Records *records = 0;
 813             int number_returned = 0;
 814             int next_position = 0;
 815             int error_code = 0;
 816             std::string addinfo;
 817
 818             result.doc = doc;
 819             result.conf = conf;
 820             fset->results.push_back(result);
 821             yaz_log(YLOG_DEBUG, "saving sparql result xmldoc=%p", doc);
 822
 823             get_result(result.doc, &fset->hits, -1, 0);
 824             m_frontend_sets[req->resultSetName] = fset;
 825
 826             result.doc = 0;
 827
 828             Odr_int number = 0;
 829             const char *element_set_name = 0;
 830             mp::util::piggyback_sr(req, fset->hits, number, &element_set_name);
 831             if (number)
 832             {
 833                 Z_ElementSetNames *esn;
 834
 835                 if (number > *req->smallSetUpperBound)
 836                     esn = req->mediumSetElementSetNames;
 837                 else
 838                     esn = req->smallSetElementSetNames;
 839                 records = fetch(package, fset,
 840                                 odr, req->preferredRecordSyntax, esn,
 841                                 1, number,
 842                                 error_code, addinfo,
 843                                 &number_returned,
 844                                 &next_position);
 845             }
 846             if (error_code)
 847             {
 848                 apdu_res =
 849                     odr.create_searchResponse(
 850                         apdu_req, error_code, addinfo.c_str());
 851             }
 852             else
 853             {
 854                 apdu_res =
 855                     odr.create_searchResponse(apdu_req, 0, 0);
 856                 Z_SearchResponse *resp = apdu_res->u.searchResponse;
 857                 *resp->resultCount = fset->hits;
 858                 *resp->numberOfRecordsReturned = number_returned;
 859                 *resp->nextResultSetPosition = next_position;
 860                 resp->records = records;
 861             }
 862         }
 863     }
 864     return apdu_res;
 865 }
 866
 867 void yf::SPARQL::Session::handle_z(mp::Package &package, Z_APDU *apdu_req)
 868 {
 869     mp::odr odr;
 870     Z_APDU *apdu_res = 0;
 871     if (apdu_req->which == Z_APDU_initRequest)
 872     {
 873         apdu_res = odr.create_initResponse(apdu_req, 0, 0);
 874         Z_InitRequest *req = apdu_req->u.initRequest;
 875         Z_InitResponse *resp = apdu_res->u.initResponse;
 876
 877         resp->implementationName = odr_strdup(odr, "sparql");
 878         if (ODR_MASK_GET(req->options, Z_Options_namedResultSets))
 879             m_support_named_result_sets = true;
 880         int i;
 881         static const int masks[] = {
 882             Z_Options_search, Z_Options_present,
 883             Z_Options_namedResultSets, -1
 884         };
 885         for (i = 0; masks[i] != -1; i++)
 886             if (ODR_MASK_GET(req->options, masks[i]))
 887                 ODR_MASK_SET(resp->options, masks[i]);
 888         static const int versions[] = {
 889             Z_ProtocolVersion_1,
 890             Z_ProtocolVersion_2,
 891             Z_ProtocolVersion_3,
 892             -1
 893         };
 894         for (i = 0; versions[i] != -1; i++)
 895             if (ODR_MASK_GET(req->protocolVersion, versions[i]))
 896                 ODR_MASK_SET(resp->protocolVersion, versions[i]);
 897             else
 898                 break;
 899         *resp->preferredMessageSize = *req->preferredMessageSize;
 900         *resp->maximumRecordSize = *req->maximumRecordSize;
 901     }
 902     else if (apdu_req->which == Z_APDU_close)
 903     {
 904         apdu_res = odr.create_close(apdu_req,
 905                                     Z_Close_finished, 0);
 906         package.session().close();
 907     }
 908     else if (apdu_req->which == Z_APDU_searchRequest)
 909     {
 910         Z_SearchRequest *req = apdu_req->u.searchRequest;
 911
 912         FrontendSets::iterator fset_it =
 913             m_frontend_sets.find(req->resultSetName);
 914         if (fset_it != m_frontend_sets.end())
 915         {
 916             // result set already exist
 917             // if replace indicator is off: we return diagnostic if
 918             // result set already exist.
 919             if (*req->replaceIndicator == 0)
 920             {
 921                 Z_APDU *apdu =
 922                     odr.create_searchResponse(
 923                         apdu_req,
 924                         YAZ_BIB1_RESULT_SET_EXISTS_AND_REPLACE_INDICATOR_OFF,
 925                         0);
 926                 package.response() = apdu;
 927             }
 928             m_frontend_sets.erase(fset_it);
 929         }
 930         if (req->query->which != Z_Query_type_1)
 931         {
 932             apdu_res = odr.create_searchResponse(
 933                 apdu_req, YAZ_BIB1_QUERY_TYPE_UNSUPP, 0);
 934         }
 935         else if (req->num_databaseNames != 1)
 936         {
 937             apdu_res = odr.create_searchResponse(
 938                 apdu_req,
 939                 YAZ_BIB1_ACCESS_TO_SPECIFIED_DATABASE_DENIED, 0);
 940         }
 941         else
 942         {
 943             std::string db = req->databaseNames[0];
 944             std::list<ConfPtr>::const_iterator it;
 945             FrontendSetPtr fset(new FrontendSet);
 946
 947             m_frontend_sets.erase(req->resultSetName);
 948             fset->db = db;
 949             if ( db != "explain" )
 950             {
 951                 it = m_sparql->db_conf.begin();
 952                 for (; it != m_sparql->db_conf.end(); it++)
 953                     if ((*it)->schema.length() > 0
 954                         && yaz_match_glob((*it)->db.c_str(), db.c_str()))
 955                     {
 956                         mp::wrbuf addinfo_wr;
 957                         mp::wrbuf sparql_wr;
 958                         int error =
 959                             yaz_sparql_from_rpn_wrbuf((*it)->s,
 960                                                     addinfo_wr, sparql_wr,
 961                                                     req->query->u.type_1);
 962                         if (error)
 963                         {
 964                             apdu_res = odr.create_searchResponse(
 965                                 apdu_req, error,
 966                                 addinfo_wr.len() ? addinfo_wr.c_str() : 0);
 967                         }
 968                         else
 969                         {
 970                             Z_APDU *apdu_1 = search(package, apdu_req, odr,
 971                                                     sparql_wr.c_str(), *it,
 972                                                     fset);
 973                             if (!apdu_res)
 974                                 apdu_res = apdu_1;
 975                         }
 976                     }
 977                 if (apdu_res == 0)
 978                 {
 979                     apdu_res = odr.create_searchResponse(
 980                         apdu_req, YAZ_BIB1_DATABASE_DOES_NOT_EXIST, db.c_str());
 981                 }
 982             }
 983             else
 984             { // The magic "explain" base
 985                 yaz_log(YLOG_LOG,"About to call explain_search");
 986                 const char *qry = "query";
 987                 apdu_res = explain_search( package, apdu_req, odr,
 988                                            qry, fset);
 989                   // TODO - Extract at least a term from the query, and
 990                   // do some filtering by that
 991                 yaz_log(YLOG_LOG,"Returned from explain_search");
 992             }
 993         }
 994     }
 995     else if (apdu_req->which == Z_APDU_presentRequest)
 996     {
 997         Z_PresentRequest *req = apdu_req->u.presentRequest;
 998         FrontendSets::iterator fset_it =
 999             m_frontend_sets.find(req->resultSetId);
1000         if (fset_it == m_frontend_sets.end())
1001         {
1002             apdu_res =
1003                 odr.create_presentResponse(
1004                     apdu_req, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
1005                     req->resultSetId);
1006             package.response() = apdu_res;
1007             return;
1008         }
1009         int number_returned = 0;
1010         int next_position = 0;
1011         int error_code = 0;
1012         std::string addinfo;
1013         Z_ElementSetNames *esn = 0;
1014         if (req->recordComposition)
1015         {
1016             if (req->recordComposition->which == Z_RecordComp_simple)
1017                 esn = req->recordComposition->u.simple;
1018             else
1019             {
1020                 apdu_res =
1021                     odr.create_presentResponse(
1022                         apdu_req,
1023                         YAZ_BIB1_ONLY_A_SINGLE_ELEMENT_SET_NAME_SUPPORTED,
1024                         0);
1025                 package.response() = apdu_res;
1026                 return;
1027             }
1028         }
1029         Z_Records *records;
1030         if ( fset_it->second->explaindblist.size() > 0 )
1031             records = explain_fetch(
1032                 package,
1033                 fset_it->second,
1034                 odr, req->preferredRecordSyntax, esn,
1035                 *req->resultSetStartPoint, *req->numberOfRecordsRequested,
1036                 error_code, addinfo,
1037                 &number_returned,
1038                 &next_position);
1039         else
1040             records = fetch(
1041                 package,
1042                 fset_it->second,
1043                 odr, req->preferredRecordSyntax, esn,
1044                 *req->resultSetStartPoint, *req->numberOfRecordsRequested,
1045                 error_code, addinfo,
1046                 &number_returned,
1047                 &next_position);
1048         if (error_code)
1049         {
1050             apdu_res =
1051                 odr.create_presentResponse(apdu_req, error_code,
1052                                            addinfo.c_str());
1053         }
1054         else
1055         {
1056             apdu_res =
1057                 odr.create_presentResponse(apdu_req, 0, 0);
1058             Z_PresentResponse *resp = apdu_res->u.presentResponse;
1059             resp->records = records;
1060             *resp->numberOfRecordsReturned = number_returned;
1061             *resp->nextResultSetPosition = next_position;
1062         }
1063     }
1064     else
1065     {
1066         apdu_res = odr.create_close(apdu_req,
1067                                     Z_Close_protocolError,
1068                                     "sparql: unhandled APDU");
1069         package.session().close();
1070     }
1071
1072     assert(apdu_res);
1073     package.response() = apdu_res;
1074 }
1075
1076 void yf::SPARQL::process(mp::Package &package) const
1077 {
1078     Z_APDU *apdu;
1079     SessionPtr p = get_session(package, &apdu);
1080     if (p && apdu)
1081     {
1082         p->handle_z(package, apdu);
1083     }
1084     else
1085         package.move();
1086     release_session(package);
1087 }
1088
1089 static mp::filter::Base* filter_creator()
1090 {
1091     return new mp::filter::SPARQL;
1092 }
1093
1094 extern "C" {
1095     struct metaproxy_1_filter_struct metaproxy_1_filter_sparql = {
1096         0,
1097         "sparql",
1098         filter_creator
1099     };
1100 }
1101
1102
1103 /*
1104  * Local variables:
1105  * c-basic-offset: 4
1106  * c-file-style: "Stroustrup"
1107  * indent-tabs-mode: nil
1108  * End:
1109  * vim: shiftwidth=4 tabstop=8 expandtab
1110  */
1111