src/filter_sparql.cpp

   1 /* This file is part of Metaproxy.
   2    Copyright (C) Index Data
   3
   4 Metaproxy is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17 */
  18
  19 #include <metaproxy/package.hpp>
  20 #include <metaproxy/util.hpp>
  21 #include <yaz/log.h>
  22 #include <yaz/srw.h>
  23 #include <yaz/diagbib1.h>
  24 #include <yaz/match_glob.h>
  25 #include <boost/scoped_ptr.hpp>
  26 #include <boost/thread/mutex.hpp>
  27 #include <boost/thread/condition.hpp>
  28 #include <boost/algorithm/string.hpp>
  29 #include "sparql.h"
  30
  31 #include <yaz/zgdu.h>
  32
  33 namespace mp = metaproxy_1;
  34 namespace yf = mp::filter;
  35
  36 namespace metaproxy_1 {
  37     namespace filter {
  38         class SPARQL : public Base {
  39             class Session;
  40             class Rep;
  41             class Conf;
  42             class Result;
  43             class FrontendSet;
  44
  45             typedef boost::shared_ptr<Session> SessionPtr;
  46             typedef boost::shared_ptr<Conf> ConfPtr;
  47
  48             typedef boost::shared_ptr<FrontendSet> FrontendSetPtr;
  49             typedef std::map<std::string,FrontendSetPtr> FrontendSets;
  50         public:
  51             SPARQL();
  52             ~SPARQL();
  53             void process(metaproxy_1::Package & package) const;
  54             void configure(const xmlNode * ptr, bool test_only,
  55                            const char *path);
  56             SessionPtr get_session(Package &package, Z_APDU **apdu) const;
  57             void release_session(Package &package) const;
  58             boost::scoped_ptr<Rep> m_p;
  59             std::list<ConfPtr> db_conf;
  60         };
  61         class SPARQL::Conf {
  62         public:
  63             std::string db;
  64             std::string uri;
  65             std::string schema;
  66             yaz_sparql_t s;
  67             ~Conf();
  68         };
  69         class SPARQL::Rep {
  70             friend class SPARQL;
  71             boost::condition m_cond_session_ready;
  72             boost::mutex m_mutex;
  73             std::map<mp::Session,SessionPtr> m_clients;
  74         };
  75         class SPARQL::Result {
  76         public:
  77             Result();
  78             ~Result();
  79         private:
  80             friend class FrontendSet;
  81             friend class Session;
  82             ConfPtr conf;
  83             xmlDoc *doc;
  84         };
  85         class SPARQL::FrontendSet {
  86         private:
  87             friend class Session;
  88             Odr_int hits;
  89             std::string db;
  90             std::list<Result> results;
  91             std::vector<ConfPtr> explaindblist;
  92         };
  93         class SPARQL::Session {
  94         public:
  95             Session(const SPARQL *);
  96             ~Session();
  97             void handle_z(Package &package, Z_APDU *apdu);
  98             Z_APDU *search(mp::Package &package,
  99                            Z_APDU *apdu_req,
 100                            mp::odr &odr,
 101                            const char *sparql_query,
 102                            ConfPtr conf,
 103                            FrontendSetPtr fset);
 104             Z_APDU *explain_search(mp::Package &package,
 105                            Z_APDU *apdu_req,
 106                            mp::odr &odr,
 107                            const char *sparql_query,
 108                            FrontendSetPtr fset);
 109             int invoke_sparql(mp::Package &package,
 110                               const char *sparql_query,
 111                               ConfPtr conf,
 112                               WRBUF w);
 113             Z_Records *fetch(
 114                 Package &package,
 115                 FrontendSetPtr fset,
 116                 ODR odr, Odr_oid *preferredRecordSyntax,
 117                 Z_ElementSetNames *esn,
 118                 int start, int number, int &error_code, std::string &addinfo,
 119                 int *number_returned, int *next_position);
 120             Z_Records *explain_fetch(
 121                 Package &package,
 122                 FrontendSetPtr fset,
 123                 ODR odr, Odr_oid *preferredRecordSyntax,
 124                 Z_ElementSetNames *esn,
 125                 int start, int number, int &error_code, std::string &addinfo,
 126                 int *number_returned, int *next_position);
 127             bool m_in_use;
 128         private:
 129             bool m_support_named_result_sets;
 130             FrontendSets m_frontend_sets;
 131             const SPARQL *m_sparql;
 132         };
 133     }
 134 }
 135
 136 yf::SPARQL::Result::~Result()
 137 {
 138     if (doc)
 139         xmlFreeDoc(doc);
 140 }
 141
 142 yf::SPARQL::Result::Result()
 143 {
 144     doc = 0;
 145 }
 146
 147 yf::SPARQL::SPARQL() : m_p(new Rep)
 148 {
 149 }
 150
 151 yf::SPARQL::~SPARQL()
 152 {
 153 }
 154
 155 void yf::SPARQL::configure(const xmlNode *xmlnode, bool test_only,
 156                            const char *path)
 157 {
 158     const xmlNode *ptr = xmlnode->children;
 159     std::string uri;
 160
 161     for (; ptr; ptr = ptr->next)
 162     {
 163         if (ptr->type != XML_ELEMENT_NODE)
 164             continue;
 165         if (!strcmp((const char *) ptr->name, "defaults"))
 166         {
 167             const struct _xmlAttr *attr;
 168             for (attr = ptr->properties; attr; attr = attr->next)
 169             {
 170                 if (!strcmp((const char *) attr->name, "uri"))
 171                     uri = mp::xml::get_text(attr->children);
 172                 else
 173                     throw mp::filter::FilterException(
 174                         "Bad attribute " + std::string((const char *)
 175                                                        attr->name));
 176             }
 177         }
 178         else if (!strcmp((const char *) ptr->name, "db"))
 179         {
 180             yaz_sparql_t s = yaz_sparql_create();
 181             ConfPtr conf(new Conf);
 182             conf->s = s;
 183             conf->uri = uri;
 184
 185             const struct _xmlAttr *attr;
 186             for (attr = ptr->properties; attr; attr = attr->next)
 187             {
 188                 if (!strcmp((const char *) attr->name, "path"))
 189                     conf->db = mp::xml::get_text(attr->children);
 190                 else if (!strcmp((const char *) attr->name, "uri"))
 191                     conf->uri = mp::xml::get_text(attr->children);
 192                 else if (!strcmp((const char *) attr->name, "schema"))
 193                     conf->schema = mp::xml::get_text(attr->children);
 194                 else if (!strcmp((const char *) attr->name, "include"))
 195                 {
 196                     std::vector<std::string> dbs;
 197                     std::string db = mp::xml::get_text(attr->children);
 198                     boost::split(dbs, db, boost::is_any_of(" \t"));
 199                     size_t i;
 200                     for (i = 0; i < dbs.size(); i++)
 201                     {
 202                         if (dbs[i].length() == 0)
 203                             continue;
 204                         std::list<ConfPtr>::const_iterator it = db_conf.begin();
 205                         while (1)
 206                             if (it == db_conf.end())
 207                             {
 208                                 throw mp::filter::FilterException(
 209                                     "include db not found: " + dbs[i]);
 210                             }
 211                             else if (dbs[i].compare((*it)->db) == 0)
 212                             {
 213                                 yaz_sparql_include(s, (*it)->s);
 214                                 break;
 215                             }
 216                             else
 217                                 it++;
 218                     }
 219                 }
 220                 else
 221                     throw mp::filter::FilterException(
 222                         "Bad attribute " + std::string((const char *)
 223                                                        attr->name));
 224             }
 225             xmlNode *p = ptr->children;
 226             for (; p; p = p->next)
 227             {
 228                 if (p->type != XML_ELEMENT_NODE)
 229                     continue;
 230                 std::string name = (const char *) p->name;
 231                 const struct _xmlAttr *attr;
 232                 for (attr = p->properties; attr; attr = attr->next)
 233                 {
 234                     if (!strcmp((const char *) attr->name, "type"))
 235                     {
 236                         name.append(".");
 237                         name.append(mp::xml::get_text(attr->children));
 238                     }
 239                     else
 240                         throw mp::filter::FilterException(
 241                             "Bad attribute " + std::string((const char *)
 242                                                            attr->name));
 243                 }
 244                 std::string value = mp::xml::get_text(p);
 245                 if (yaz_sparql_add_pattern(s, name.c_str(), value.c_str()))
 246                 {
 247                     throw mp::filter::FilterException(
 248                         "Bad SPARQL config " + name);
 249                 }
 250             }
 251             if (!conf->uri.length())
 252             {
 253                 throw mp::filter::FilterException("Missing uri");
 254             }
 255             if (!conf->db.length())
 256             {
 257                 throw mp::filter::FilterException("Missing path");
 258             }
 259             db_conf.push_back(conf);
 260         }
 261         else
 262         {
 263             throw mp::filter::FilterException
 264                 ("Bad element "
 265                  + std::string((const char *) ptr->name)
 266                  + " in sparql filter");
 267         }
 268     }
 269 }
 270
 271 yf::SPARQL::Conf::~Conf()
 272 {
 273     yaz_sparql_destroy(s);
 274 }
 275
 276 yf::SPARQL::Session::Session(const SPARQL *sparql) :
 277     m_in_use(true),
 278     m_support_named_result_sets(false),
 279     m_sparql(sparql)
 280 {
 281 }
 282
 283 yf::SPARQL::Session::~Session()
 284 {
 285 }
 286
 287 yf::SPARQL::SessionPtr yf::SPARQL::get_session(Package & package,
 288                                                Z_APDU **apdu) const
 289 {
 290     SessionPtr ptr0;
 291
 292     Z_GDU *gdu = package.request().get();
 293
 294     boost::mutex::scoped_lock lock(m_p->m_mutex);
 295
 296     std::map<mp::Session,SPARQL::SessionPtr>::iterator it;
 297
 298     if (gdu && gdu->which == Z_GDU_Z3950)
 299         *apdu = gdu->u.z3950;
 300     else
 301         *apdu = 0;
 302
 303     while (true)
 304     {
 305         it = m_p->m_clients.find(package.session());
 306         if (it == m_p->m_clients.end())
 307             break;
 308         if (!it->second->m_in_use)
 309         {
 310             it->second->m_in_use = true;
 311             return it->second;
 312         }
 313         m_p->m_cond_session_ready.wait(lock);
 314     }
 315     if (!*apdu)
 316         return ptr0;
 317
 318     // new Z39.50 session ..
 319     SessionPtr p(new Session(this));
 320     m_p->m_clients[package.session()] = p;
 321     return p;
 322 }
 323
 324 void yf::SPARQL::release_session(Package &package) const
 325 {
 326     boost::mutex::scoped_lock lock(m_p->m_mutex);
 327     std::map<mp::Session,SessionPtr>::iterator it;
 328
 329     it = m_p->m_clients.find(package.session());
 330     if (it != m_p->m_clients.end())
 331     {
 332         it->second->m_in_use = false;
 333
 334         if (package.session().is_closed())
 335             m_p->m_clients.erase(it);
 336         m_p->m_cond_session_ready.notify_all();
 337     }
 338 }
 339
 340 static bool get_result(xmlDoc *doc, Odr_int *sz, Odr_int pos, xmlDoc **ndoc)
 341 {
 342     xmlNode *ptr = xmlDocGetRootElement(doc);
 343     xmlNode *q0;
 344     Odr_int cur = 0;
 345
 346     if (ndoc)
 347         *ndoc = xmlNewDoc(BAD_CAST "1.0");
 348
 349     if (ptr->type == XML_ELEMENT_NODE &&
 350         !strcmp((const char *) ptr->name, "RDF"))
 351     {
 352         if (ndoc)
 353         {
 354             q0 = xmlCopyNode(ptr, 2);
 355             xmlDocSetRootElement(*ndoc, q0);
 356         }
 357         ptr = ptr->children;
 358
 359         while (ptr && ptr->type != XML_ELEMENT_NODE)
 360             ptr = ptr->next;
 361         if (ptr && ptr->type == XML_ELEMENT_NODE &&
 362             !strcmp((const char *) ptr->name, "Description"))
 363         {
 364             xmlNode *p = ptr->children;
 365
 366             while (p && p->type != XML_ELEMENT_NODE)
 367                 p = p->next;
 368             if (p && p->type == XML_ELEMENT_NODE &&
 369                 !strcmp((const char *) p->name, "type"))
 370             { /* SELECT RESULT */
 371                 for (ptr = ptr->children; ptr; ptr = ptr->next)
 372                     if (ptr->type == XML_ELEMENT_NODE &&
 373                         !strcmp((const char *) ptr->name, "solution"))
 374                     {
 375                         if (cur++ == pos)
 376                         {
 377                             if (ndoc)
 378                             {
 379                                 xmlNode *q1 = xmlCopyNode(ptr, 1);
 380                                 xmlAddChild(q0, q1);
 381                             }
 382                             break;
 383                         }
 384                     }
 385             }
 386             else
 387             {   /* CONSTRUCT result */
 388                 for (; ptr; ptr = ptr->next)
 389                     if (ptr->type == XML_ELEMENT_NODE &&
 390                         !strcmp((const char *) ptr->name, "Description"))
 391                     {
 392                         if (cur++ == pos)
 393                         {
 394                             if (ndoc)
 395                             {
 396                                 xmlNode *q1 = xmlCopyNode(ptr, 1);
 397                                 xmlAddChild(q0, q1);
 398                             }
 399                             return true;
 400                         }
 401                     }
 402             }
 403         }
 404     }
 405     else
 406     {
 407         for (; ptr; ptr = ptr->next)
 408             if (ptr->type == XML_ELEMENT_NODE &&
 409                 !strcmp((const char *) ptr->name, "sparql"))
 410                 break;
 411         if (ptr)
 412         {
 413             if (ndoc)
 414             {
 415                 q0 = xmlCopyNode(ptr, 2);
 416                 xmlDocSetRootElement(*ndoc, q0);
 417             }
 418             for (ptr = ptr->children; ptr; ptr = ptr->next)
 419                 if (ptr->type == XML_ELEMENT_NODE &&
 420                     !strcmp((const char *) ptr->name, "results"))
 421                     break;
 422         }
 423         if (ptr)
 424         {
 425             xmlNode *q1 = 0;
 426             if (ndoc)
 427             {
 428                 q1 = xmlCopyNode(ptr, 0);
 429                 xmlAddChild(q0, q1);
 430             }
 431             for (ptr = ptr->children; ptr; ptr = ptr->next)
 432                 if (ptr->type == XML_ELEMENT_NODE &&
 433                     !strcmp((const char *) ptr->name, "result"))
 434                 {
 435                     if (cur++ == pos)
 436                     {
 437                         if (ndoc)
 438                         {
 439                             xmlNode *q2 = xmlCopyNode(ptr, 1);
 440                             xmlAddChild(q1, q2);
 441                         }
 442                         return true;
 443                     }
 444                 }
 445         }
 446     }
 447     if (sz)
 448         *sz = cur;
 449     return false;
 450 }
 451
 452 Z_Records *yf::SPARQL::Session::fetch(
 453     Package &package,
 454     FrontendSetPtr fset,
 455     ODR odr, Odr_oid *preferredRecordSyntax,
 456     Z_ElementSetNames *esn,
 457     int start, int number, int &error_code, std::string &addinfo,
 458     int *number_returned, int *next_position)
 459 {
 460     Z_Records *rec = (Z_Records *) odr_malloc(odr, sizeof(Z_Records));
 461     std::list<Result>::iterator it = fset->results.begin();
 462     const char *schema = 0;
 463     bool uri_lookup = false;
 464     bool fetch_logged = false;
 465     if (esn && esn->which == Z_ElementSetNames_generic)
 466         schema = esn->u.generic;
 467
 468     for (; it != fset->results.end(); it++)
 469     {
 470         if (yaz_sparql_lookup_schema(it->conf->s, schema))
 471         {
 472             uri_lookup = true;
 473             break;
 474         }
 475         if (!schema || !strcmp(esn->u.generic, it->conf->schema.c_str()))
 476             break;
 477     }
 478     if (it == fset->results.end())
 479     {
 480         rec->which = Z_Records_NSD;
 481         rec->u.nonSurrogateDiagnostic =
 482             zget_DefaultDiagFormat(
 483                 odr,
 484                 YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_,
 485                 schema);
 486         return rec;
 487     }
 488     rec->which = Z_Records_DBOSD;
 489     rec->u.databaseOrSurDiagnostics = (Z_NamePlusRecordList *)
 490         odr_malloc(odr, sizeof(Z_NamePlusRecordList));
 491     rec->u.databaseOrSurDiagnostics->records = (Z_NamePlusRecord **)
 492         odr_malloc(odr, sizeof(Z_NamePlusRecord *) * number);
 493     int i;
 494     for (i = 0; i < number; i++)
 495     {
 496         rec->u.databaseOrSurDiagnostics->records[i] = (Z_NamePlusRecord *)
 497             odr_malloc(odr, sizeof(Z_NamePlusRecord));
 498         Z_NamePlusRecord *npr = rec->u.databaseOrSurDiagnostics->records[i];
 499         npr->databaseName = odr_strdup(odr, fset->db.c_str());
 500         npr->which = Z_NamePlusRecord_databaseRecord;
 501         xmlDoc *ndoc = 0;
 502
 503         if (!get_result(it->doc, 0, start - 1 + i, &ndoc))
 504         {
 505             if (ndoc)
 506                 xmlFreeDoc(ndoc);
 507             break;
 508         }
 509         xmlNode *ndoc_root = xmlDocGetRootElement(ndoc);
 510         if (!ndoc_root)
 511         {
 512             xmlFreeDoc(ndoc);
 513             break;
 514         }
 515         if (uri_lookup)
 516         {
 517             std::string uri;
 518             xmlNode *n = ndoc_root;
 519             while (n)
 520             {
 521                 if (n->type == XML_ELEMENT_NODE)
 522                 {
 523                     if (!strcmp((const char *) n->name, "uri") ||
 524                         !strcmp((const char *) n->name, "bnode") )
 525                     {
 526                         uri = mp::xml::get_text(n->children);
 527
 528                     }
 529                     n = n->children;
 530                 }
 531                 else
 532                     n = n->next;
 533             }
 534             if (!uri.length())
 535             {
 536                 rec->which = Z_Records_NSD;
 537                 rec->u.nonSurrogateDiagnostic =
 538                     zget_DefaultDiagFormat(
 539                         odr,
 540                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS, 0);
 541                 xmlFreeDoc(ndoc);
 542                 return rec;
 543             }
 544             else
 545             {
 546                 mp::wrbuf addinfo, query, w;
 547                 int error = yaz_sparql_from_uri_wrbuf(it->conf->s,
 548                                                       addinfo, query,
 549                                                       uri.c_str(), schema);
 550                 if (!error)
 551                 {
 552                     if (!fetch_logged)
 553                     { // Log the fetch query only once
 554                         package.log("sparql", YLOG_LOG,
 555                             "fetch query: for %s \n%s",
 556                             uri.c_str(), query.c_str() );
 557                         fetch_logged = true;
 558                     }
 559                     else
 560                     {
 561                         package.log("sparql", YLOG_LOG,
 562                             "fetch uri:%s", uri.c_str() );
 563                     }
 564                     error = invoke_sparql(package, query.c_str(),
 565                                           it->conf, w);
 566                 }
 567                 if (error)
 568                 {
 569                     rec->which = Z_Records_NSD;
 570                     rec->u.nonSurrogateDiagnostic =
 571                         zget_DefaultDiagFormat(
 572                             odr,
 573                             error,
 574                             addinfo.len() ? addinfo.c_str() : 0);
 575                     xmlFreeDoc(ndoc);
 576                     return rec;
 577                 }
 578                 npr->u.databaseRecord =
 579                     z_ext_record_xml(odr, w.c_str(), w.len());
 580             }
 581         }
 582         else
 583         {
 584             xmlBufferPtr buf = xmlBufferCreate();
 585             xmlNodeDump(buf, ndoc, ndoc_root, 0, 0);
 586             yaz_log(YLOG_LOG, "record %s %.*s", uri_lookup ? "uri" : "normal",
 587                     (int) buf->use, (const char *) buf->content);
 588             npr->u.databaseRecord =
 589                 z_ext_record_xml(odr, (const char *) buf->content, buf->use);
 590             xmlBufferFree(buf);
 591         }
 592         xmlFreeDoc(ndoc);
 593     }
 594     rec->u.databaseOrSurDiagnostics->num_records = i;
 595     *number_returned = i;
 596     if (start + number > fset->hits)
 597         *next_position = 0;
 598     else
 599         *next_position = start + number;
 600     return rec;
 601 }
 602
 603 int yf::SPARQL::Session::invoke_sparql(mp::Package &package,
 604                                        const char *sparql_query,
 605                                        ConfPtr conf,
 606                                        WRBUF w)
 607 {
 608     Package http_package(package.session(), package.origin());
 609     mp::odr odr;
 610
 611     http_package.copy_filter(package);
 612     Z_GDU *gdu = z_get_HTTP_Request_uri(odr, conf->uri.c_str(), 0, 1);
 613
 614     z_HTTP_header_add(odr, &gdu->u.HTTP_Request->headers,
 615                       "Content-Type", "application/x-www-form-urlencoded");
 616     z_HTTP_header_add(odr, &gdu->u.HTTP_Request->headers,
 617                       "Accept", "application/sparql-results+xml,"
 618                       "application/rdf+xml");
 619     const char *names[2];
 620     names[0] = "query";
 621     names[1] = 0;
 622     const char *values[1];
 623     values[0] = sparql_query;
 624     char *path = 0;
 625     yaz_array_to_uri(&path, odr, (char **) names, (char **) values);
 626
 627     gdu->u.HTTP_Request->content_buf = path;
 628     gdu->u.HTTP_Request->content_len = strlen(path);
 629
 630     yaz_log(YLOG_DEBUG, "sparql: HTTP request\n%s", sparql_query);
 631
 632     http_package.request() = gdu;
 633     http_package.move();
 634
 635     Z_GDU *gdu_resp = http_package.response().get();
 636
 637     if (!gdu_resp || gdu_resp->which != Z_GDU_HTTP_Response)
 638     {
 639         wrbuf_puts(w, "no HTTP response from backend");
 640         return YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
 641     }
 642     else if (gdu_resp->u.HTTP_Response->code != 200)
 643     {
 644         Z_HTTP_Response *resp = gdu_resp->u.HTTP_Response;
 645         wrbuf_printf(w, "sparql: HTTP error %d from backend",
 646                      resp->code);
 647         package.log("sparql", YLOG_LOG,
 648             "HTTP error %d from backend ",
 649             resp->code );
 650         package.log("sparql", YLOG_LOG,
 651             "%.*s" , resp->content_len, resp->content_buf );
 652         return YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
 653     }
 654     Z_HTTP_Response *resp = gdu_resp->u.HTTP_Response;
 655     wrbuf_write(w, resp->content_buf, resp->content_len);
 656     return 0;
 657 }
 658
 659 Z_Records *yf::SPARQL::Session::explain_fetch(
 660     Package &package,
 661     FrontendSetPtr fset,
 662     ODR odr, Odr_oid *preferredRecordSyntax,
 663     Z_ElementSetNames *esn,
 664     int start, int number, int &error_code, std::string &addinfo,
 665     int *number_returned, int *next_position)
 666 {
 667     Z_Records *rec = (Z_Records *) odr_malloc(odr, sizeof(Z_Records));
 668     rec->which = Z_Records_DBOSD;
 669     rec->u.databaseOrSurDiagnostics = (Z_NamePlusRecordList *)
 670         odr_malloc(odr, sizeof(Z_NamePlusRecordList));
 671     rec->u.databaseOrSurDiagnostics->records = (Z_NamePlusRecord **)
 672         odr_malloc(odr, sizeof(Z_NamePlusRecord *) * number);
 673     int i;
 674     for (i = 0; i < number; i++)
 675     {
 676         int idx = start + i - 1;
 677         ConfPtr cp = fset->explaindblist[ idx];
 678         package.log("sparql", YLOG_LOG, "fetch explain %d:%s", idx, cp->db.c_str() );
 679         mp::wrbuf w;
 680         wrbuf_puts(w,"<explain xmlns=\"http://explain.z3950.org/dtd/2.0/\">\n");
 681         wrbuf_puts(w,"  <databaseInfo>\n");
 682         wrbuf_puts(w,"    <title>");
 683         wrbuf_xmlputs(w, cp->db.c_str());
 684         wrbuf_puts(w,"</title>\n");
 685         wrbuf_puts(w,"  </databaseInfo>\n");
 686         yaz_sparql_explain_indexes( cp->s, w, 2);
 687         wrbuf_puts(w,"</explain>\n");
 688
 689         rec->u.databaseOrSurDiagnostics->records[i] = (Z_NamePlusRecord *)
 690             odr_malloc(odr, sizeof(Z_NamePlusRecord));
 691         Z_NamePlusRecord *npr = rec->u.databaseOrSurDiagnostics->records[i];
 692         npr->databaseName = odr_strdup(odr, fset->db.c_str());
 693         npr->which = Z_NamePlusRecord_databaseRecord;
 694         npr->u.databaseRecord =
 695             z_ext_record_xml(odr, w.buf(), w.len() );
 696     }
 697     rec->u.databaseOrSurDiagnostics->num_records = i;
 698     *number_returned = i;
 699     if (start + number > fset->hits)
 700         *next_position = 0;
 701     else
 702         *next_position = start + number;
 703     return rec;
 704 }
 705
 706
 707
 708 Z_APDU *yf::SPARQL::Session::explain_search(mp::Package &package,
 709                            Z_APDU *apdu_req,
 710                            mp::odr &odr,
 711                            const char *sparql_query,
 712                            FrontendSetPtr fset)
 713 {
 714     Z_SearchRequest *req = apdu_req->u.searchRequest;
 715     Z_APDU *apdu_res = 0;
 716     //mp::wrbuf w;
 717
 718     package.log("sparql", YLOG_LOG, "Explain search" );
 719     int numbases = 0;
 720     //std::list<std::string> dblist;
 721     std::list<ConfPtr>::const_iterator it = m_sparql->db_conf.begin();
 722     m_frontend_sets[req->resultSetName] = fset;
 723     fset->explaindblist.clear();
 724     fset->explaindblist.reserve(m_sparql->db_conf.size());
 725
 726     for (; it != m_sparql->db_conf.end(); it++)
 727         if ((*it)->schema.length() > 0 )  // searchable db
 728         {
 729             numbases++;
 730             package.log("sparql", YLOG_LOG, "Explain %d: '%s'",
 731                         numbases, (*it)->db.c_str() );
 732             fset->explaindblist.push_back(*it);
 733 /*
 734             //yf::SPARQL::Result res;
 735             //res.conf = *it;
 736             std::string z =
 737               "<explain xmlns='http://explain.z3950.org/dtd/2.0/'>"
 738                 "<databaseInfo>"
 739                   "<title>" +
 740                     (*it)->db +
 741                   "</title>"
 742                 "</databaseInfo>"
 743               "</explain>";
 744             //res.doc = xmlParseMemory(z.c_str(), z.size());
 745             dblist.push_back(z);
 746 */
 747         }
 748     int number_returned = 0;
 749     int next_position = 0;
 750     Z_Records *records = 0;
 751     int error_code = 0;
 752     std::string addinfo;
 753
 754     Odr_int number = 0;
 755     const char *element_set_name = 0;
 756     mp::util::piggyback_sr(req, numbases, number, &element_set_name);
 757     if (number)
 758     {
 759         Z_ElementSetNames *esn;
 760
 761         if (number > *req->smallSetUpperBound)
 762             esn = req->mediumSetElementSetNames;
 763         else
 764             esn = req->smallSetElementSetNames;
 765         records = explain_fetch(package, fset,
 766                         odr, req->preferredRecordSyntax, esn,
 767                         1, number,
 768                         error_code, addinfo,
 769                         &number_returned,
 770                         &next_position);
 771     }
 772
 773     if (error_code)
 774     {
 775         apdu_res = odr.create_searchResponse(
 776                 apdu_req, error_code, addinfo.c_str());
 777     }
 778     else
 779     {
 780         apdu_res = odr.create_searchResponse(apdu_req, 0, 0);
 781         Z_SearchResponse *resp = apdu_res->u.searchResponse;
 782         *resp->resultCount = numbases;
 783         *resp->numberOfRecordsReturned = number_returned;
 784         *resp->nextResultSetPosition = next_position;
 785         resp->records = records;
 786     }
 787
 788     return apdu_res;
 789 }
 790
 791 Z_APDU *yf::SPARQL::Session::search(mp::Package &package,
 792                                     Z_APDU *apdu_req,
 793                                     mp::odr &odr,
 794                                     const char *sparql_query,
 795                                     ConfPtr conf, FrontendSetPtr fset)
 796 {
 797     Z_SearchRequest *req = apdu_req->u.searchRequest;
 798     Z_APDU *apdu_res = 0;
 799     mp::wrbuf w;
 800
 801     package.log("sparql", YLOG_LOG,
 802         "search query:\n%s", sparql_query );
 803
 804     int error = invoke_sparql(package, sparql_query, conf, w);
 805     if (error)
 806     {
 807         apdu_res = odr.create_searchResponse(apdu_req, error,
 808                                              w.len() ?
 809                                              w.c_str() : 0);
 810     }
 811     else
 812     {
 813         xmlDocPtr doc = xmlParseMemory(w.c_str(), w.len());
 814         if (!doc)
 815         {
 816             apdu_res = odr.create_searchResponse(
 817                 apdu_req,
 818                 YAZ_BIB1_TEMPORARY_SYSTEM_ERROR,
 819                 "invalid XML from backendbackend");
 820         }
 821         else
 822         {
 823             Result result;
 824             Z_Records *records = 0;
 825             int number_returned = 0;
 826             int next_position = 0;
 827             int error_code = 0;
 828             std::string addinfo;
 829
 830             result.doc = doc;
 831             result.conf = conf;
 832             fset->results.push_back(result);
 833             yaz_log(YLOG_DEBUG, "saving sparql result xmldoc=%p", doc);
 834
 835             get_result(result.doc, &fset->hits, -1, 0);
 836             m_frontend_sets[req->resultSetName] = fset;
 837
 838             result.doc = 0;
 839
 840             Odr_int number = 0;
 841             const char *element_set_name = 0;
 842             mp::util::piggyback_sr(req, fset->hits, number, &element_set_name);
 843             if (number)
 844             {
 845                 Z_ElementSetNames *esn;
 846
 847                 if (number > *req->smallSetUpperBound)
 848                     esn = req->mediumSetElementSetNames;
 849                 else
 850                     esn = req->smallSetElementSetNames;
 851                 records = fetch(package, fset,
 852                                 odr, req->preferredRecordSyntax, esn,
 853                                 1, number,
 854                                 error_code, addinfo,
 855                                 &number_returned,
 856                                 &next_position);
 857             }
 858             if (error_code)
 859             {
 860                 apdu_res =
 861                     odr.create_searchResponse(
 862                         apdu_req, error_code, addinfo.c_str());
 863             }
 864             else
 865             {
 866                 apdu_res =
 867                     odr.create_searchResponse(apdu_req, 0, 0);
 868                 Z_SearchResponse *resp = apdu_res->u.searchResponse;
 869                 *resp->resultCount = fset->hits;
 870                 *resp->numberOfRecordsReturned = number_returned;
 871                 *resp->nextResultSetPosition = next_position;
 872                 resp->records = records;
 873             }
 874         }
 875     }
 876     return apdu_res;
 877 }
 878
 879 void yf::SPARQL::Session::handle_z(mp::Package &package, Z_APDU *apdu_req)
 880 {
 881     mp::odr odr;
 882     Z_APDU *apdu_res = 0;
 883     if (apdu_req->which == Z_APDU_initRequest)
 884     {
 885         apdu_res = odr.create_initResponse(apdu_req, 0, 0);
 886         Z_InitRequest *req = apdu_req->u.initRequest;
 887         Z_InitResponse *resp = apdu_res->u.initResponse;
 888
 889         resp->implementationName = odr_strdup(odr, "sparql");
 890         if (ODR_MASK_GET(req->options, Z_Options_namedResultSets))
 891             m_support_named_result_sets = true;
 892         int i;
 893         static const int masks[] = {
 894             Z_Options_search, Z_Options_present,
 895             Z_Options_namedResultSets, -1
 896         };
 897         for (i = 0; masks[i] != -1; i++)
 898             if (ODR_MASK_GET(req->options, masks[i]))
 899                 ODR_MASK_SET(resp->options, masks[i]);
 900         static const int versions[] = {
 901             Z_ProtocolVersion_1,
 902             Z_ProtocolVersion_2,
 903             Z_ProtocolVersion_3,
 904             -1
 905         };
 906         for (i = 0; versions[i] != -1; i++)
 907             if (ODR_MASK_GET(req->protocolVersion, versions[i]))
 908                 ODR_MASK_SET(resp->protocolVersion, versions[i]);
 909             else
 910                 break;
 911         *resp->preferredMessageSize = *req->preferredMessageSize;
 912         *resp->maximumRecordSize = *req->maximumRecordSize;
 913     }
 914     else if (apdu_req->which == Z_APDU_close)
 915     {
 916         apdu_res = odr.create_close(apdu_req,
 917                                     Z_Close_finished, 0);
 918         package.session().close();
 919     }
 920     else if (apdu_req->which == Z_APDU_searchRequest)
 921     {
 922         Z_SearchRequest *req = apdu_req->u.searchRequest;
 923
 924         FrontendSets::iterator fset_it =
 925             m_frontend_sets.find(req->resultSetName);
 926         if (fset_it != m_frontend_sets.end())
 927         {
 928             // result set already exist
 929             // if replace indicator is off: we return diagnostic if
 930             // result set already exist.
 931             if (*req->replaceIndicator == 0)
 932             {
 933                 Z_APDU *apdu =
 934                     odr.create_searchResponse(
 935                         apdu_req,
 936                         YAZ_BIB1_RESULT_SET_EXISTS_AND_REPLACE_INDICATOR_OFF,
 937                         0);
 938                 package.response() = apdu;
 939             }
 940             m_frontend_sets.erase(fset_it);
 941         }
 942         if (req->query->which != Z_Query_type_1)
 943         {
 944             apdu_res = odr.create_searchResponse(
 945                 apdu_req, YAZ_BIB1_QUERY_TYPE_UNSUPP, 0);
 946         }
 947         else if (req->num_databaseNames != 1)
 948         {
 949             apdu_res = odr.create_searchResponse(
 950                 apdu_req,
 951                 YAZ_BIB1_ACCESS_TO_SPECIFIED_DATABASE_DENIED, 0);
 952         }
 953         else
 954         {
 955             std::string db = req->databaseNames[0];
 956             std::list<ConfPtr>::const_iterator it;
 957             FrontendSetPtr fset(new FrontendSet);
 958
 959             m_frontend_sets.erase(req->resultSetName);
 960             fset->db = db;
 961             if ( db != "explain" )
 962             {
 963                 it = m_sparql->db_conf.begin();
 964                 for (; it != m_sparql->db_conf.end(); it++)
 965                     if ((*it)->schema.length() > 0
 966                         && yaz_match_glob((*it)->db.c_str(), db.c_str()))
 967                     {
 968                         mp::wrbuf addinfo_wr;
 969                         mp::wrbuf sparql_wr;
 970                         int error =
 971                             yaz_sparql_from_rpn_wrbuf((*it)->s,
 972                                                     addinfo_wr, sparql_wr,
 973                                                     req->query->u.type_1);
 974                         if (error)
 975                         {
 976                             apdu_res = odr.create_searchResponse(
 977                                 apdu_req, error,
 978                                 addinfo_wr.len() ? addinfo_wr.c_str() : 0);
 979                         }
 980                         else
 981                         {
 982                             Z_APDU *apdu_1 = search(package, apdu_req, odr,
 983                                                     sparql_wr.c_str(), *it,
 984                                                     fset);
 985                             if (!apdu_res)
 986                                 apdu_res = apdu_1;
 987                         }
 988                     }
 989                 if (apdu_res == 0)
 990                 {
 991                     apdu_res = odr.create_searchResponse(
 992                         apdu_req, YAZ_BIB1_DATABASE_DOES_NOT_EXIST, db.c_str());
 993                 }
 994             }
 995             else
 996             { // The magic "explain" base
 997                 yaz_log(YLOG_LOG,"About to call explain_search");
 998                 const char *qry = "query";
 999                 apdu_res = explain_search( package, apdu_req, odr,
1000                                            qry, fset);
1001                   // TODO - Extract at least a term from the query, and
1002                   // do some filtering by that
1003                 yaz_log(YLOG_LOG,"Returned from explain_search");
1004             }
1005         }
1006     }
1007     else if (apdu_req->which == Z_APDU_presentRequest)
1008     {
1009         Z_PresentRequest *req = apdu_req->u.presentRequest;
1010         FrontendSets::iterator fset_it =
1011             m_frontend_sets.find(req->resultSetId);
1012         if (fset_it == m_frontend_sets.end())
1013         {
1014             apdu_res =
1015                 odr.create_presentResponse(
1016                     apdu_req, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
1017                     req->resultSetId);
1018             package.response() = apdu_res;
1019             return;
1020         }
1021         int number_returned = 0;
1022         int next_position = 0;
1023         int error_code = 0;
1024         std::string addinfo;
1025         Z_ElementSetNames *esn = 0;
1026         if (req->recordComposition)
1027         {
1028             if (req->recordComposition->which == Z_RecordComp_simple)
1029                 esn = req->recordComposition->u.simple;
1030             else
1031             {
1032                 apdu_res =
1033                     odr.create_presentResponse(
1034                         apdu_req,
1035                         YAZ_BIB1_ONLY_A_SINGLE_ELEMENT_SET_NAME_SUPPORTED,
1036                         0);
1037                 package.response() = apdu_res;
1038                 return;
1039             }
1040         }
1041         Z_Records *records;
1042         if ( fset_it->second->explaindblist.size() > 0 )
1043             records = explain_fetch(
1044                 package,
1045                 fset_it->second,
1046                 odr, req->preferredRecordSyntax, esn,
1047                 *req->resultSetStartPoint, *req->numberOfRecordsRequested,
1048                 error_code, addinfo,
1049                 &number_returned,
1050                 &next_position);
1051         else
1052             records = fetch(
1053                 package,
1054                 fset_it->second,
1055                 odr, req->preferredRecordSyntax, esn,
1056                 *req->resultSetStartPoint, *req->numberOfRecordsRequested,
1057                 error_code, addinfo,
1058                 &number_returned,
1059                 &next_position);
1060         if (error_code)
1061         {
1062             apdu_res =
1063                 odr.create_presentResponse(apdu_req, error_code,
1064                                            addinfo.c_str());
1065         }
1066         else
1067         {
1068             apdu_res =
1069                 odr.create_presentResponse(apdu_req, 0, 0);
1070             Z_PresentResponse *resp = apdu_res->u.presentResponse;
1071             resp->records = records;
1072             *resp->numberOfRecordsReturned = number_returned;
1073             *resp->nextResultSetPosition = next_position;
1074         }
1075     }
1076     else
1077     {
1078         apdu_res = odr.create_close(apdu_req,
1079                                     Z_Close_protocolError,
1080                                     "sparql: unhandled APDU");
1081         package.session().close();
1082     }
1083
1084     assert(apdu_res);
1085     package.response() = apdu_res;
1086 }
1087
1088 void yf::SPARQL::process(mp::Package &package) const
1089 {
1090     Z_APDU *apdu;
1091     SessionPtr p = get_session(package, &apdu);
1092     if (p && apdu)
1093     {
1094         p->handle_z(package, apdu);
1095     }
1096     else
1097         package.move();
1098     release_session(package);
1099 }
1100
1101 static mp::filter::Base* filter_creator()
1102 {
1103     return new mp::filter::SPARQL;
1104 }
1105
1106 extern "C" {
1107     struct metaproxy_1_filter_struct metaproxy_1_filter_sparql = {
1108         0,
1109         "sparql",
1110         filter_creator
1111     };
1112 }
1113
1114
1115 /*
1116  * Local variables:
1117  * c-basic-offset: 4
1118  * c-file-style: "Stroustrup"
1119  * indent-tabs-mode: nil
1120  * End:
1121  * vim: shiftwidth=4 tabstop=8 expandtab
1122  */
1123