src/filter_multi.cpp

   1 /* $Id: filter_multi.cpp,v 1.28 2007-11-18 10:44:40 adam Exp $
   2    Copyright (c) 2005-2007, Index Data.
   3
   4 This file is part of Metaproxy.
   5
   6 Metaproxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with Metaproxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #include <yaz/log.h>
  23
  24 #include "config.hpp"
  25
  26 #include "filter.hpp"
  27 #include "package.hpp"
  28
  29 #include <boost/thread/thread.hpp>
  30 #include <boost/thread/mutex.hpp>
  31 #include <boost/thread/condition.hpp>
  32 #include <boost/shared_ptr.hpp>
  33
  34 #include "util.hpp"
  35 #include "filter_multi.hpp"
  36
  37 #include <yaz/zgdu.h>
  38 #include <yaz/otherinfo.h>
  39 #include <yaz/diagbib1.h>
  40
  41 #include <vector>
  42 #include <algorithm>
  43 #include <map>
  44 #include <iostream>
  45
  46 namespace mp = metaproxy_1;
  47 namespace yf = mp::filter;
  48
  49 namespace metaproxy_1 {
  50     namespace filter {
  51         enum multi_merge_type {
  52             round_robin,
  53             serve_order
  54         };
  55         struct Multi::BackendSet {
  56             BackendPtr m_backend;
  57             int m_count;
  58             bool operator < (const BackendSet &k) const;
  59             bool operator == (const BackendSet &k) const;
  60         };
  61         struct Multi::ScanTermInfo {
  62             std::string m_norm_term;
  63             std::string m_display_term;
  64             int m_count;
  65             bool operator < (const ScanTermInfo &) const;
  66             bool operator == (const ScanTermInfo &) const;
  67             Z_Entry *get_entry(ODR odr);
  68         };
  69         struct Multi::FrontendSet {
  70             struct PresentJob {
  71                 BackendPtr m_backend;
  72                 int m_pos;
  73                 int m_inside_pos;
  74             };
  75             FrontendSet(std::string setname);
  76             FrontendSet();
  77             ~FrontendSet();
  78
  79             void round_robin(int pos, int number, std::list<PresentJob> &job);
  80             void serve_order(int pos, int number, std::list<PresentJob> &job);
  81
  82             std::list<BackendSet> m_backend_sets;
  83             std::string m_setname;
  84         };
  85         struct Multi::Backend {
  86             PackagePtr m_package;
  87             std::string m_backend_database;
  88             std::string m_vhost;
  89             std::string m_route;
  90             void operator() (void);  // thread operation
  91         };
  92         struct Multi::Frontend {
  93             Frontend(Rep *rep);
  94             ~Frontend();
  95             bool m_is_multi;
  96             bool m_in_use;
  97             std::list<BackendPtr> m_backend_list;
  98             std::map<std::string,Multi::FrontendSet> m_sets;
  99
 100             void multi_move(std::list<BackendPtr> &blist);
 101             void init(Package &package, Z_GDU *gdu);
 102             void close(Package &package);
 103             void search(Package &package, Z_APDU *apdu);
 104             void present(Package &package, Z_APDU *apdu);
 105             void scan1(Package &package, Z_APDU *apdu);
 106             void scan2(Package &package, Z_APDU *apdu);
 107             Rep *m_p;
 108         };
 109         struct Multi::Map {
 110             Map(std::list<std::string> hosts, std::string route);
 111             Map();
 112             std::list<std::string> m_hosts;
 113             std::string m_route;
 114         };
 115         class Multi::Rep {
 116             friend class Multi;
 117             friend struct Frontend;
 118
 119             Rep();
 120             FrontendPtr get_frontend(Package &package);
 121             void release_frontend(Package &package);
 122         private:
 123             std::map<std::string,std::string> m_target_route;
 124             boost::mutex m_mutex;
 125             boost::condition m_cond_session_ready;
 126             std::map<mp::Session, FrontendPtr> m_clients;
 127             bool m_hide_unavailable;
 128             multi_merge_type m_merge_type;
 129         };
 130     }
 131 }
 132
 133 yf::Multi::Rep::Rep()
 134 {
 135     m_hide_unavailable = false;
 136     m_merge_type = round_robin;
 137 }
 138
 139 bool yf::Multi::BackendSet::operator < (const BackendSet &k) const
 140 {
 141     return m_count < k.m_count;
 142 }
 143
 144 yf::Multi::Frontend::Frontend(Rep *rep)
 145 {
 146     m_p = rep;
 147     m_is_multi = false;
 148 }
 149
 150 yf::Multi::Frontend::~Frontend()
 151 {
 152 }
 153
 154 yf::Multi::FrontendPtr yf::Multi::Rep::get_frontend(mp::Package &package)
 155 {
 156     boost::mutex::scoped_lock lock(m_mutex);
 157
 158     std::map<mp::Session,yf::Multi::FrontendPtr>::iterator it;
 159
 160     while(true)
 161     {
 162         it = m_clients.find(package.session());
 163         if (it == m_clients.end())
 164             break;
 165
 166         if (!it->second->m_in_use)
 167         {
 168             it->second->m_in_use = true;
 169             return it->second;
 170         }
 171         m_cond_session_ready.wait(lock);
 172     }
 173     FrontendPtr f(new Frontend(this));
 174     m_clients[package.session()] = f;
 175     f->m_in_use = true;
 176     return f;
 177 }
 178
 179 void yf::Multi::Rep::release_frontend(mp::Package &package)
 180 {
 181     boost::mutex::scoped_lock lock(m_mutex);
 182     std::map<mp::Session,yf::Multi::FrontendPtr>::iterator it;
 183
 184     it = m_clients.find(package.session());
 185     if (it != m_clients.end())
 186     {
 187         if (package.session().is_closed())
 188         {
 189             it->second->close(package);
 190             m_clients.erase(it);
 191         }
 192         else
 193         {
 194             it->second->m_in_use = false;
 195         }
 196         m_cond_session_ready.notify_all();
 197     }
 198 }
 199
 200 yf::Multi::FrontendSet::FrontendSet(std::string setname)
 201     :  m_setname(setname)
 202 {
 203 }
 204
 205
 206 yf::Multi::FrontendSet::FrontendSet()
 207 {
 208 }
 209
 210
 211 yf::Multi::FrontendSet::~FrontendSet()
 212 {
 213 }
 214
 215 yf::Multi::Map::Map(std::list<std::string> hosts, std::string route)
 216     : m_hosts(hosts), m_route(route)
 217 {
 218 }
 219
 220 yf::Multi::Map::Map()
 221 {
 222 }
 223
 224 yf::Multi::Multi() : m_p(new Multi::Rep)
 225 {
 226 }
 227
 228 yf::Multi::~Multi() {
 229 }
 230
 231
 232 void yf::Multi::Backend::operator() (void)
 233 {
 234     m_package->move(m_route);
 235 }
 236
 237
 238 void yf::Multi::Frontend::close(mp::Package &package)
 239 {
 240     std::list<BackendPtr>::const_iterator bit;
 241     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 242     {
 243         BackendPtr b = *bit;
 244
 245         b->m_package->copy_filter(package);
 246         b->m_package->request() = (Z_GDU *) 0;
 247         b->m_package->session().close();
 248         b->m_package->move(b->m_route);
 249     }
 250 }
 251
 252 void yf::Multi::Frontend::multi_move(std::list<BackendPtr> &blist)
 253 {
 254     std::list<BackendPtr>::const_iterator bit;
 255     boost::thread_group g;
 256     for (bit = blist.begin(); bit != blist.end(); bit++)
 257     {
 258         g.add_thread(new boost::thread(**bit));
 259     }
 260     g.join_all();
 261 }
 262
 263 void yf::Multi::FrontendSet::serve_order(int start, int number,
 264                                          std::list<PresentJob> &jobs)
 265 {
 266     int i;
 267     for (i = 0; i < number; i++)
 268     {
 269         std::list<BackendSet>::const_iterator bsit;
 270         int voffset = 0;
 271         int offset = start + i - 1;
 272         for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end();
 273              bsit++)
 274         {
 275             if (offset >= voffset && offset < voffset + bsit->m_count)
 276             {
 277                 PresentJob job;
 278                 job.m_backend = bsit->m_backend;
 279                 job.m_inside_pos = 0;
 280                 job.m_pos = offset - voffset + 1;
 281                 jobs.push_back(job);
 282                 break;
 283             }
 284             voffset += bsit->m_count;
 285         }
 286     }
 287 }
 288
 289 void yf::Multi::FrontendSet::round_robin(int start, int number,
 290                                          std::list<PresentJob> &jobs)
 291 {
 292     std::list<int> pos;
 293     std::list<int> inside_pos;
 294     std::list<BackendSet>::const_iterator bsit;
 295     for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); bsit++)
 296     {
 297         pos.push_back(1);
 298         inside_pos.push_back(0);
 299     }
 300
 301     int p = 1;
 302 #if 1
 303     // optimization step!
 304     int omin = 0;
 305     while(true)
 306     {
 307         int min = 0;
 308         int no_left = 0;
 309         // find min count for each set which is > omin
 310         for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); bsit++)
 311         {
 312             if (bsit->m_count > omin)
 313             {
 314                 if (no_left == 0 || bsit->m_count < min)
 315                     min = bsit->m_count;
 316                 no_left++;
 317             }
 318         }
 319         if (no_left == 0) // if nothing greater than omin, bail out.
 320             break;
 321         int skip = no_left * min;
 322         if (p + skip > start)  // step gets us "into" present range?
 323         {
 324             // Yes. skip until start.. Rounding off is deliberate!
 325             min = (start-p) / no_left;
 326             p += no_left * min;
 327
 328             // update positions in each set..
 329             std::list<int>::iterator psit = pos.begin();
 330             for (psit = pos.begin(); psit != pos.end(); psit++)
 331                 *psit += min;
 332             break;
 333         }
 334         // skip on each set.. before "present range"..
 335         p = p + skip;
 336
 337         std::cout << "\nSKIP min=" << min << " no_left=" << no_left << "\n\n";
 338
 339         std::list<int>::iterator psit = pos.begin();
 340         for (psit = pos.begin(); psit != pos.end(); psit++)
 341             *psit += min;
 342
 343         omin = min; // update so we consider next class (with higher count)
 344     }
 345 #endif
 346     int fetched = 0;
 347     bool more = true;
 348     while (more)
 349     {
 350         more = false;
 351         std::list<int>::iterator psit = pos.begin();
 352         std::list<int>::iterator esit = inside_pos.begin();
 353         bsit = m_backend_sets.begin();
 354
 355         for (; bsit != m_backend_sets.end(); psit++,esit++,bsit++)
 356         {
 357             if (fetched >= number)
 358             {
 359                 more = false;
 360                 break;
 361             }
 362             if (*psit <= bsit->m_count)
 363             {
 364                 if (p >= start)
 365                 {
 366                     PresentJob job;
 367                     job.m_backend = bsit->m_backend;
 368                     job.m_pos = *psit;
 369                     job.m_inside_pos = *esit;
 370                     jobs.push_back(job);
 371                     (*esit)++;
 372                     fetched++;
 373                 }
 374                 (*psit)++;
 375                 p++;
 376                 more = true;
 377             }
 378         }
 379     }
 380 }
 381
 382 void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu)
 383 {
 384     Z_InitRequest *req = gdu->u.z3950->u.initRequest;
 385
 386     std::list<std::string> targets;
 387
 388     mp::util::get_vhost_otherinfo(req->otherInfo, targets);
 389
 390     if (targets.size() < 1)
 391     {
 392         package.move();
 393         return;
 394     }
 395
 396     std::list<std::string>::const_iterator t_it = targets.begin();
 397     for (; t_it != targets.end(); t_it++)
 398     {
 399         Session s;
 400         Backend *b = new Backend;
 401         b->m_vhost = *t_it;
 402
 403         b->m_route = m_p->m_target_route[*t_it];
 404         // b->m_route unset
 405         b->m_package = PackagePtr(new Package(s, package.origin()));
 406
 407         m_backend_list.push_back(BackendPtr(b));
 408     }
 409     m_is_multi = true;
 410
 411     // create init request
 412     std::list<BackendPtr>::iterator bit;
 413     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 414     {
 415         mp::odr odr;
 416         BackendPtr b = *bit;
 417         Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest);
 418
 419         std::list<std::string>vhost_one;
 420         vhost_one.push_back(b->m_vhost);
 421         mp::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo,
 422                                        odr, vhost_one);
 423
 424         Z_InitRequest *req = init_apdu->u.initRequest;
 425
 426         ODR_MASK_SET(req->options, Z_Options_search);
 427         ODR_MASK_SET(req->options, Z_Options_present);
 428         ODR_MASK_SET(req->options, Z_Options_namedResultSets);
 429         ODR_MASK_SET(req->options, Z_Options_scan);
 430
 431         ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
 432         ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
 433         ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3);
 434
 435         b->m_package->request() = init_apdu;
 436
 437         b->m_package->copy_filter(package);
 438     }
 439     multi_move(m_backend_list);
 440
 441     // create the frontend init response based on each backend init response
 442     mp::odr odr;
 443
 444     Z_APDU *f_apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
 445     Z_InitResponse *f_resp = f_apdu->u.initResponse;
 446
 447     ODR_MASK_SET(f_resp->options, Z_Options_search);
 448     ODR_MASK_SET(f_resp->options, Z_Options_present);
 449     ODR_MASK_SET(f_resp->options, Z_Options_namedResultSets);
 450
 451     ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_1);
 452     ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_2);
 453     ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_3);
 454
 455     int no_failed = 0;
 456     int no_succeeded = 0;
 457     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); )
 458     {
 459         PackagePtr p = (*bit)->m_package;
 460
 461         if (p->session().is_closed())
 462         {
 463             // failed. Remove from list and increment number of failed
 464             no_failed++;
 465             bit = m_backend_list.erase(bit);
 466             continue;
 467         }
 468         no_succeeded++;
 469
 470         Z_GDU *gdu = p->response().get();
 471         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
 472             Z_APDU_initResponse)
 473         {
 474             int i;
 475             Z_APDU *b_apdu = gdu->u.z3950;
 476             Z_InitResponse *b_resp = b_apdu->u.initResponse;
 477
 478             // common options for all backends
 479             for (i = 0; i <= Z_Options_stringSchema; i++)
 480             {
 481                 if (!ODR_MASK_GET(b_resp->options, i))
 482                     ODR_MASK_CLEAR(f_resp->options, i);
 483             }
 484             // common protocol version
 485             for (i = 0; i <= Z_ProtocolVersion_3; i++)
 486                 if (!ODR_MASK_GET(b_resp->protocolVersion, i))
 487                     ODR_MASK_CLEAR(f_resp->protocolVersion, i);
 488             // reject if any of the backends reject
 489             if (!*b_resp->result)
 490                 *f_resp->result = 0;
 491         }
 492         else
 493         {
 494             // if any target does not return init return that (close or
 495             // similar )
 496             package.response() = p->response();
 497             return;
 498         }
 499         bit++;
 500     }
 501     if (m_p->m_hide_unavailable)
 502     {
 503         if (no_succeeded == 0)
 504             package.session().close();
 505     }
 506     else
 507     {
 508         if (no_failed)
 509             package.session().close();
 510     }
 511     package.response() = f_apdu;
 512 }
 513
 514 void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
 515 {
 516     // create search request
 517     Z_SearchRequest *req = apdu_req->u.searchRequest;
 518
 519     // save these for later
 520     int smallSetUpperBound = *req->smallSetUpperBound;
 521     int largeSetLowerBound = *req->largeSetLowerBound;
 522     int mediumSetPresentNumber = *req->mediumSetPresentNumber;
 523
 524     // they are altered now - to disable piggyback
 525     *req->smallSetUpperBound = 0;
 526     *req->largeSetLowerBound = 1;
 527     *req->mediumSetPresentNumber = 1;
 528
 529     int default_num_db = req->num_databaseNames;
 530     char **default_db = req->databaseNames;
 531
 532     std::list<BackendPtr>::const_iterator bit;
 533     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 534     {
 535         PackagePtr p = (*bit)->m_package;
 536         mp::odr odr;
 537
 538         if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
 539                                                 &req->num_databaseNames,
 540                                                 &req->databaseNames))
 541         {
 542             req->num_databaseNames = default_num_db;
 543             req->databaseNames = default_db;
 544         }
 545         p->request() = apdu_req;
 546         p->copy_filter(package);
 547     }
 548     multi_move(m_backend_list);
 549
 550     // look at each response
 551     FrontendSet resultSet(std::string(req->resultSetName));
 552
 553     int result_set_size = 0;
 554     Z_Records *z_records_diag = 0;  // no diagnostics (yet)
 555     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 556     {
 557         PackagePtr p = (*bit)->m_package;
 558
 559         if (p->session().is_closed()) // if any backend closes, close frontend
 560             package.session().close();
 561
 562         Z_GDU *gdu = p->response().get();
 563         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
 564             Z_APDU_searchResponse)
 565         {
 566             Z_APDU *b_apdu = gdu->u.z3950;
 567             Z_SearchResponse *b_resp = b_apdu->u.searchResponse;
 568
 569             // see we get any errors (AKA diagnstics)
 570             if (b_resp->records)
 571             {
 572                 if (b_resp->records->which == Z_Records_NSD
 573                     || b_resp->records->which == Z_Records_multipleNSD)
 574                     z_records_diag = b_resp->records;
 575                 // we may set this multiple times (TOO BAD!)
 576             }
 577             BackendSet backendSet;
 578             backendSet.m_backend = *bit;
 579             backendSet.m_count = *b_resp->resultCount;
 580             result_set_size += *b_resp->resultCount;
 581             resultSet.m_backend_sets.push_back(backendSet);
 582         }
 583         else
 584         {
 585             // if any target does not return search response - return that
 586             package.response() = p->response();
 587             return;
 588         }
 589     }
 590
 591     mp::odr odr;
 592     Z_APDU *f_apdu = odr.create_searchResponse(apdu_req, 0, 0);
 593     Z_SearchResponse *f_resp = f_apdu->u.searchResponse;
 594
 595     *f_resp->resultCount = result_set_size;
 596     if (z_records_diag)
 597     {
 598         // search error
 599         f_resp->records = z_records_diag;
 600         package.response() = f_apdu;
 601         return;
 602     }
 603     // assume OK
 604     m_sets[resultSet.m_setname] = resultSet;
 605
 606     int number;
 607     mp::util::piggyback(smallSetUpperBound,
 608                          largeSetLowerBound,
 609                          mediumSetPresentNumber,
 610                          result_set_size,
 611                          number);
 612     Package pp(package.session(), package.origin());
 613     if (number > 0)
 614     {
 615         pp.copy_filter(package);
 616         Z_APDU *p_apdu = zget_APDU(odr, Z_APDU_presentRequest);
 617         Z_PresentRequest *p_req = p_apdu->u.presentRequest;
 618         p_req->preferredRecordSyntax = req->preferredRecordSyntax;
 619         p_req->resultSetId = req->resultSetName;
 620         *p_req->resultSetStartPoint = 1;
 621         *p_req->numberOfRecordsRequested = number;
 622         pp.request() = p_apdu;
 623         present(pp, p_apdu);
 624
 625         if (pp.session().is_closed())
 626             package.session().close();
 627
 628         Z_GDU *gdu = pp.response().get();
 629         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
 630             Z_APDU_presentResponse)
 631         {
 632             Z_PresentResponse *p_res = gdu->u.z3950->u.presentResponse;
 633             f_resp->records = p_res->records;
 634             *f_resp->numberOfRecordsReturned =
 635                 *p_res->numberOfRecordsReturned;
 636             *f_resp->nextResultSetPosition =
 637                 *p_res->nextResultSetPosition;
 638         }
 639         else
 640         {
 641             package.response() = pp.response();
 642             return;
 643         }
 644     }
 645     package.response() = f_apdu; // in this scope because of p
 646 }
 647
 648 void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
 649 {
 650     // create present request
 651     Z_PresentRequest *req = apdu_req->u.presentRequest;
 652
 653     Sets_it it;
 654     it = m_sets.find(std::string(req->resultSetId));
 655     if (it == m_sets.end())
 656     {
 657         mp::odr odr;
 658         Z_APDU *apdu =
 659             odr.create_presentResponse(
 660                 apdu_req,
 661                 YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 662                 req->resultSetId);
 663         package.response() = apdu;
 664         return;
 665     }
 666     std::list<Multi::FrontendSet::PresentJob> jobs;
 667     int start = *req->resultSetStartPoint;
 668     int number = *req->numberOfRecordsRequested;
 669
 670     if (m_p->m_merge_type == round_robin)
 671         it->second.round_robin(start, number, jobs);
 672     else if (m_p->m_merge_type == serve_order)
 673         it->second.serve_order(start, number, jobs);
 674
 675     if (0)
 676     {
 677         std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
 678         for (jit = jobs.begin(); jit != jobs.end(); jit++)
 679         {
 680             yaz_log(YLOG_LOG, "job pos=%d inside_pos=%d",
 681                     jit->m_pos, jit->m_inside_pos);
 682         }
 683     }
 684
 685     std::list<BackendPtr> present_backend_list;
 686
 687     std::list<BackendSet>::const_iterator bsit;
 688     bsit = it->second.m_backend_sets.begin();
 689     for (; bsit != it->second.m_backend_sets.end(); bsit++)
 690     {
 691         std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
 692         int start = -1;
 693         int end = -1;
 694
 695         for (jit = jobs.begin(); jit != jobs.end(); jit++)
 696         {
 697             if (jit->m_backend == bsit->m_backend)
 698             {
 699                 if (start == -1 || jit->m_pos < start)
 700                     start = jit->m_pos;
 701                 if (end == -1 || jit->m_pos > end)
 702                     end = jit->m_pos;
 703             }
 704         }
 705         if (start != -1)
 706         {
 707             PackagePtr p = bsit->m_backend->m_package;
 708
 709             *req->resultSetStartPoint = start;
 710             *req->numberOfRecordsRequested = end - start + 1;
 711
 712             p->request() = apdu_req;
 713             p->copy_filter(package);
 714
 715             present_backend_list.push_back(bsit->m_backend);
 716         }
 717     }
 718     multi_move(present_backend_list);
 719
 720     // look at each response
 721     Z_Records *z_records_diag = 0;
 722
 723     std::list<BackendPtr>::const_iterator pbit = present_backend_list.begin();
 724     for (; pbit != present_backend_list.end(); pbit++)
 725     {
 726         PackagePtr p = (*pbit)->m_package;
 727
 728         if (p->session().is_closed()) // if any backend closes, close frontend
 729             package.session().close();
 730
 731         Z_GDU *gdu = p->response().get();
 732         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
 733             Z_APDU_presentResponse)
 734         {
 735             Z_APDU *b_apdu = gdu->u.z3950;
 736             Z_PresentResponse *b_resp = b_apdu->u.presentResponse;
 737
 738             // see we get any errors (AKA diagnstics)
 739             if (b_resp->records)
 740             {
 741                 if (b_resp->records->which != Z_Records_DBOSD)
 742                     z_records_diag = b_resp->records;
 743                 // we may set this multiple times (TOO BAD!)
 744             }
 745         }
 746         else
 747         {
 748             // if any target does not return present response - return that
 749             package.response() = p->response();
 750             return;
 751         }
 752     }
 753
 754     mp::odr odr;
 755     Z_APDU *f_apdu = odr.create_presentResponse(apdu_req, 0, 0);
 756     Z_PresentResponse *f_resp = f_apdu->u.presentResponse;
 757
 758     if (z_records_diag)
 759     {
 760         f_resp->records = z_records_diag;
 761         *f_resp->presentStatus = Z_PresentStatus_failure;
 762     }
 763     else
 764     {
 765         f_resp->records = (Z_Records *) odr_malloc(odr, sizeof(Z_Records));
 766         Z_Records * records = f_resp->records;
 767         records->which = Z_Records_DBOSD;
 768         records->u.databaseOrSurDiagnostics =
 769             (Z_NamePlusRecordList *)
 770             odr_malloc(odr, sizeof(Z_NamePlusRecordList));
 771         Z_NamePlusRecordList *nprl = records->u.databaseOrSurDiagnostics;
 772         nprl->num_records = jobs.size();
 773         nprl->records = (Z_NamePlusRecord**)
 774             odr_malloc(odr, sizeof(Z_NamePlusRecord *) * nprl->num_records);
 775         int i = 0;
 776         std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
 777         for (jit = jobs.begin(); jit != jobs.end(); jit++, i++)
 778         {
 779             PackagePtr p = jit->m_backend->m_package;
 780
 781             Z_GDU *gdu = p->response().get();
 782             Z_APDU *b_apdu = gdu->u.z3950;
 783             Z_PresentResponse *b_resp = b_apdu->u.presentResponse;
 784
 785             nprl->records[i] = (Z_NamePlusRecord*)
 786                 odr_malloc(odr, sizeof(Z_NamePlusRecord));
 787             int inside_pos = jit->m_inside_pos;
 788             if (inside_pos >= b_resp->records->
 789                 u.databaseOrSurDiagnostics->num_records)
 790                 break;
 791             *nprl->records[i] = *b_resp->records->
 792                 u.databaseOrSurDiagnostics->records[inside_pos];
 793             nprl->records[i]->databaseName =
 794                     odr_strdup(odr, jit->m_backend->m_vhost.c_str());
 795         }
 796         nprl->num_records = i; // usually same as jobs.size();
 797         *f_resp->nextResultSetPosition = start + i;
 798         *f_resp->numberOfRecordsReturned = i;
 799     }
 800     package.response() = f_apdu;
 801 }
 802
 803 void yf::Multi::Frontend::scan1(mp::Package &package, Z_APDU *apdu_req)
 804 {
 805     if (m_backend_list.size() > 1)
 806     {
 807         mp::odr odr;
 808         Z_APDU *f_apdu =
 809             odr.create_scanResponse(
 810                 apdu_req, YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP, 0);
 811         package.response() = f_apdu;
 812         return;
 813     }
 814     Z_ScanRequest *req = apdu_req->u.scanRequest;
 815
 816     int default_num_db = req->num_databaseNames;
 817     char **default_db = req->databaseNames;
 818
 819     std::list<BackendPtr>::const_iterator bit;
 820     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 821     {
 822         PackagePtr p = (*bit)->m_package;
 823         mp::odr odr;
 824
 825         if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
 826                                                 &req->num_databaseNames,
 827                                                 &req->databaseNames))
 828         {
 829             req->num_databaseNames = default_num_db;
 830             req->databaseNames = default_db;
 831         }
 832         p->request() = apdu_req;
 833         p->copy_filter(package);
 834     }
 835     multi_move(m_backend_list);
 836
 837     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 838     {
 839         PackagePtr p = (*bit)->m_package;
 840
 841         if (p->session().is_closed()) // if any backend closes, close frontend
 842             package.session().close();
 843
 844         Z_GDU *gdu = p->response().get();
 845         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
 846             Z_APDU_scanResponse)
 847         {
 848             package.response() = p->response();
 849             break;
 850         }
 851         else
 852         {
 853             // if any target does not return scan response - return that
 854             package.response() = p->response();
 855             return;
 856         }
 857     }
 858 }
 859
 860 bool yf::Multi::ScanTermInfo::operator < (const ScanTermInfo &k) const
 861 {
 862     return m_norm_term < k.m_norm_term;
 863 }
 864
 865 bool yf::Multi::ScanTermInfo::operator == (const ScanTermInfo &k) const
 866 {
 867     return m_norm_term == k.m_norm_term;
 868 }
 869
 870 Z_Entry *yf::Multi::ScanTermInfo::get_entry(ODR odr)
 871 {
 872     Z_Entry *e = (Z_Entry *)odr_malloc(odr, sizeof(*e));
 873     e->which = Z_Entry_termInfo;
 874     Z_TermInfo *t;
 875     t = e->u.termInfo = (Z_TermInfo *) odr_malloc(odr, sizeof(*t));
 876     t->suggestedAttributes = 0;
 877     t->displayTerm = 0;
 878     t->alternativeTerm = 0;
 879     t->byAttributes = 0;
 880     t->otherTermInfo = 0;
 881     t->globalOccurrences = odr_intdup(odr, m_count);
 882     t->term = (Z_Term *)
 883         odr_malloc(odr, sizeof(*t->term));
 884     t->term->which = Z_Term_general;
 885     Odr_oct *o;
 886     t->term->u.general = o = (Odr_oct *)odr_malloc(odr, sizeof(Odr_oct));
 887
 888     o->len = o->size = m_norm_term.size();
 889     o->buf = (unsigned char *) odr_malloc(odr, o->len);
 890     memcpy(o->buf, m_norm_term.c_str(), o->len);
 891     return e;
 892 }
 893
 894 void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req)
 895 {
 896     Z_ScanRequest *req = apdu_req->u.scanRequest;
 897
 898     int default_num_db = req->num_databaseNames;
 899     char **default_db = req->databaseNames;
 900
 901     std::list<BackendPtr>::const_iterator bit;
 902     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 903     {
 904         PackagePtr p = (*bit)->m_package;
 905         mp::odr odr;
 906
 907         if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
 908                                                 &req->num_databaseNames,
 909                                                 &req->databaseNames))
 910         {
 911             req->num_databaseNames = default_num_db;
 912             req->databaseNames = default_db;
 913         }
 914         p->request() = apdu_req;
 915         p->copy_filter(package);
 916     }
 917     multi_move(m_backend_list);
 918
 919     ScanTermInfoList entries_before;
 920     ScanTermInfoList entries_after;
 921     int no_before = 0;
 922     int no_after = 0;
 923
 924     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 925     {
 926         PackagePtr p = (*bit)->m_package;
 927
 928         if (p->session().is_closed()) // if any backend closes, close frontend
 929             package.session().close();
 930
 931         Z_GDU *gdu = p->response().get();
 932         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
 933             Z_APDU_scanResponse)
 934         {
 935             Z_ScanResponse *res = gdu->u.z3950->u.scanResponse;
 936
 937             if (res->entries && res->entries->nonsurrogateDiagnostics)
 938             {
 939                 // failure
 940                 mp::odr odr;
 941                 Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 1, 0);
 942                 Z_ScanResponse *f_res = f_apdu->u.scanResponse;
 943
 944                 f_res->entries->nonsurrogateDiagnostics =
 945                     res->entries->nonsurrogateDiagnostics;
 946                 f_res->entries->num_nonsurrogateDiagnostics =
 947                     res->entries->num_nonsurrogateDiagnostics;
 948
 949                 package.response() = f_apdu;
 950                 return;
 951             }
 952
 953             if (res->entries && res->entries->entries)
 954             {
 955                 Z_Entry **entries = res->entries->entries;
 956                 int num_entries = res->entries->num_entries;
 957                 int position = 1;
 958                 if (req->preferredPositionInResponse)
 959                     position = *req->preferredPositionInResponse;
 960                 if (res->positionOfTerm)
 961                     position = *res->positionOfTerm;
 962
 963                 // before
 964                 int i;
 965                 for (i = 0; i<position-1 && i<num_entries; i++)
 966                 {
 967                     Z_Entry *ent = entries[i];
 968
 969                     if (ent->which == Z_Entry_termInfo)
 970                     {
 971                         ScanTermInfo my;
 972
 973                         int *occur = ent->u.termInfo->globalOccurrences;
 974                         my.m_count = occur ? *occur : 0;
 975
 976                         if (ent->u.termInfo->term->which == Z_Term_general)
 977                         {
 978                             my.m_norm_term = std::string(
 979                                 (const char *)
 980                                 ent->u.termInfo->term->u.general->buf,
 981                                 ent->u.termInfo->term->u.general->len);
 982                         }
 983                         if (my.m_norm_term.length())
 984                         {
 985                             ScanTermInfoList::iterator it =
 986                                 entries_before.begin();
 987                             while (it != entries_before.end() && my <*it)
 988                                 it++;
 989                             if (my == *it)
 990                             {
 991                                 it->m_count += my.m_count;
 992                             }
 993                             else
 994                             {
 995                                 entries_before.insert(it, my);
 996                                 no_before++;
 997                             }
 998                         }
 999                     }
1000                 }
1001                 // after
1002                 if (position <= 0)
1003                     i = 0;
1004                 else
1005                     i = position-1;
1006                 for ( ; i<num_entries; i++)
1007                 {
1008                     Z_Entry *ent = entries[i];
1009
1010                     if (ent->which == Z_Entry_termInfo)
1011                     {
1012                         ScanTermInfo my;
1013
1014                         int *occur = ent->u.termInfo->globalOccurrences;
1015                         my.m_count = occur ? *occur : 0;
1016
1017                         if (ent->u.termInfo->term->which == Z_Term_general)
1018                         {
1019                             my.m_norm_term = std::string(
1020                                 (const char *)
1021                                 ent->u.termInfo->term->u.general->buf,
1022                                 ent->u.termInfo->term->u.general->len);
1023                         }
1024                         if (my.m_norm_term.length())
1025                         {
1026                             ScanTermInfoList::iterator it =
1027                                 entries_after.begin();
1028                             while (it != entries_after.end() && *it < my)
1029                                 it++;
1030                             if (my == *it)
1031                             {
1032                                 it->m_count += my.m_count;
1033                             }
1034                             else
1035                             {
1036                                 entries_after.insert(it, my);
1037                                 no_after++;
1038                             }
1039                         }
1040                     }
1041                 }
1042
1043             }
1044         }
1045         else
1046         {
1047             // if any target does not return scan response - return that
1048             package.response() = p->response();
1049             return;
1050         }
1051     }
1052
1053     if (true)
1054     {
1055         std::cout << "BEFORE\n";
1056         ScanTermInfoList::iterator it = entries_before.begin();
1057         for(; it != entries_before.end(); it++)
1058         {
1059             std::cout << " " << it->m_norm_term << " " << it->m_count << "\n";
1060         }
1061
1062         std::cout << "AFTER\n";
1063         it = entries_after.begin();
1064         for(; it != entries_after.end(); it++)
1065         {
1066             std::cout << " " << it->m_norm_term << " " << it->m_count << "\n";
1067         }
1068     }
1069
1070     if (false)
1071     {
1072         mp::odr odr;
1073         Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 1, "not implemented");
1074         package.response() = f_apdu;
1075     }
1076     else
1077     {
1078         mp::odr odr;
1079         Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 0, 0);
1080         Z_ScanResponse *resp = f_apdu->u.scanResponse;
1081
1082         int number_returned = *req->numberOfTermsRequested;
1083         int position_returned = *req->preferredPositionInResponse;
1084
1085         resp->entries->num_entries = number_returned;
1086         resp->entries->entries = (Z_Entry**)
1087             odr_malloc(odr, sizeof(Z_Entry*) * number_returned);
1088         int i;
1089
1090         int lbefore = entries_before.size();
1091         if (lbefore < position_returned-1)
1092             position_returned = lbefore+1;
1093
1094         ScanTermInfoList::iterator it = entries_before.begin();
1095         for (i = 0; i<position_returned-1 && it != entries_before.end(); i++, it++)
1096         {
1097             resp->entries->entries[position_returned-2-i] = it->get_entry(odr);
1098         }
1099
1100         it = entries_after.begin();
1101
1102         if (position_returned <= 0)
1103             i = 0;
1104         else
1105             i = position_returned-1;
1106         for (; i<number_returned && it != entries_after.end(); i++, it++)
1107         {
1108             resp->entries->entries[i] = it->get_entry(odr);
1109         }
1110
1111         number_returned = i;
1112
1113         resp->positionOfTerm = odr_intdup(odr, position_returned);
1114         resp->numberOfEntriesReturned = odr_intdup(odr, number_returned);
1115         resp->entries->num_entries = number_returned;
1116
1117         package.response() = f_apdu;
1118     }
1119 }
1120
1121
1122 void yf::Multi::process(mp::Package &package) const
1123 {
1124     FrontendPtr f = m_p->get_frontend(package);
1125
1126     Z_GDU *gdu = package.request().get();
1127
1128     if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
1129         Z_APDU_initRequest && !f->m_is_multi)
1130     {
1131         f->init(package, gdu);
1132     }
1133     else if (!f->m_is_multi)
1134         package.move();
1135     else if (gdu && gdu->which == Z_GDU_Z3950)
1136     {
1137         Z_APDU *apdu = gdu->u.z3950;
1138         if (apdu->which == Z_APDU_initRequest)
1139         {
1140             mp::odr odr;
1141
1142             package.response() = odr.create_close(
1143                 apdu,
1144                 Z_Close_protocolError,
1145                 "double init");
1146
1147             package.session().close();
1148         }
1149         else if (apdu->which == Z_APDU_searchRequest)
1150         {
1151             f->search(package, apdu);
1152         }
1153         else if (apdu->which == Z_APDU_presentRequest)
1154         {
1155             f->present(package, apdu);
1156         }
1157         else if (apdu->which == Z_APDU_scanRequest)
1158         {
1159             f->scan2(package, apdu);
1160         }
1161         else
1162         {
1163             mp::odr odr;
1164
1165             package.response() = odr.create_close(
1166                 apdu, Z_Close_protocolError,
1167                 "unsupported APDU in filter multi");
1168
1169             package.session().close();
1170         }
1171     }
1172     m_p->release_frontend(package);
1173 }
1174
1175 void mp::filter::Multi::configure(const xmlNode * ptr)
1176 {
1177     for (ptr = ptr->children; ptr; ptr = ptr->next)
1178     {
1179         if (ptr->type != XML_ELEMENT_NODE)
1180             continue;
1181         if (!strcmp((const char *) ptr->name, "target"))
1182         {
1183             std::string route = mp::xml::get_route(ptr);
1184             std::string target = mp::xml::get_text(ptr);
1185             std::cout << "route=" << route << " target=" << target << "\n";
1186             m_p->m_target_route[target] = route;
1187         }
1188         else if (!strcmp((const char *) ptr->name, "hideunavailable"))
1189         {
1190             m_p->m_hide_unavailable = true;
1191         }
1192         else if (!strcmp((const char *) ptr->name, "mergetype"))
1193         {
1194             std::string mergetype = mp::xml::get_text(ptr);
1195             if (mergetype == "roundrobin")
1196                 m_p->m_merge_type = round_robin;
1197             else if (mergetype == "serveorder")
1198                 m_p->m_merge_type = serve_order;
1199             else
1200                 throw mp::filter::FilterException
1201                     ("Bad mergetype "  + mergetype + " in multi filter");
1202
1203         }
1204         else
1205         {
1206             throw mp::filter::FilterException
1207                 ("Bad element "
1208                  + std::string((const char *) ptr->name)
1209                  + " in multi filter");
1210         }
1211     }
1212 }
1213
1214 static mp::filter::Base* filter_creator()
1215 {
1216     return new mp::filter::Multi;
1217 }
1218
1219 extern "C" {
1220     struct metaproxy_1_filter_struct metaproxy_1_filter_multi = {
1221         0,
1222         "multi",
1223         filter_creator
1224     };
1225 }
1226
1227
1228 /*
1229  * Local variables:
1230  * c-basic-offset: 4
1231  * indent-tabs-mode: nil
1232  * c-file-style: "stroustrup"
1233  * End:
1234  * vim: shiftwidth=4 tabstop=8 expandtab
1235  */