src/filter_multi.cpp

   1 /* $Id: filter_multi.cpp,v 1.29 2007-11-26 21:21:12 adam Exp $
   2    Copyright (c) 2005-2007, Index Data.
   3
   4 This file is part of Metaproxy.
   5
   6 Metaproxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with Metaproxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #include <yaz/log.h>
  23
  24 #include "config.hpp"
  25
  26 #include "filter.hpp"
  27 #include "package.hpp"
  28
  29 #include <boost/thread/thread.hpp>
  30 #include <boost/thread/mutex.hpp>
  31 #include <boost/thread/condition.hpp>
  32 #include <boost/shared_ptr.hpp>
  33
  34 #include "util.hpp"
  35 #include "filter_multi.hpp"
  36
  37 #include <yaz/zgdu.h>
  38 #include <yaz/otherinfo.h>
  39 #include <yaz/diagbib1.h>
  40
  41 #include <vector>
  42 #include <algorithm>
  43 #include <map>
  44 #include <iostream>
  45
  46 namespace mp = metaproxy_1;
  47 namespace yf = mp::filter;
  48
  49 namespace metaproxy_1 {
  50     namespace filter {
  51         enum multi_merge_type {
  52             round_robin,
  53             serve_order
  54         };
  55         struct Multi::BackendSet {
  56             BackendPtr m_backend;
  57             int m_count;
  58             bool operator < (const BackendSet &k) const;
  59             bool operator == (const BackendSet &k) const;
  60         };
  61         struct Multi::ScanTermInfo {
  62             std::string m_norm_term;
  63             std::string m_display_term;
  64             int m_count;
  65             bool operator < (const ScanTermInfo &) const;
  66             bool operator == (const ScanTermInfo &) const;
  67             Z_Entry *get_entry(ODR odr);
  68         };
  69         struct Multi::FrontendSet {
  70             class PresentJob {
  71             public:
  72                 BackendPtr m_backend;
  73                 int m_pos; // position for backend (1=first, 2=second,..
  74                 int m_start; // present request start
  75                 PresentJob(BackendPtr ptr, int pos) :
  76                     m_backend(ptr), m_pos(pos), m_start(0) {};
  77             };
  78             FrontendSet(std::string setname);
  79             FrontendSet();
  80             ~FrontendSet();
  81
  82             void round_robin(int pos, int number, std::list<PresentJob> &job);
  83             void serve_order(int pos, int number, std::list<PresentJob> &job);
  84
  85             std::list<BackendSet> m_backend_sets;
  86             std::string m_setname;
  87         };
  88         struct Multi::Backend {
  89             PackagePtr m_package;
  90             std::string m_backend_database;
  91             std::string m_vhost;
  92             std::string m_route;
  93             void operator() (void);  // thread operation
  94         };
  95         struct Multi::Frontend {
  96             Frontend(Rep *rep);
  97             ~Frontend();
  98             bool m_is_multi;
  99             bool m_in_use;
 100             std::list<BackendPtr> m_backend_list;
 101             std::map<std::string,Multi::FrontendSet> m_sets;
 102
 103             void multi_move(std::list<BackendPtr> &blist);
 104             void init(Package &package, Z_GDU *gdu);
 105             void close(Package &package);
 106             void search(Package &package, Z_APDU *apdu);
 107             void present(Package &package, Z_APDU *apdu);
 108             void scan1(Package &package, Z_APDU *apdu);
 109             void scan2(Package &package, Z_APDU *apdu);
 110             Rep *m_p;
 111         };
 112         struct Multi::Map {
 113             Map(std::list<std::string> hosts, std::string route);
 114             Map();
 115             std::list<std::string> m_hosts;
 116             std::string m_route;
 117         };
 118         class Multi::Rep {
 119             friend class Multi;
 120             friend struct Frontend;
 121
 122             Rep();
 123             FrontendPtr get_frontend(Package &package);
 124             void release_frontend(Package &package);
 125         private:
 126             std::map<std::string,std::string> m_target_route;
 127             boost::mutex m_mutex;
 128             boost::condition m_cond_session_ready;
 129             std::map<mp::Session, FrontendPtr> m_clients;
 130             bool m_hide_unavailable;
 131             multi_merge_type m_merge_type;
 132         };
 133     }
 134 }
 135
 136 yf::Multi::Rep::Rep()
 137 {
 138     m_hide_unavailable = false;
 139     m_merge_type = round_robin;
 140 }
 141
 142 bool yf::Multi::BackendSet::operator < (const BackendSet &k) const
 143 {
 144     return m_count < k.m_count;
 145 }
 146
 147 yf::Multi::Frontend::Frontend(Rep *rep)
 148 {
 149     m_p = rep;
 150     m_is_multi = false;
 151 }
 152
 153 yf::Multi::Frontend::~Frontend()
 154 {
 155 }
 156
 157 yf::Multi::FrontendPtr yf::Multi::Rep::get_frontend(mp::Package &package)
 158 {
 159     boost::mutex::scoped_lock lock(m_mutex);
 160
 161     std::map<mp::Session,yf::Multi::FrontendPtr>::iterator it;
 162
 163     while(true)
 164     {
 165         it = m_clients.find(package.session());
 166         if (it == m_clients.end())
 167             break;
 168
 169         if (!it->second->m_in_use)
 170         {
 171             it->second->m_in_use = true;
 172             return it->second;
 173         }
 174         m_cond_session_ready.wait(lock);
 175     }
 176     FrontendPtr f(new Frontend(this));
 177     m_clients[package.session()] = f;
 178     f->m_in_use = true;
 179     return f;
 180 }
 181
 182 void yf::Multi::Rep::release_frontend(mp::Package &package)
 183 {
 184     boost::mutex::scoped_lock lock(m_mutex);
 185     std::map<mp::Session,yf::Multi::FrontendPtr>::iterator it;
 186
 187     it = m_clients.find(package.session());
 188     if (it != m_clients.end())
 189     {
 190         if (package.session().is_closed())
 191         {
 192             it->second->close(package);
 193             m_clients.erase(it);
 194         }
 195         else
 196         {
 197             it->second->m_in_use = false;
 198         }
 199         m_cond_session_ready.notify_all();
 200     }
 201 }
 202
 203 yf::Multi::FrontendSet::FrontendSet(std::string setname)
 204     :  m_setname(setname)
 205 {
 206 }
 207
 208
 209 yf::Multi::FrontendSet::FrontendSet()
 210 {
 211 }
 212
 213
 214 yf::Multi::FrontendSet::~FrontendSet()
 215 {
 216 }
 217
 218 yf::Multi::Map::Map(std::list<std::string> hosts, std::string route)
 219     : m_hosts(hosts), m_route(route)
 220 {
 221 }
 222
 223 yf::Multi::Map::Map()
 224 {
 225 }
 226
 227 yf::Multi::Multi() : m_p(new Multi::Rep)
 228 {
 229 }
 230
 231 yf::Multi::~Multi() {
 232 }
 233
 234
 235 void yf::Multi::Backend::operator() (void)
 236 {
 237     m_package->move(m_route);
 238 }
 239
 240
 241 void yf::Multi::Frontend::close(mp::Package &package)
 242 {
 243     std::list<BackendPtr>::const_iterator bit;
 244     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 245     {
 246         BackendPtr b = *bit;
 247
 248         b->m_package->copy_filter(package);
 249         b->m_package->request() = (Z_GDU *) 0;
 250         b->m_package->session().close();
 251         b->m_package->move(b->m_route);
 252     }
 253 }
 254
 255 void yf::Multi::Frontend::multi_move(std::list<BackendPtr> &blist)
 256 {
 257     std::list<BackendPtr>::const_iterator bit;
 258     boost::thread_group g;
 259     for (bit = blist.begin(); bit != blist.end(); bit++)
 260     {
 261         g.add_thread(new boost::thread(**bit));
 262     }
 263     g.join_all();
 264 }
 265
 266 void yf::Multi::FrontendSet::serve_order(int start, int number,
 267                                          std::list<PresentJob> &jobs)
 268 {
 269     int i;
 270     for (i = 0; i < number; i++)
 271     {
 272         std::list<BackendSet>::const_iterator bsit;
 273         int voffset = 0;
 274         int offset = start + i - 1;
 275         for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end();
 276              bsit++)
 277         {
 278             if (offset >= voffset && offset < voffset + bsit->m_count)
 279             {
 280                 PresentJob job(bsit->m_backend, offset - voffset + 1);
 281                 jobs.push_back(job);
 282                 break;
 283             }
 284             voffset += bsit->m_count;
 285         }
 286     }
 287 }
 288
 289 void yf::Multi::FrontendSet::round_robin(int start, int number,
 290                                          std::list<PresentJob> &jobs)
 291 {
 292     std::list<int> pos;
 293     std::list<BackendSet>::const_iterator bsit;
 294     for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); bsit++)
 295     {
 296         pos.push_back(1);
 297     }
 298
 299     int p = 1;
 300 #if 1
 301     // optimization step!
 302     int omin = 0;
 303     while(true)
 304     {
 305         int min = 0;
 306         int no_left = 0;
 307         // find min count for each set which is > omin
 308         for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); bsit++)
 309         {
 310             if (bsit->m_count > omin)
 311             {
 312                 if (no_left == 0 || bsit->m_count < min)
 313                     min = bsit->m_count;
 314                 no_left++;
 315             }
 316         }
 317         if (no_left == 0) // if nothing greater than omin, bail out.
 318             break;
 319         int skip = no_left * min;
 320         if (p + skip > start)  // step gets us "into" present range?
 321         {
 322             // Yes. skip until start.. Rounding off is deliberate!
 323             min = (start-p) / no_left;
 324             p += no_left * min;
 325
 326             // update positions in each set..
 327             std::list<int>::iterator psit = pos.begin();
 328             for (psit = pos.begin(); psit != pos.end(); psit++)
 329                 *psit += min;
 330             break;
 331         }
 332         // skip on each set.. before "present range"..
 333         p = p + skip;
 334
 335         std::cout << "\nSKIP min=" << min << " no_left=" << no_left << "\n\n";
 336
 337         std::list<int>::iterator psit = pos.begin();
 338         for (psit = pos.begin(); psit != pos.end(); psit++)
 339             *psit += min;
 340
 341         omin = min; // update so we consider next class (with higher count)
 342     }
 343 #endif
 344     int fetched = 0;
 345     bool more = true;
 346     while (more)
 347     {
 348         more = false;
 349         std::list<int>::iterator psit = pos.begin();
 350         bsit = m_backend_sets.begin();
 351
 352         for (; bsit != m_backend_sets.end(); psit++,bsit++)
 353         {
 354             if (fetched >= number)
 355             {
 356                 more = false;
 357                 break;
 358             }
 359             if (*psit <= bsit->m_count)
 360             {
 361                 if (p >= start)
 362                 {
 363                     PresentJob job(bsit->m_backend, *psit);
 364                     jobs.push_back(job);
 365                     fetched++;
 366                 }
 367                 (*psit)++;
 368                 p++;
 369                 more = true;
 370             }
 371         }
 372     }
 373 }
 374
 375 void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu)
 376 {
 377     Z_InitRequest *req = gdu->u.z3950->u.initRequest;
 378
 379     std::list<std::string> targets;
 380
 381     mp::util::get_vhost_otherinfo(req->otherInfo, targets);
 382
 383     if (targets.size() < 1)
 384     {
 385         package.move();
 386         return;
 387     }
 388
 389     std::list<std::string>::const_iterator t_it = targets.begin();
 390     for (; t_it != targets.end(); t_it++)
 391     {
 392         Session s;
 393         Backend *b = new Backend;
 394         b->m_vhost = *t_it;
 395
 396         b->m_route = m_p->m_target_route[*t_it];
 397         // b->m_route unset
 398         b->m_package = PackagePtr(new Package(s, package.origin()));
 399
 400         m_backend_list.push_back(BackendPtr(b));
 401     }
 402     m_is_multi = true;
 403
 404     // create init request
 405     std::list<BackendPtr>::iterator bit;
 406     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 407     {
 408         mp::odr odr;
 409         BackendPtr b = *bit;
 410         Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest);
 411
 412         std::list<std::string>vhost_one;
 413         vhost_one.push_back(b->m_vhost);
 414         mp::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo,
 415                                        odr, vhost_one);
 416
 417         Z_InitRequest *req = init_apdu->u.initRequest;
 418
 419         ODR_MASK_SET(req->options, Z_Options_search);
 420         ODR_MASK_SET(req->options, Z_Options_present);
 421         ODR_MASK_SET(req->options, Z_Options_namedResultSets);
 422         ODR_MASK_SET(req->options, Z_Options_scan);
 423
 424         ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
 425         ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
 426         ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3);
 427
 428         b->m_package->request() = init_apdu;
 429
 430         b->m_package->copy_filter(package);
 431     }
 432     multi_move(m_backend_list);
 433
 434     // create the frontend init response based on each backend init response
 435     mp::odr odr;
 436
 437     Z_APDU *f_apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
 438     Z_InitResponse *f_resp = f_apdu->u.initResponse;
 439
 440     ODR_MASK_SET(f_resp->options, Z_Options_search);
 441     ODR_MASK_SET(f_resp->options, Z_Options_present);
 442     ODR_MASK_SET(f_resp->options, Z_Options_namedResultSets);
 443
 444     ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_1);
 445     ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_2);
 446     ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_3);
 447
 448     int no_failed = 0;
 449     int no_succeeded = 0;
 450     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); )
 451     {
 452         PackagePtr p = (*bit)->m_package;
 453
 454         if (p->session().is_closed())
 455         {
 456             // failed. Remove from list and increment number of failed
 457             no_failed++;
 458             bit = m_backend_list.erase(bit);
 459             continue;
 460         }
 461         no_succeeded++;
 462
 463         Z_GDU *gdu = p->response().get();
 464         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
 465             Z_APDU_initResponse)
 466         {
 467             int i;
 468             Z_APDU *b_apdu = gdu->u.z3950;
 469             Z_InitResponse *b_resp = b_apdu->u.initResponse;
 470
 471             // common options for all backends
 472             for (i = 0; i <= Z_Options_stringSchema; i++)
 473             {
 474                 if (!ODR_MASK_GET(b_resp->options, i))
 475                     ODR_MASK_CLEAR(f_resp->options, i);
 476             }
 477             // common protocol version
 478             for (i = 0; i <= Z_ProtocolVersion_3; i++)
 479                 if (!ODR_MASK_GET(b_resp->protocolVersion, i))
 480                     ODR_MASK_CLEAR(f_resp->protocolVersion, i);
 481             // reject if any of the backends reject
 482             if (!*b_resp->result)
 483                 *f_resp->result = 0;
 484         }
 485         else
 486         {
 487             // if any target does not return init return that (close or
 488             // similar )
 489             package.response() = p->response();
 490             return;
 491         }
 492         bit++;
 493     }
 494     if (m_p->m_hide_unavailable)
 495     {
 496         if (no_succeeded == 0)
 497             package.session().close();
 498     }
 499     else
 500     {
 501         if (no_failed)
 502             package.session().close();
 503     }
 504     package.response() = f_apdu;
 505 }
 506
 507 void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
 508 {
 509     // create search request
 510     Z_SearchRequest *req = apdu_req->u.searchRequest;
 511
 512     // save these for later
 513     int smallSetUpperBound = *req->smallSetUpperBound;
 514     int largeSetLowerBound = *req->largeSetLowerBound;
 515     int mediumSetPresentNumber = *req->mediumSetPresentNumber;
 516
 517     // they are altered now - to disable piggyback
 518     *req->smallSetUpperBound = 0;
 519     *req->largeSetLowerBound = 1;
 520     *req->mediumSetPresentNumber = 1;
 521
 522     int default_num_db = req->num_databaseNames;
 523     char **default_db = req->databaseNames;
 524
 525     std::list<BackendPtr>::const_iterator bit;
 526     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 527     {
 528         PackagePtr p = (*bit)->m_package;
 529         mp::odr odr;
 530
 531         if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
 532                                                 &req->num_databaseNames,
 533                                                 &req->databaseNames))
 534         {
 535             req->num_databaseNames = default_num_db;
 536             req->databaseNames = default_db;
 537         }
 538         p->request() = apdu_req;
 539         p->copy_filter(package);
 540     }
 541     multi_move(m_backend_list);
 542
 543     // look at each response
 544     FrontendSet resultSet(std::string(req->resultSetName));
 545
 546     int result_set_size = 0;
 547     Z_Records *z_records_diag = 0;  // no diagnostics (yet)
 548     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 549     {
 550         PackagePtr p = (*bit)->m_package;
 551
 552         if (p->session().is_closed()) // if any backend closes, close frontend
 553             package.session().close();
 554
 555         Z_GDU *gdu = p->response().get();
 556         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
 557             Z_APDU_searchResponse)
 558         {
 559             Z_APDU *b_apdu = gdu->u.z3950;
 560             Z_SearchResponse *b_resp = b_apdu->u.searchResponse;
 561
 562             // see we get any errors (AKA diagnstics)
 563             if (b_resp->records)
 564             {
 565                 if (b_resp->records->which == Z_Records_NSD
 566                     || b_resp->records->which == Z_Records_multipleNSD)
 567                     z_records_diag = b_resp->records;
 568                 // we may set this multiple times (TOO BAD!)
 569             }
 570             BackendSet backendSet;
 571             backendSet.m_backend = *bit;
 572             backendSet.m_count = *b_resp->resultCount;
 573             result_set_size += *b_resp->resultCount;
 574             resultSet.m_backend_sets.push_back(backendSet);
 575         }
 576         else
 577         {
 578             // if any target does not return search response - return that
 579             package.response() = p->response();
 580             return;
 581         }
 582     }
 583
 584     mp::odr odr;
 585     Z_APDU *f_apdu = odr.create_searchResponse(apdu_req, 0, 0);
 586     Z_SearchResponse *f_resp = f_apdu->u.searchResponse;
 587
 588     *f_resp->resultCount = result_set_size;
 589     if (z_records_diag)
 590     {
 591         // search error
 592         f_resp->records = z_records_diag;
 593         package.response() = f_apdu;
 594         return;
 595     }
 596     // assume OK
 597     m_sets[resultSet.m_setname] = resultSet;
 598
 599     int number;
 600     mp::util::piggyback(smallSetUpperBound,
 601                          largeSetLowerBound,
 602                          mediumSetPresentNumber,
 603                          result_set_size,
 604                          number);
 605     Package pp(package.session(), package.origin());
 606     if (number > 0)
 607     {
 608         pp.copy_filter(package);
 609         Z_APDU *p_apdu = zget_APDU(odr, Z_APDU_presentRequest);
 610         Z_PresentRequest *p_req = p_apdu->u.presentRequest;
 611         p_req->preferredRecordSyntax = req->preferredRecordSyntax;
 612         p_req->resultSetId = req->resultSetName;
 613         *p_req->resultSetStartPoint = 1;
 614         *p_req->numberOfRecordsRequested = number;
 615         pp.request() = p_apdu;
 616         present(pp, p_apdu);
 617
 618         if (pp.session().is_closed())
 619             package.session().close();
 620
 621         Z_GDU *gdu = pp.response().get();
 622         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
 623             Z_APDU_presentResponse)
 624         {
 625             Z_PresentResponse *p_res = gdu->u.z3950->u.presentResponse;
 626             f_resp->records = p_res->records;
 627             *f_resp->numberOfRecordsReturned =
 628                 *p_res->numberOfRecordsReturned;
 629             *f_resp->nextResultSetPosition =
 630                 *p_res->nextResultSetPosition;
 631         }
 632         else
 633         {
 634             package.response() = pp.response();
 635             return;
 636         }
 637     }
 638     package.response() = f_apdu; // in this scope because of p
 639 }
 640
 641 void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
 642 {
 643     // create present request
 644     Z_PresentRequest *req = apdu_req->u.presentRequest;
 645
 646     Sets_it it;
 647     it = m_sets.find(std::string(req->resultSetId));
 648     if (it == m_sets.end())
 649     {
 650         mp::odr odr;
 651         Z_APDU *apdu =
 652             odr.create_presentResponse(
 653                 apdu_req,
 654                 YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 655                 req->resultSetId);
 656         package.response() = apdu;
 657         return;
 658     }
 659     std::list<Multi::FrontendSet::PresentJob> jobs;
 660     int start = *req->resultSetStartPoint;
 661     int number = *req->numberOfRecordsRequested;
 662
 663     if (m_p->m_merge_type == round_robin)
 664         it->second.round_robin(start, number, jobs);
 665     else if (m_p->m_merge_type == serve_order)
 666         it->second.serve_order(start, number, jobs);
 667
 668     if (0)
 669     {
 670         std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
 671         for (jit = jobs.begin(); jit != jobs.end(); jit++)
 672         {
 673             yaz_log(YLOG_LOG, "job pos=%d", jit->m_pos);
 674         }
 675     }
 676
 677     std::list<BackendPtr> present_backend_list;
 678
 679     std::list<BackendSet>::const_iterator bsit;
 680     bsit = it->second.m_backend_sets.begin();
 681     for (; bsit != it->second.m_backend_sets.end(); bsit++)
 682     {
 683         int start = -1;
 684         int end = -1;
 685         {
 686             std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
 687             for (jit = jobs.begin(); jit != jobs.end(); jit++)
 688             {
 689                 if (jit->m_backend == bsit->m_backend)
 690                 {
 691                     if (start == -1 || jit->m_pos < start)
 692                         start = jit->m_pos;
 693                     if (end == -1 || jit->m_pos > end)
 694                         end = jit->m_pos;
 695                 }
 696             }
 697         }
 698         if (start != -1)
 699         {
 700             std::list<Multi::FrontendSet::PresentJob>::iterator jit;
 701             for (jit = jobs.begin(); jit != jobs.end(); jit++)
 702             {
 703                 if (jit->m_backend == bsit->m_backend)
 704                 {
 705                     if (jit->m_pos >= start && jit->m_pos <= end)
 706                         jit->m_start = start;
 707                 }
 708             }
 709
 710             PackagePtr p = bsit->m_backend->m_package;
 711
 712             *req->resultSetStartPoint = start;
 713             *req->numberOfRecordsRequested = end - start + 1;
 714
 715             p->request() = apdu_req;
 716             p->copy_filter(package);
 717
 718             present_backend_list.push_back(bsit->m_backend);
 719         }
 720     }
 721     multi_move(present_backend_list);
 722
 723     // look at each response
 724     Z_Records *z_records_diag = 0;
 725
 726     std::list<BackendPtr>::const_iterator pbit = present_backend_list.begin();
 727     for (; pbit != present_backend_list.end(); pbit++)
 728     {
 729         PackagePtr p = (*pbit)->m_package;
 730
 731         if (p->session().is_closed()) // if any backend closes, close frontend
 732             package.session().close();
 733
 734         Z_GDU *gdu = p->response().get();
 735         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
 736             Z_APDU_presentResponse)
 737         {
 738             Z_APDU *b_apdu = gdu->u.z3950;
 739             Z_PresentResponse *b_resp = b_apdu->u.presentResponse;
 740
 741             // see we get any errors (AKA diagnstics)
 742             if (b_resp->records)
 743             {
 744                 if (b_resp->records->which != Z_Records_DBOSD)
 745                     z_records_diag = b_resp->records;
 746                 // we may set this multiple times (TOO BAD!)
 747             }
 748         }
 749         else
 750         {
 751             // if any target does not return present response - return that
 752             package.response() = p->response();
 753             return;
 754         }
 755     }
 756
 757     mp::odr odr;
 758     Z_APDU *f_apdu = odr.create_presentResponse(apdu_req, 0, 0);
 759     Z_PresentResponse *f_resp = f_apdu->u.presentResponse;
 760
 761     if (z_records_diag)
 762     {
 763         f_resp->records = z_records_diag;
 764         *f_resp->presentStatus = Z_PresentStatus_failure;
 765     }
 766     else
 767     {
 768         f_resp->records = (Z_Records *) odr_malloc(odr, sizeof(Z_Records));
 769         Z_Records * records = f_resp->records;
 770         records->which = Z_Records_DBOSD;
 771         records->u.databaseOrSurDiagnostics =
 772             (Z_NamePlusRecordList *)
 773             odr_malloc(odr, sizeof(Z_NamePlusRecordList));
 774         Z_NamePlusRecordList *nprl = records->u.databaseOrSurDiagnostics;
 775         nprl->num_records = jobs.size();
 776         nprl->records = (Z_NamePlusRecord**)
 777             odr_malloc(odr, sizeof(Z_NamePlusRecord *) * nprl->num_records);
 778         int i = 0;
 779         std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
 780         for (jit = jobs.begin(); jit != jobs.end(); jit++, i++)
 781         {
 782             PackagePtr p = jit->m_backend->m_package;
 783
 784             Z_GDU *gdu = p->response().get();
 785             Z_APDU *b_apdu = gdu->u.z3950;
 786             Z_PresentResponse *b_resp = b_apdu->u.presentResponse;
 787
 788             nprl->records[i] = (Z_NamePlusRecord*)
 789                 odr_malloc(odr, sizeof(Z_NamePlusRecord));
 790             int inside_pos = jit->m_pos - jit->m_start;
 791             if (inside_pos >= b_resp->records->
 792                 u.databaseOrSurDiagnostics->num_records)
 793                 break;
 794             *nprl->records[i] = *b_resp->records->
 795                 u.databaseOrSurDiagnostics->records[inside_pos];
 796             nprl->records[i]->databaseName =
 797                     odr_strdup(odr, jit->m_backend->m_vhost.c_str());
 798         }
 799         nprl->num_records = i; // usually same as jobs.size();
 800         *f_resp->nextResultSetPosition = start + i;
 801         *f_resp->numberOfRecordsReturned = i;
 802     }
 803     package.response() = f_apdu;
 804 }
 805
 806 void yf::Multi::Frontend::scan1(mp::Package &package, Z_APDU *apdu_req)
 807 {
 808     if (m_backend_list.size() > 1)
 809     {
 810         mp::odr odr;
 811         Z_APDU *f_apdu =
 812             odr.create_scanResponse(
 813                 apdu_req, YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP, 0);
 814         package.response() = f_apdu;
 815         return;
 816     }
 817     Z_ScanRequest *req = apdu_req->u.scanRequest;
 818
 819     int default_num_db = req->num_databaseNames;
 820     char **default_db = req->databaseNames;
 821
 822     std::list<BackendPtr>::const_iterator bit;
 823     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 824     {
 825         PackagePtr p = (*bit)->m_package;
 826         mp::odr odr;
 827
 828         if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
 829                                                 &req->num_databaseNames,
 830                                                 &req->databaseNames))
 831         {
 832             req->num_databaseNames = default_num_db;
 833             req->databaseNames = default_db;
 834         }
 835         p->request() = apdu_req;
 836         p->copy_filter(package);
 837     }
 838     multi_move(m_backend_list);
 839
 840     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 841     {
 842         PackagePtr p = (*bit)->m_package;
 843
 844         if (p->session().is_closed()) // if any backend closes, close frontend
 845             package.session().close();
 846
 847         Z_GDU *gdu = p->response().get();
 848         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
 849             Z_APDU_scanResponse)
 850         {
 851             package.response() = p->response();
 852             break;
 853         }
 854         else
 855         {
 856             // if any target does not return scan response - return that
 857             package.response() = p->response();
 858             return;
 859         }
 860     }
 861 }
 862
 863 bool yf::Multi::ScanTermInfo::operator < (const ScanTermInfo &k) const
 864 {
 865     return m_norm_term < k.m_norm_term;
 866 }
 867
 868 bool yf::Multi::ScanTermInfo::operator == (const ScanTermInfo &k) const
 869 {
 870     return m_norm_term == k.m_norm_term;
 871 }
 872
 873 Z_Entry *yf::Multi::ScanTermInfo::get_entry(ODR odr)
 874 {
 875     Z_Entry *e = (Z_Entry *)odr_malloc(odr, sizeof(*e));
 876     e->which = Z_Entry_termInfo;
 877     Z_TermInfo *t;
 878     t = e->u.termInfo = (Z_TermInfo *) odr_malloc(odr, sizeof(*t));
 879     t->suggestedAttributes = 0;
 880     t->displayTerm = 0;
 881     t->alternativeTerm = 0;
 882     t->byAttributes = 0;
 883     t->otherTermInfo = 0;
 884     t->globalOccurrences = odr_intdup(odr, m_count);
 885     t->term = (Z_Term *)
 886         odr_malloc(odr, sizeof(*t->term));
 887     t->term->which = Z_Term_general;
 888     Odr_oct *o;
 889     t->term->u.general = o = (Odr_oct *)odr_malloc(odr, sizeof(Odr_oct));
 890
 891     o->len = o->size = m_norm_term.size();
 892     o->buf = (unsigned char *) odr_malloc(odr, o->len);
 893     memcpy(o->buf, m_norm_term.c_str(), o->len);
 894     return e;
 895 }
 896
 897 void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req)
 898 {
 899     Z_ScanRequest *req = apdu_req->u.scanRequest;
 900
 901     int default_num_db = req->num_databaseNames;
 902     char **default_db = req->databaseNames;
 903
 904     std::list<BackendPtr>::const_iterator bit;
 905     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 906     {
 907         PackagePtr p = (*bit)->m_package;
 908         mp::odr odr;
 909
 910         if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
 911                                                 &req->num_databaseNames,
 912                                                 &req->databaseNames))
 913         {
 914             req->num_databaseNames = default_num_db;
 915             req->databaseNames = default_db;
 916         }
 917         p->request() = apdu_req;
 918         p->copy_filter(package);
 919     }
 920     multi_move(m_backend_list);
 921
 922     ScanTermInfoList entries_before;
 923     ScanTermInfoList entries_after;
 924     int no_before = 0;
 925     int no_after = 0;
 926
 927     for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
 928     {
 929         PackagePtr p = (*bit)->m_package;
 930
 931         if (p->session().is_closed()) // if any backend closes, close frontend
 932             package.session().close();
 933
 934         Z_GDU *gdu = p->response().get();
 935         if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
 936             Z_APDU_scanResponse)
 937         {
 938             Z_ScanResponse *res = gdu->u.z3950->u.scanResponse;
 939
 940             if (res->entries && res->entries->nonsurrogateDiagnostics)
 941             {
 942                 // failure
 943                 mp::odr odr;
 944                 Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 1, 0);
 945                 Z_ScanResponse *f_res = f_apdu->u.scanResponse;
 946
 947                 f_res->entries->nonsurrogateDiagnostics =
 948                     res->entries->nonsurrogateDiagnostics;
 949                 f_res->entries->num_nonsurrogateDiagnostics =
 950                     res->entries->num_nonsurrogateDiagnostics;
 951
 952                 package.response() = f_apdu;
 953                 return;
 954             }
 955
 956             if (res->entries && res->entries->entries)
 957             {
 958                 Z_Entry **entries = res->entries->entries;
 959                 int num_entries = res->entries->num_entries;
 960                 int position = 1;
 961                 if (req->preferredPositionInResponse)
 962                     position = *req->preferredPositionInResponse;
 963                 if (res->positionOfTerm)
 964                     position = *res->positionOfTerm;
 965
 966                 // before
 967                 int i;
 968                 for (i = 0; i<position-1 && i<num_entries; i++)
 969                 {
 970                     Z_Entry *ent = entries[i];
 971
 972                     if (ent->which == Z_Entry_termInfo)
 973                     {
 974                         ScanTermInfo my;
 975
 976                         int *occur = ent->u.termInfo->globalOccurrences;
 977                         my.m_count = occur ? *occur : 0;
 978
 979                         if (ent->u.termInfo->term->which == Z_Term_general)
 980                         {
 981                             my.m_norm_term = std::string(
 982                                 (const char *)
 983                                 ent->u.termInfo->term->u.general->buf,
 984                                 ent->u.termInfo->term->u.general->len);
 985                         }
 986                         if (my.m_norm_term.length())
 987                         {
 988                             ScanTermInfoList::iterator it =
 989                                 entries_before.begin();
 990                             while (it != entries_before.end() && my <*it)
 991                                 it++;
 992                             if (my == *it)
 993                             {
 994                                 it->m_count += my.m_count;
 995                             }
 996                             else
 997                             {
 998                                 entries_before.insert(it, my);
 999                                 no_before++;
1000                             }
1001                         }
1002                     }
1003                 }
1004                 // after
1005                 if (position <= 0)
1006                     i = 0;
1007                 else
1008                     i = position-1;
1009                 for ( ; i<num_entries; i++)
1010                 {
1011                     Z_Entry *ent = entries[i];
1012
1013                     if (ent->which == Z_Entry_termInfo)
1014                     {
1015                         ScanTermInfo my;
1016
1017                         int *occur = ent->u.termInfo->globalOccurrences;
1018                         my.m_count = occur ? *occur : 0;
1019
1020                         if (ent->u.termInfo->term->which == Z_Term_general)
1021                         {
1022                             my.m_norm_term = std::string(
1023                                 (const char *)
1024                                 ent->u.termInfo->term->u.general->buf,
1025                                 ent->u.termInfo->term->u.general->len);
1026                         }
1027                         if (my.m_norm_term.length())
1028                         {
1029                             ScanTermInfoList::iterator it =
1030                                 entries_after.begin();
1031                             while (it != entries_after.end() && *it < my)
1032                                 it++;
1033                             if (my == *it)
1034                             {
1035                                 it->m_count += my.m_count;
1036                             }
1037                             else
1038                             {
1039                                 entries_after.insert(it, my);
1040                                 no_after++;
1041                             }
1042                         }
1043                     }
1044                 }
1045
1046             }
1047         }
1048         else
1049         {
1050             // if any target does not return scan response - return that
1051             package.response() = p->response();
1052             return;
1053         }
1054     }
1055
1056     if (true)
1057     {
1058         std::cout << "BEFORE\n";
1059         ScanTermInfoList::iterator it = entries_before.begin();
1060         for(; it != entries_before.end(); it++)
1061         {
1062             std::cout << " " << it->m_norm_term << " " << it->m_count << "\n";
1063         }
1064
1065         std::cout << "AFTER\n";
1066         it = entries_after.begin();
1067         for(; it != entries_after.end(); it++)
1068         {
1069             std::cout << " " << it->m_norm_term << " " << it->m_count << "\n";
1070         }
1071     }
1072
1073     if (false)
1074     {
1075         mp::odr odr;
1076         Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 1, "not implemented");
1077         package.response() = f_apdu;
1078     }
1079     else
1080     {
1081         mp::odr odr;
1082         Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 0, 0);
1083         Z_ScanResponse *resp = f_apdu->u.scanResponse;
1084
1085         int number_returned = *req->numberOfTermsRequested;
1086         int position_returned = *req->preferredPositionInResponse;
1087
1088         resp->entries->num_entries = number_returned;
1089         resp->entries->entries = (Z_Entry**)
1090             odr_malloc(odr, sizeof(Z_Entry*) * number_returned);
1091         int i;
1092
1093         int lbefore = entries_before.size();
1094         if (lbefore < position_returned-1)
1095             position_returned = lbefore+1;
1096
1097         ScanTermInfoList::iterator it = entries_before.begin();
1098         for (i = 0; i<position_returned-1 && it != entries_before.end(); i++, it++)
1099         {
1100             resp->entries->entries[position_returned-2-i] = it->get_entry(odr);
1101         }
1102
1103         it = entries_after.begin();
1104
1105         if (position_returned <= 0)
1106             i = 0;
1107         else
1108             i = position_returned-1;
1109         for (; i<number_returned && it != entries_after.end(); i++, it++)
1110         {
1111             resp->entries->entries[i] = it->get_entry(odr);
1112         }
1113
1114         number_returned = i;
1115
1116         resp->positionOfTerm = odr_intdup(odr, position_returned);
1117         resp->numberOfEntriesReturned = odr_intdup(odr, number_returned);
1118         resp->entries->num_entries = number_returned;
1119
1120         package.response() = f_apdu;
1121     }
1122 }
1123
1124
1125 void yf::Multi::process(mp::Package &package) const
1126 {
1127     FrontendPtr f = m_p->get_frontend(package);
1128
1129     Z_GDU *gdu = package.request().get();
1130
1131     if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
1132         Z_APDU_initRequest && !f->m_is_multi)
1133     {
1134         f->init(package, gdu);
1135     }
1136     else if (!f->m_is_multi)
1137         package.move();
1138     else if (gdu && gdu->which == Z_GDU_Z3950)
1139     {
1140         Z_APDU *apdu = gdu->u.z3950;
1141         if (apdu->which == Z_APDU_initRequest)
1142         {
1143             mp::odr odr;
1144
1145             package.response() = odr.create_close(
1146                 apdu,
1147                 Z_Close_protocolError,
1148                 "double init");
1149
1150             package.session().close();
1151         }
1152         else if (apdu->which == Z_APDU_searchRequest)
1153         {
1154             f->search(package, apdu);
1155         }
1156         else if (apdu->which == Z_APDU_presentRequest)
1157         {
1158             f->present(package, apdu);
1159         }
1160         else if (apdu->which == Z_APDU_scanRequest)
1161         {
1162             f->scan2(package, apdu);
1163         }
1164         else
1165         {
1166             mp::odr odr;
1167
1168             package.response() = odr.create_close(
1169                 apdu, Z_Close_protocolError,
1170                 "unsupported APDU in filter multi");
1171
1172             package.session().close();
1173         }
1174     }
1175     m_p->release_frontend(package);
1176 }
1177
1178 void mp::filter::Multi::configure(const xmlNode * ptr)
1179 {
1180     for (ptr = ptr->children; ptr; ptr = ptr->next)
1181     {
1182         if (ptr->type != XML_ELEMENT_NODE)
1183             continue;
1184         if (!strcmp((const char *) ptr->name, "target"))
1185         {
1186             std::string route = mp::xml::get_route(ptr);
1187             std::string target = mp::xml::get_text(ptr);
1188             std::cout << "route=" << route << " target=" << target << "\n";
1189             m_p->m_target_route[target] = route;
1190         }
1191         else if (!strcmp((const char *) ptr->name, "hideunavailable"))
1192         {
1193             m_p->m_hide_unavailable = true;
1194         }
1195         else if (!strcmp((const char *) ptr->name, "mergetype"))
1196         {
1197             std::string mergetype = mp::xml::get_text(ptr);
1198             if (mergetype == "roundrobin")
1199                 m_p->m_merge_type = round_robin;
1200             else if (mergetype == "serveorder")
1201                 m_p->m_merge_type = serve_order;
1202             else
1203                 throw mp::filter::FilterException
1204                     ("Bad mergetype "  + mergetype + " in multi filter");
1205
1206         }
1207         else
1208         {
1209             throw mp::filter::FilterException
1210                 ("Bad element "
1211                  + std::string((const char *) ptr->name)
1212                  + " in multi filter");
1213         }
1214     }
1215 }
1216
1217 static mp::filter::Base* filter_creator()
1218 {
1219     return new mp::filter::Multi;
1220 }
1221
1222 extern "C" {
1223     struct metaproxy_1_filter_struct metaproxy_1_filter_multi = {
1224         0,
1225         "multi",
1226         filter_creator
1227     };
1228 }
1229
1230
1231 /*
1232  * Local variables:
1233  * c-basic-offset: 4
1234  * indent-tabs-mode: nil
1235  * c-file-style: "stroustrup"
1236  * End:
1237  * vim: shiftwidth=4 tabstop=8 expandtab
1238  */