src/yaz-proxy.cpp

   1 /*
   2  * Copyright (c) 1998-2003, Index Data.
   3  * See the file LICENSE for details.
   4  *
   5  * $Id: yaz-proxy.cpp,v 1.54 2003-10-09 12:11:10 adam Exp $
   6  */
   7
   8 #include <assert.h>
   9 #include <time.h>
  10
  11 #include <yaz/log.h>
  12 #include <yaz/diagbib1.h>
  13 #include <yaz++/proxy.h>
  14
  15 static const char *apdu_name(Z_APDU *apdu)
  16 {
  17     switch (apdu->which)
  18     {
  19     case Z_APDU_initRequest:
  20         return "initRequest";
  21     case Z_APDU_initResponse:
  22         return "initResponse";
  23     case Z_APDU_searchRequest:
  24         return "searchRequest";
  25     case Z_APDU_searchResponse:
  26         return "searchResponse";
  27     case Z_APDU_presentRequest:
  28         return "presentRequest";
  29     case Z_APDU_presentResponse:
  30         return "presentResponse";
  31     case Z_APDU_deleteResultSetRequest:
  32         return "deleteResultSetRequest";
  33     case Z_APDU_deleteResultSetResponse:
  34         return "deleteResultSetResponse";
  35     case Z_APDU_scanRequest:
  36         return "scanRequest";
  37     case Z_APDU_scanResponse:
  38         return "scanResponse";
  39     case Z_APDU_sortRequest:
  40         return "sortRequest";
  41     case Z_APDU_sortResponse:
  42         return "sortResponse";
  43     case Z_APDU_extendedServicesRequest:
  44         return "extendedServicesRequest";
  45     case Z_APDU_extendedServicesResponse:
  46         return "extendedServicesResponse";
  47     case Z_APDU_close:
  48         return "close";
  49     }
  50     return "other";
  51 }
  52
  53 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable) :
  54     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
  55 {
  56     m_PDU_Observable = the_PDU_Observable;
  57     m_client = 0;
  58     m_parent = 0;
  59     m_clientPool = 0;
  60     m_seqno = 1;
  61     m_keepalive_limit_bw = 500000;
  62     m_keepalive_limit_pdu = 1000;
  63     m_proxyTarget = 0;
  64     m_default_target = 0;
  65     m_proxy_authentication = 0;
  66     m_max_clients = 150;
  67     m_seed = time(0);
  68     m_client_idletime = 600;
  69     m_target_idletime = 600;
  70     m_optimize = xstrdup ("1");
  71     strcpy(m_session_str, "x");
  72     m_session_no=0;
  73     m_bytes_sent = m_bytes_recv = 0;
  74     m_bw_hold_PDU = 0;
  75     m_bw_max = 0;
  76     m_pdu_max = 0;
  77     m_max_record_retrieve = 0;
  78     m_reconfig_flag = 0;
  79     m_config_fname = 0;
  80     m_request_no = 0;
  81     m_invalid_session = 0;
  82 }
  83
  84 Yaz_Proxy::~Yaz_Proxy()
  85 {
  86     yaz_log(LOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
  87             m_bytes_sent, m_bytes_recv);
  88     xfree (m_proxyTarget);
  89     xfree (m_default_target);
  90     xfree (m_proxy_authentication);
  91     xfree (m_optimize);
  92     if (m_parent)
  93         m_parent->check_reconfigure();
  94 }
  95
  96 int Yaz_Proxy::set_config(const char *config)
  97 {
  98     xfree(m_config_fname);
  99     m_config_fname = xstrdup(config);
 100     int r = m_config.read_xml(config);
 101     return r;
 102 }
 103
 104 void Yaz_Proxy::set_default_target(const char *target)
 105 {
 106     xfree (m_default_target);
 107     m_default_target = 0;
 108     if (target)
 109         m_default_target = (char *) xstrdup (target);
 110 }
 111
 112 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 113 {
 114     xfree (m_proxy_authentication);
 115     m_proxy_authentication = 0;
 116     if (auth)
 117         m_proxy_authentication = (char *) xstrdup (auth);
 118 }
 119
 120 void Yaz_Proxy::check_reconfigure()
 121 {
 122     if (m_reconfig_flag)
 123     {
 124         yaz_log(LOG_LOG, "reconfigure");
 125         yaz_log_reopen();
 126         if (m_config_fname)
 127         {
 128             yaz_log(LOG_LOG, "reconfigure config %s", m_config_fname);
 129             int r = m_config.read_xml(m_config_fname);
 130             if (r)
 131                 yaz_log(LOG_WARN, "reconfigure failed");
 132         }
 133         else
 134             yaz_log(LOG_LOG, "reconfigure");
 135         m_reconfig_flag = 0;
 136     }
 137 }
 138
 139 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 140                                             *the_PDU_Observable, int fd)
 141 {
 142     check_reconfigure();
 143     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable);
 144     new_proxy->m_parent = this;
 145     new_proxy->m_config = m_config;
 146     new_proxy->m_config_fname = 0;
 147     new_proxy->timeout(m_client_idletime);
 148     new_proxy->m_target_idletime = m_target_idletime;
 149     new_proxy->set_default_target(m_default_target);
 150     new_proxy->set_APDU_log(get_APDU_log());
 151     new_proxy->set_proxy_authentication(m_proxy_authentication);
 152     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 153     m_session_no++;
 154     yaz_log (LOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 155              the_PDU_Observable->getpeername());
 156     return new_proxy;
 157 }
 158
 159 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 160 {
 161     int oid[OID_SIZE];
 162     Z_OtherInformationUnit *oi;
 163     struct oident ent;
 164     ent.proto = PROTO_Z3950;
 165     ent.oclass = CLASS_USERINFO;
 166     ent.value = (oid_value) VAL_COOKIE;
 167     assert (oid_ent_to_oid (&ent, oid));
 168
 169     if (oid_ent_to_oid (&ent, oid) &&
 170         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 171         oi->which == Z_OtherInfo_characterInfo)
 172         return oi->information.characterInfo;
 173     return 0;
 174 }
 175
 176 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 177 {
 178     int oid[OID_SIZE];
 179     Z_OtherInformationUnit *oi;
 180     struct oident ent;
 181     ent.proto = PROTO_Z3950;
 182     ent.oclass = CLASS_USERINFO;
 183     ent.value = (oid_value) VAL_PROXY;
 184     if (oid_ent_to_oid (&ent, oid) &&
 185         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 186         oi->which == Z_OtherInfo_characterInfo)
 187         return oi->information.characterInfo;
 188     return 0;
 189 }
 190
 191 const char *Yaz_Proxy::load_balance(const char **url)
 192 {
 193     int zurl_in_use[MAX_ZURL_PLEX];
 194     Yaz_ProxyClient *c;
 195     int i;
 196
 197     for (i = 0; i<MAX_ZURL_PLEX; i++)
 198         zurl_in_use[i] = 0;
 199     for (c = m_parent->m_clientPool; c; c = c->m_next)
 200     {
 201         for (i = 0; url[i]; i++)
 202             if (!strcmp(url[i], c->get_hostname()))
 203                 zurl_in_use[i]++;
 204     }
 205     int min = 100000;
 206     const char *ret = 0;
 207     for (i = 0; url[i]; i++)
 208     {
 209         yaz_log(LOG_DEBUG, "%szurl=%s use=%d",
 210                 m_session_str, url[i], zurl_in_use[i]);
 211         if (min > zurl_in_use[i])
 212         {
 213             ret = url[i];
 214             min = zurl_in_use[i];
 215         }
 216     }
 217     return ret;
 218 }
 219
 220 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu)
 221 {
 222     assert (m_parent);
 223     Yaz_Proxy *parent = m_parent;
 224     Z_OtherInformation **oi;
 225     Yaz_ProxyClient *c = m_client;
 226
 227     get_otherInfoAPDU(apdu, &oi);
 228     char *cookie = get_cookie(oi);
 229
 230     if (!m_proxyTarget)
 231     {
 232         const char *url[MAX_ZURL_PLEX];
 233         const char *proxy_host = get_proxy(oi);
 234         if (proxy_host)
 235         {
 236             xfree(m_default_target);
 237             m_default_target = xstrdup(proxy_host);
 238             proxy_host = m_default_target;
 239         }
 240
 241         int client_idletime = -1;
 242         m_config.get_target_info(proxy_host, url, &m_bw_max,
 243                                  &m_pdu_max, &m_max_record_retrieve,
 244                                  &m_target_idletime, &client_idletime,
 245                                  &parent->m_max_clients,
 246                                  &m_keepalive_limit_bw, &m_keepalive_limit_pdu);
 247         if (client_idletime != -1)
 248         {
 249             m_client_idletime = client_idletime;
 250             timeout(m_client_idletime);
 251         }
 252         if (!url[0])
 253         {
 254             yaz_log(LOG_LOG, "%sNo default target", m_session_str);
 255             return 0;
 256         }
 257         // we don't handle multiplexing for cookie session, so we just
 258         // pick the first one in this case (anonymous users will be able
 259         // to use any backend)
 260         if (cookie && *cookie)
 261             m_proxyTarget = (char*) xstrdup(url[0]);
 262         else
 263             m_proxyTarget = (char*) xstrdup(load_balance(url));
 264     }
 265     if (cookie && *cookie)
 266     {
 267         Yaz_ProxyClient *cc = 0;
 268
 269         for (c = parent->m_clientPool; c; c = c->m_next)
 270         {
 271             assert (c->m_prev);
 272             assert (*c->m_prev == c);
 273             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 274                 !strcmp(m_proxyTarget, c->get_hostname()))
 275             {
 276                 cc = c;
 277             }
 278         }
 279         if (cc)
 280         {
 281             // found it in cache
 282             c = cc;
 283             // The following handles "cancel"
 284             // If connection is busy (waiting for PDU) and
 285             // we have an initRequest we can safely do re-open
 286             if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 287             {
 288                 yaz_log (LOG_LOG, "%s REOPEN target=%s", m_session_str,
 289                          c->get_hostname());
 290                 c->close();
 291                 c->client(m_proxyTarget);
 292                 c->m_init_flag = 0;
 293
 294                 c->m_last_ok = 0;
 295                 c->m_cache.clear();
 296                 c->m_last_resultCount = 0;
 297                 c->m_sr_transform = 0;
 298                 c->m_waiting = 0;
 299                 c->m_resultSetStartPoint = 0;
 300                 c->timeout(m_target_idletime);
 301             }
 302             c->m_seqno = parent->m_seqno;
 303             if (c->m_server && c->m_server != this)
 304                 c->m_server->m_client = 0;
 305             c->m_server = this;
 306             (parent->m_seqno)++;
 307             yaz_log (LOG_DEBUG, "get_client 1 %p %p", this, c);
 308             return c;
 309         }
 310     }
 311     else if (!c)
 312     {
 313         Yaz_ProxyClient *cc = 0;
 314
 315         for (c = parent->m_clientPool; c; c = c->m_next)
 316         {
 317             assert (c->m_prev);
 318             assert (*c->m_prev == c);
 319             if (c->m_server == 0 && c->m_cookie == 0 &&
 320                 !strcmp(m_proxyTarget, c->get_hostname()))
 321             {
 322                 cc = c;
 323             }
 324         }
 325         if (cc)
 326         {
 327             // found it in cache
 328             c = cc;
 329
 330             yaz_log (LOG_LOG, "%sREUSE %d %d %s",
 331                      m_session_str,
 332                      c->m_seqno, parent->m_seqno, c->get_hostname());
 333
 334             c->m_seqno = parent->m_seqno;
 335             assert(c->m_server == 0);
 336             c->m_server = this;
 337
 338             (parent->m_seqno)++;
 339             return c;
 340         }
 341     }
 342     if (!m_client)
 343     {
 344         if (apdu->which != Z_APDU_initRequest)
 345         {
 346             yaz_log (LOG_LOG, "no first INIT!");
 347             return 0;
 348         }
 349         Z_InitRequest *initRequest = apdu->u.initRequest;
 350
 351         if (!initRequest->idAuthentication)
 352         {
 353             if (m_proxy_authentication)
 354             {
 355                 initRequest->idAuthentication =
 356                     (Z_IdAuthentication *)
 357                     odr_malloc (odr_encode(),
 358                                 sizeof(*initRequest->idAuthentication));
 359                 initRequest->idAuthentication->which =
 360                     Z_IdAuthentication_open;
 361                 initRequest->idAuthentication->u.open =
 362                     odr_strdup (odr_encode(), m_proxy_authentication);
 363             }
 364         }
 365
 366         // go through list of clients - and find the lowest/oldest one.
 367         Yaz_ProxyClient *c_min = 0;
 368         int min_seq = -1;
 369         int no_of_clients = 0;
 370         if (parent->m_clientPool)
 371             yaz_log (LOG_DEBUG, "Existing sessions");
 372         for (c = parent->m_clientPool; c; c = c->m_next)
 373         {
 374             yaz_log (LOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 375                                c->m_waiting, c->get_hostname(),
 376                                c->m_cookie ? c->m_cookie : "");
 377             no_of_clients++;
 378             if (min_seq < 0 || c->m_seqno < min_seq)
 379             {
 380                 min_seq = c->m_seqno;
 381                 c_min = c;
 382             }
 383         }
 384         if (no_of_clients >= parent->m_max_clients)
 385         {
 386             c = c_min;
 387             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 388             {
 389                 yaz_log (LOG_LOG, "%sMAXCLIENTS Destroy %d",
 390                          m_session_str, c->m_seqno);
 391                 if (c->m_server && c->m_server != this)
 392                     delete c->m_server;
 393                 c->m_server = 0;
 394             }
 395             else
 396             {
 397                 yaz_log (LOG_LOG, "%sMAXCLIENTS Reuse %d %d %s",
 398                          m_session_str,
 399                          c->m_seqno, parent->m_seqno, c->get_hostname());
 400                 xfree (c->m_cookie);
 401                 c->m_cookie = 0;
 402                 if (cookie)
 403                     c->m_cookie = xstrdup(cookie);
 404                 c->m_seqno = parent->m_seqno;
 405                 if (c->m_server && c->m_server != this)
 406                 {
 407                     c->m_server->m_client = 0;
 408                     delete c->m_server;
 409                 }
 410                 (parent->m_seqno)++;
 411                 return c;
 412             }
 413         }
 414         else
 415         {
 416             yaz_log (LOG_LOG, "%sNEW %d %s",
 417                      m_session_str, parent->m_seqno, m_proxyTarget);
 418             c = new Yaz_ProxyClient(m_PDU_Observable->clone());
 419             c->m_next = parent->m_clientPool;
 420             if (c->m_next)
 421                 c->m_next->m_prev = &c->m_next;
 422             parent->m_clientPool = c;
 423             c->m_prev = &parent->m_clientPool;
 424         }
 425
 426         xfree (c->m_cookie);
 427         c->m_cookie = 0;
 428         if (cookie)
 429             c->m_cookie = xstrdup(cookie);
 430
 431         c->m_seqno = parent->m_seqno;
 432         c->client(m_proxyTarget);
 433         c->m_init_flag = 0;
 434         c->m_last_resultCount = 0;
 435         c->m_last_ok = 0;
 436         c->m_cache.clear();
 437         c->m_sr_transform = 0;
 438         c->m_waiting = 0;
 439         c->m_resultSetStartPoint = 0;
 440         c->timeout(30);
 441
 442         (parent->m_seqno)++;
 443     }
 444     yaz_log (LOG_DEBUG, "get_client 3 %p %p", this, c);
 445     return c;
 446 }
 447
 448 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 449 {
 450     int i;
 451     for (i = 0; i<num; i++)
 452     {
 453         oident *ent;
 454         Z_DefaultDiagFormat *r;
 455         Z_DiagRec *p = pp[i];
 456         if (p->which != Z_DiagRec_defaultFormat)
 457         {
 458             yaz_log(LOG_LOG, "%sError no diagnostics", m_session_str);
 459             return;
 460         }
 461         else
 462             r = p->u.defaultFormat;
 463         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 464             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 465             yaz_log(LOG_LOG, "%sError unknown diagnostic set", m_session_str);
 466         switch (r->which)
 467         {
 468         case Z_DefaultDiagFormat_v2Addinfo:
 469             yaz_log(LOG_LOG, "%sError %d %s:%s",
 470                     m_session_str,
 471                     *r->condition, diagbib1_str(*r->condition),
 472                     r->u.v2Addinfo);
 473             break;
 474         case Z_DefaultDiagFormat_v3Addinfo:
 475             yaz_log(LOG_LOG, "%sError %d %s:%s",
 476                     m_session_str,
 477                     *r->condition, diagbib1_str(*r->condition),
 478                     r->u.v3Addinfo);
 479             break;
 480         }
 481     }
 482 }
 483
 484 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 485 {
 486     int len = 0;
 487     if (apdu->which == Z_APDU_searchResponse)
 488     {
 489         Z_SearchResponse *sr = apdu->u.searchResponse;
 490         Z_Records *p = sr->records;
 491         if (p && p->which == Z_Records_NSD)
 492         {
 493             Z_DiagRec dr, *dr_p = &dr;
 494             dr.which = Z_DiagRec_defaultFormat;
 495             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 496
 497             display_diagrecs(&dr_p, 1);
 498         }
 499         else
 500         {
 501             if (sr->resultCount)
 502             {
 503                 yaz_log(LOG_LOG, "%s%d hits", m_session_str,
 504                         *sr->resultCount);
 505                 if (*sr->resultCount < 0)
 506                     m_invalid_session = 1;
 507             }
 508         }
 509     }
 510     else if (apdu->which == Z_APDU_presentResponse)
 511     {
 512         Z_PresentResponse *sr = apdu->u.presentResponse;
 513         Z_Records *p = sr->records;
 514         if (p && p->which == Z_Records_NSD)
 515         {
 516             Z_DiagRec dr, *dr_p = &dr;
 517             dr.which = Z_DiagRec_defaultFormat;
 518             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 519
 520             display_diagrecs(&dr_p, 1);
 521         }
 522     }
 523     int r = send_Z_PDU(apdu, &len);
 524     yaz_log (LOG_DEBUG, "%sSending %s to client %d bytes", m_session_str,
 525              apdu_name(apdu), len);
 526     m_bytes_sent += len;
 527     m_bw_stat.add_bytes(len);
 528     return r;
 529 }
 530
 531 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
 532 {
 533     int len = 0;
 534     int r = send_Z_PDU(apdu, &len);
 535     yaz_log (LOG_DEBUG, "%sSending %s to %s %d bytes",
 536              get_session_str(),
 537              apdu_name(apdu), get_hostname(), len);
 538     m_bytes_sent += len;
 539     return r;
 540 }
 541
 542 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
 543 {
 544     if (*m_parent->m_optimize == '0')
 545         return apdu;      // don't optimize result sets..
 546     if (apdu->which == Z_APDU_presentRequest)
 547     {
 548         Z_PresentRequest *pr = apdu->u.presentRequest;
 549         Z_NamePlusRecordList *npr;
 550         int toget = *pr->numberOfRecordsRequested;
 551         int start = *pr->resultSetStartPoint;
 552
 553         if (m_client->m_last_resultSetId &&
 554             !strcmp(m_client->m_last_resultSetId, pr->resultSetId))
 555         {
 556             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
 557                                           pr->preferredRecordSyntax,
 558                                           pr->recordComposition))
 559             {
 560                 yaz_log (LOG_LOG, "%sReturned cached records for present request",
 561                          m_session_str);
 562                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
 563                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
 564
 565                 new_apdu->u.presentResponse->numberOfRecordsReturned
 566                     = odr_intdup(odr_encode(), toget);
 567
 568                 new_apdu->u.presentResponse->records = (Z_Records*)
 569                     odr_malloc(odr_encode(), sizeof(Z_Records));
 570                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
 571                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
 572                 new_apdu->u.presentResponse->nextResultSetPosition =
 573                     odr_intdup(odr_encode(), start+toget);
 574
 575                 send_to_client(new_apdu);
 576                 return 0;
 577             }
 578         }
 579     }
 580
 581     if (apdu->which != Z_APDU_searchRequest)
 582         return apdu;
 583     Z_SearchRequest *sr = apdu->u.searchRequest;
 584     Yaz_Z_Query *this_query = new Yaz_Z_Query;
 585     Yaz_Z_Databases this_databases;
 586
 587     this_databases.set(sr->num_databaseNames, (const char **)
 588                        sr->databaseNames);
 589
 590     this_query->set_Z_Query(sr->query);
 591
 592     char query_str[80];
 593     this_query->print(query_str, sizeof(query_str)-1);
 594     yaz_log(LOG_LOG, "%sQuery %s", m_session_str, query_str);
 595
 596     if (m_client->m_last_ok && m_client->m_last_query &&
 597         m_client->m_last_query->match(this_query) &&
 598         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
 599         m_client->m_last_databases.match(this_databases))
 600     {
 601         delete this_query;
 602         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
 603             m_client->m_last_resultCount < *sr->largeSetLowerBound)
 604         {
 605             Z_NamePlusRecordList *npr;
 606             int toget = *sr->mediumSetPresentNumber;
 607             Z_RecordComposition *comp = 0;
 608
 609             if (toget > m_client->m_last_resultCount)
 610                 toget = m_client->m_last_resultCount;
 611
 612             if (sr->mediumSetElementSetNames)
 613             {
 614                 comp = (Z_RecordComposition *)
 615                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
 616                 comp->which = Z_RecordComp_simple;
 617                 comp->u.simple = sr->mediumSetElementSetNames;
 618             }
 619
 620             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
 621                                           sr->preferredRecordSyntax, comp))
 622             {
 623                 yaz_log (LOG_LOG, "%sReturned cached records for medium set",
 624                          m_session_str);
 625                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
 626                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
 627                 new_apdu->u.searchResponse->resultCount =
 628                     &m_client->m_last_resultCount;
 629
 630                 new_apdu->u.searchResponse->numberOfRecordsReturned
 631                     = odr_intdup(odr_encode(), toget);
 632
 633                 new_apdu->u.searchResponse->presentStatus =
 634                     odr_intdup(odr_encode(), Z_PresentStatus_success);
 635                 new_apdu->u.searchResponse->records = (Z_Records*)
 636                     odr_malloc(odr_encode(), sizeof(Z_Records));
 637                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
 638                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
 639                 new_apdu->u.searchResponse->nextResultSetPosition =
 640                     odr_intdup(odr_encode(), toget+1);
 641                 send_to_client(new_apdu);
 642                 return 0;
 643             }
 644             else
 645             {
 646                 // medium Set
 647                 // send present request (medium size)
 648                 yaz_log (LOG_LOG, "%sOptimizing search for medium set",
 649                          m_session_str);
 650
 651                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
 652                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
 653                 pr->referenceId = sr->referenceId;
 654                 pr->resultSetId = sr->resultSetName;
 655                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
 656                 *pr->numberOfRecordsRequested = toget;
 657                 pr->recordComposition = comp;
 658                 m_client->m_sr_transform = 1;
 659                 return new_apdu;
 660             }
 661         }
 662         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
 663             m_client->m_last_resultCount <= 0)
 664         {
 665             // large set. Return pseudo-search response immediately
 666             yaz_log (LOG_LOG, "%sOptimizing search for large set",
 667                      m_session_str);
 668             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
 669             new_apdu->u.searchResponse->referenceId = sr->referenceId;
 670             new_apdu->u.searchResponse->resultCount =
 671                 &m_client->m_last_resultCount;
 672             send_to_client(new_apdu);
 673             return 0;
 674         }
 675         else
 676         {
 677             Z_NamePlusRecordList *npr;
 678             int toget = m_client->m_last_resultCount;
 679             Z_RecordComposition *comp = 0;
 680             // small set
 681             // send a present request (small set)
 682
 683             if (sr->smallSetElementSetNames)
 684             {
 685                 comp = (Z_RecordComposition *)
 686                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
 687                 comp->which = Z_RecordComp_simple;
 688                 comp->u.simple = sr->smallSetElementSetNames;
 689             }
 690
 691             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
 692                                           sr->preferredRecordSyntax, comp))
 693             {
 694                 yaz_log (LOG_LOG, "%sReturned cached records for small set",
 695                          m_session_str);
 696                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
 697                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
 698                 new_apdu->u.searchResponse->resultCount =
 699                     &m_client->m_last_resultCount;
 700
 701                 new_apdu->u.searchResponse->numberOfRecordsReturned
 702                     = odr_intdup(odr_encode(), toget);
 703
 704                 new_apdu->u.searchResponse->presentStatus =
 705                     odr_intdup(odr_encode(), Z_PresentStatus_success);
 706                 new_apdu->u.searchResponse->records = (Z_Records*)
 707                     odr_malloc(odr_encode(), sizeof(Z_Records));
 708                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
 709                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
 710                 new_apdu->u.searchResponse->nextResultSetPosition =
 711                     odr_intdup(odr_encode(), toget+1);
 712                 send_to_client(new_apdu);
 713                 return 0;
 714             }
 715             else
 716             {
 717                 yaz_log (LOG_LOG, "%sOptimizing search for small set",
 718                          m_session_str);
 719                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
 720                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
 721                 pr->referenceId = sr->referenceId;
 722                 pr->resultSetId = sr->resultSetName;
 723                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
 724                 *pr->numberOfRecordsRequested = toget;
 725                 pr->recordComposition = comp;
 726                 m_client->m_sr_transform = 1;
 727                 return new_apdu;
 728             }
 729         }
 730     }
 731     else  // query doesn't match
 732     {
 733         delete m_client->m_last_query;
 734         m_client->m_last_query = this_query;
 735         m_client->m_last_ok = 0;
 736         m_client->m_cache.clear();
 737         m_client->m_resultSetStartPoint = 0;
 738
 739         xfree (m_client->m_last_resultSetId);
 740         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
 741
 742         m_client->m_last_databases.set(sr->num_databaseNames,
 743                                        (const char **) sr->databaseNames);
 744     }
 745     return apdu;
 746 }
 747
 748
 749 void Yaz_Proxy::recv_Z_PDU(Z_APDU *apdu, int len)
 750 {
 751     char *cp = strchr(m_session_str, ' ');
 752     m_request_no++;
 753     if (cp)
 754         sprintf(cp+1, "%d ", m_request_no);
 755
 756     int reduce = 0;
 757     m_bytes_recv += len;
 758
 759     yaz_log (LOG_DEBUG, "%sReceiving %s from client %d bytes", m_session_str,
 760              apdu_name(apdu), len);
 761
 762     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
 763         shutdown();
 764
 765     m_bw_stat.add_bytes(len);
 766     m_pdu_stat.add_bytes(1);
 767
 768     int bw_total = m_bw_stat.get_total();
 769     int pdu_total = m_pdu_stat.get_total();
 770
 771     yaz_log(LOG_LOG, "%sstat bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
 772             m_session_str, bw_total, pdu_total, m_bw_max, m_pdu_max);
 773     if (m_bw_max)
 774     {
 775         if (bw_total > m_bw_max)
 776         {
 777             reduce = (bw_total/m_bw_max);
 778         }
 779     }
 780     if (m_pdu_max)
 781     {
 782         if (pdu_total > m_pdu_max)
 783         {
 784             int nreduce = (60/m_pdu_max);
 785             reduce = (reduce > nreduce) ? reduce : nreduce;
 786         }
 787     }
 788     if (reduce)
 789     {
 790         yaz_log(LOG_LOG, "%sLimit delay=%d", m_session_str, reduce);
 791         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
 792         timeout(reduce);       // call us reduce seconds later
 793     }
 794     else
 795         recv_Z_PDU_0(apdu);    // all fine. Proceed receive PDU as usual
 796 }
 797
 798 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
 799 {
 800     if (m_max_record_retrieve)
 801     {
 802         if (apdu->which == Z_APDU_presentRequest)
 803         {
 804             Z_PresentRequest *pr = apdu->u.presentRequest;
 805             if (pr->numberOfRecordsRequested &&
 806                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
 807                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
 808         }
 809     }
 810 }
 811
 812 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
 813                                                      int error,
 814                                                      const char *addinfo)
 815 {
 816     Z_Records *rec = (Z_Records *)
 817         odr_malloc (odr, sizeof(*rec));
 818     int *err = (int *)
 819         odr_malloc (odr, sizeof(*err));
 820     Z_DiagRec *drec = (Z_DiagRec *)
 821         odr_malloc (odr, sizeof(*drec));
 822     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
 823         odr_malloc (odr, sizeof(*dr));
 824     *err = error;
 825     rec->which = Z_Records_NSD;
 826     rec->u.nonSurrogateDiagnostic = dr;
 827     dr->diagnosticSetId =
 828         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
 829     dr->condition = err;
 830     dr->which = Z_DefaultDiagFormat_v2Addinfo;
 831     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
 832     return rec;
 833 }
 834
 835 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
 836 {
 837     if (apdu->which == Z_APDU_searchRequest)
 838     {
 839         Z_SearchRequest *sr = apdu->u.searchRequest;
 840         int err;
 841         char *addinfo = 0;
 842         err = m_config.check_query(odr_encode(), m_default_target, sr->query,
 843                                    &addinfo);
 844         if (err)
 845         {
 846             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
 847
 848             new_apdu->u.searchResponse->referenceId = sr->referenceId;
 849             new_apdu->u.searchResponse->records =
 850                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
 851             *new_apdu->u.searchResponse->searchStatus = 0;
 852
 853             send_to_client(new_apdu);
 854
 855             return 0;
 856         }
 857     }
 858     return apdu;
 859 }
 860
 861 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
 862 {
 863     if (apdu->which == Z_APDU_searchRequest)
 864     {
 865         Z_SearchRequest *sr = apdu->u.searchRequest;
 866         if (*sr->smallSetUpperBound > 0 || *sr->largeSetLowerBound > 1)
 867         {
 868             int err;
 869             char *addinfo = 0;
 870             err = m_config.check_syntax(odr_encode(), m_default_target,
 871                                         sr->preferredRecordSyntax,
 872                                         &addinfo);
 873             if (err)
 874             {
 875                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
 876
 877                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
 878                 new_apdu->u.searchResponse->records =
 879                     create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
 880                 *new_apdu->u.searchResponse->searchStatus = 0;
 881
 882                 send_to_client(new_apdu);
 883
 884                 return 0;
 885             }
 886         }
 887     }
 888     else if (apdu->which == Z_APDU_presentRequest)
 889     {
 890         Z_PresentRequest *pr = apdu->u.presentRequest;
 891         int err;
 892         char *addinfo = 0;
 893         err = m_config.check_syntax(odr_encode(), m_default_target,
 894                                     pr->preferredRecordSyntax,
 895                                     &addinfo);
 896         if (err)
 897         {
 898             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
 899
 900             new_apdu->u.presentResponse->referenceId = pr->referenceId;
 901             new_apdu->u.presentResponse->records =
 902                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
 903             *new_apdu->u.presentResponse->presentStatus =
 904                 Z_PresentStatus_failure;
 905
 906             send_to_client(new_apdu);
 907
 908             return 0;
 909         }
 910     }
 911     return apdu;
 912 }
 913
 914 void Yaz_Proxy::recv_Z_PDU_0(Z_APDU *apdu)
 915 {
 916     // Determine our client.
 917     m_client = get_client(apdu);
 918     if (!m_client)
 919     {
 920         delete this;
 921         return;
 922     }
 923     m_client->m_server = this;
 924
 925     if (apdu->which == Z_APDU_initRequest)
 926     {
 927         if (apdu->u.initRequest->implementationId)
 928             yaz_log(LOG_LOG, "%simplementationId: %s",
 929                     m_session_str, apdu->u.initRequest->implementationId);
 930         if (apdu->u.initRequest->implementationName)
 931             yaz_log(LOG_LOG, "%simplementationName: %s",
 932                     m_session_str, apdu->u.initRequest->implementationName);
 933         if (apdu->u.initRequest->implementationVersion)
 934             yaz_log(LOG_LOG, "%simplementationVersion: %s",
 935                     m_session_str, apdu->u.initRequest->implementationVersion);
 936         if (m_client->m_init_flag)
 937         {
 938             Z_APDU *apdu = m_client->m_initResponse;
 939             apdu->u.initResponse->otherInfo = 0;
 940             if (m_client->m_cookie && *m_client->m_cookie)
 941                 set_otherInformationString(apdu, VAL_COOKIE, 1,
 942                                            m_client->m_cookie);
 943             send_to_client(apdu);
 944             return;
 945         }
 946         m_client->m_init_flag = 1;
 947     }
 948     handle_max_record_retrieve(apdu);
 949
 950     if (apdu)
 951         apdu = handle_syntax_validation(apdu);
 952
 953     if (apdu)
 954         apdu = handle_query_validation(apdu);
 955
 956     if (apdu)
 957         apdu = result_set_optimize(apdu);
 958     if (!apdu)
 959     {
 960         m_client->timeout(m_target_idletime);  // mark it active even
 961         // though we didn't use it
 962         return;
 963     }
 964
 965     // delete other info part from PDU before sending to target
 966     Z_OtherInformation **oi;
 967     get_otherInfoAPDU(apdu, &oi);
 968     if (oi)
 969         *oi = 0;
 970
 971     if (apdu->which == Z_APDU_presentRequest &&
 972         m_client->m_resultSetStartPoint == 0)
 973     {
 974         Z_PresentRequest *pr = apdu->u.presentRequest;
 975         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
 976         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
 977     } else {
 978         m_client->m_resultSetStartPoint = 0;
 979     }
 980     if (m_client->send_to_target(apdu) < 0)
 981     {
 982         delete m_client;
 983         m_client = 0;
 984         delete this;
 985     }
 986     else
 987         m_client->m_waiting = 1;
 988 }
 989
 990 void Yaz_Proxy::connectNotify()
 991 {
 992 }
 993
 994 void Yaz_Proxy::shutdown()
 995 {
 996     // only keep if keep_alive flag is set...
 997     if (m_client &&
 998         !m_invalid_session &&
 999         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
1000         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
1001         m_client->m_waiting == 0)
1002     {
1003         yaz_log(LOG_LOG, "%sShutdown (client to proxy) keepalive %s",
1004                  m_session_str,
1005                  m_client->get_hostname());
1006         yaz_log(LOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1007                 m_session_str, m_client->m_pdu_recv,
1008                 m_client->m_bytes_sent + m_client->m_bytes_recv,
1009                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
1010         assert (m_client->m_waiting != 2);
1011         // Tell client (if any) that no server connection is there..
1012         m_client->m_server = 0;
1013     }
1014     else if (m_client)
1015     {
1016         yaz_log (LOG_LOG, "%sShutdown (client to proxy) close %s",
1017                  m_session_str,
1018                  m_client->get_hostname());
1019         assert (m_client->m_waiting != 2);
1020         delete m_client;
1021     }
1022     else if (!m_parent)
1023     {
1024         yaz_log (LOG_LOG, "%sshutdown (client to proxy) bad state",
1025                  m_session_str);
1026         assert (m_parent);
1027     }
1028     else
1029     {
1030         yaz_log (LOG_LOG, "%sShutdown (client to proxy)",
1031                  m_session_str);
1032     }
1033     delete this;
1034 }
1035
1036 const char *Yaz_ProxyClient::get_session_str()
1037 {
1038     if (!m_server)
1039         return "0";
1040     return m_server->get_session_str();
1041 }
1042
1043 void Yaz_ProxyClient::shutdown()
1044 {
1045     yaz_log (LOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
1046              get_hostname());
1047     delete m_server;
1048     delete this;
1049 }
1050
1051 void Yaz_Proxy::failNotify()
1052 {
1053     yaz_log (LOG_LOG, "%sConnection closed by client",
1054              get_session_str());
1055     shutdown();
1056 }
1057
1058 void Yaz_ProxyClient::failNotify()
1059 {
1060     yaz_log (LOG_LOG, "%sConnection closed by target %s",
1061              get_session_str(), get_hostname());
1062     shutdown();
1063 }
1064
1065 void Yaz_ProxyClient::connectNotify()
1066 {
1067     yaz_log (LOG_LOG, "%sConnection accepted by %s", get_session_str(),
1068              get_hostname());
1069     int to;
1070     if (m_server)
1071         to = m_server->get_target_idletime();
1072     else
1073         to = 600;
1074     timeout(to);
1075 }
1076
1077 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
1078                                                   *the_PDU_Observable, int fd)
1079 {
1080     return new Yaz_ProxyClient(the_PDU_Observable);
1081 }
1082
1083 Yaz_ProxyClient::~Yaz_ProxyClient()
1084 {
1085     if (m_prev)
1086         *m_prev = m_next;
1087     if (m_next)
1088         m_next->m_prev = m_prev;
1089     m_waiting = 2;     // for debugging purposes only.
1090     odr_destroy(m_init_odr);
1091     delete m_last_query;
1092     xfree (m_last_resultSetId);
1093     xfree (m_cookie);
1094 }
1095
1096 void Yaz_Proxy::timeoutNotify()
1097 {
1098     if (m_bw_hold_PDU)
1099     {
1100         timeout(m_client_idletime);
1101         Z_APDU *apdu = m_bw_hold_PDU;
1102         m_bw_hold_PDU = 0;
1103         recv_Z_PDU_0(apdu);
1104     }
1105     else
1106     {
1107         yaz_log (LOG_LOG, "%sTimeout (client to proxy)", m_session_str);
1108         shutdown();
1109     }
1110 }
1111
1112 void Yaz_ProxyClient::timeoutNotify()
1113 {
1114     yaz_log (LOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
1115              get_hostname());
1116     shutdown();
1117 }
1118
1119 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable) :
1120     Yaz_Z_Assoc (the_PDU_Observable)
1121 {
1122     m_cookie = 0;
1123     m_next = 0;
1124     m_prev = 0;
1125     m_init_flag = 0;
1126     m_last_query = 0;
1127     m_last_resultSetId = 0;
1128     m_last_resultCount = 0;
1129     m_last_ok = 0;
1130     m_sr_transform = 0;
1131     m_waiting = 0;
1132     m_init_odr = odr_createmem (ODR_DECODE);
1133     m_initResponse = 0;
1134     m_resultSetStartPoint = 0;
1135     m_bytes_sent = m_bytes_recv = 0;
1136     m_pdu_recv = 0;
1137 }
1138
1139 const char *Yaz_Proxy::option(const char *name, const char *value)
1140 {
1141     if (!strcmp (name, "optimize")) {
1142         if (value) {
1143             xfree (m_optimize);
1144             m_optimize = xstrdup (value);
1145         }
1146         return m_optimize;
1147     }
1148     return 0;
1149 }
1150
1151 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
1152 {
1153     m_bytes_recv += len;
1154     m_pdu_recv++;
1155     m_waiting = 0;
1156     yaz_log (LOG_DEBUG, "%sReceiving %s from %s %d bytes", get_session_str(),
1157              apdu_name(apdu), get_hostname(), len);
1158     if (apdu->which == Z_APDU_initResponse)
1159     {
1160         NMEM nmem = odr_extract_mem (odr_decode());
1161         odr_reset (m_init_odr);
1162         nmem_transfer (m_init_odr->mem, nmem);
1163         m_initResponse = apdu;
1164
1165         Z_InitResponse *ir = apdu->u.initResponse;
1166         char *im0 = ir->implementationName;
1167
1168         char *im1 = (char*)
1169             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
1170         *im1 = '\0';
1171         if (im0)
1172         {
1173             strcat(im1, im0);
1174             strcat(im1, " ");
1175         }
1176         strcat(im1, "(YAZ Proxy)");
1177         ir->implementationName = im1;
1178
1179         nmem_destroy (nmem);
1180     }
1181     if (apdu->which == Z_APDU_searchResponse)
1182     {
1183         Z_SearchResponse *sr = apdu->u.searchResponse;
1184         m_last_resultCount = *sr->resultCount;
1185         int status = *sr->searchStatus;
1186         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
1187         {
1188             m_last_ok = 1;
1189
1190             if (sr->records && sr->records->which == Z_Records_DBOSD)
1191             {
1192                 m_cache.add(odr_decode(),
1193                             sr->records->u.databaseOrSurDiagnostics, 1,
1194                             *sr->resultCount);
1195             }
1196         }
1197     }
1198     if (apdu->which == Z_APDU_presentResponse)
1199     {
1200         Z_PresentResponse *pr = apdu->u.presentResponse;
1201         if (m_sr_transform)
1202         {
1203             m_sr_transform = 0;
1204             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1205             Z_SearchResponse *sr = new_apdu->u.searchResponse;
1206             sr->referenceId = pr->referenceId;
1207             *sr->resultCount = m_last_resultCount;
1208             sr->records = pr->records;
1209             sr->nextResultSetPosition = pr->nextResultSetPosition;
1210             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
1211             apdu = new_apdu;
1212         }
1213         if (pr->records &&
1214             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
1215         {
1216             m_cache.add(odr_decode(),
1217                         pr->records->u.databaseOrSurDiagnostics,
1218                         m_resultSetStartPoint, -1);
1219             m_resultSetStartPoint = 0;
1220         }
1221     }
1222     if (m_cookie)
1223         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
1224     if (m_server)
1225     {
1226         m_server->send_to_client(apdu);
1227     }
1228     if (apdu->which == Z_APDU_close)
1229     {
1230         shutdown();
1231     }
1232 }