src/yaz-proxy.cpp

   1 /*
   2  * Copyright (c) 1998-2003, Index Data.
   3  * See the file LICENSE for details.
   4  *
   5  * $Id: yaz-proxy.cpp,v 1.49 2003-10-03 13:01:42 adam Exp $
   6  */
   7
   8 #include <assert.h>
   9 #include <time.h>
  10
  11 #include <yaz/log.h>
  12 #include <yaz/diagbib1.h>
  13 #include <yaz++/proxy.h>
  14
  15 static const char *apdu_name(Z_APDU *apdu)
  16 {
  17     switch (apdu->which)
  18     {
  19     case Z_APDU_initRequest:
  20         return "initRequest";
  21     case Z_APDU_initResponse:
  22         return "initResponse";
  23     case Z_APDU_searchRequest:
  24         return "searchRequest";
  25     case Z_APDU_searchResponse:
  26         return "searchResponse";
  27     case Z_APDU_presentRequest:
  28         return "presentRequest";
  29     case Z_APDU_presentResponse:
  30         return "presentResponse";
  31     case Z_APDU_deleteResultSetRequest:
  32         return "deleteResultSetRequest";
  33     case Z_APDU_deleteResultSetResponse:
  34         return "deleteResultSetResponse";
  35     case Z_APDU_scanRequest:
  36         return "scanRequest";
  37     case Z_APDU_scanResponse:
  38         return "scanResponse";
  39     case Z_APDU_sortRequest:
  40         return "sortRequest";
  41     case Z_APDU_sortResponse:
  42         return "sortResponse";
  43     case Z_APDU_extendedServicesRequest:
  44         return "extendedServicesRequest";
  45     case Z_APDU_extendedServicesResponse:
  46         return "extendedServicesResponse";
  47     case Z_APDU_close:
  48         return "close";
  49     }
  50     return "other";
  51 }
  52
  53 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable) :
  54     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
  55 {
  56     m_PDU_Observable = the_PDU_Observable;
  57     m_client = 0;
  58     m_parent = 0;
  59     m_clientPool = 0;
  60     m_seqno = 1;
  61     m_keepalive = 0;
  62     m_proxyTarget = 0;
  63     m_default_target = 0;
  64     m_proxy_authentication = 0;
  65     m_max_clients = 150;
  66     m_seed = time(0);
  67     m_client_idletime = 600;
  68     m_target_idletime = 600;
  69     m_optimize = xstrdup ("1");
  70     strcpy(m_session_str, "x");
  71     m_session_no=0;
  72     m_bytes_sent = m_bytes_recv = 0;
  73     m_bw_hold_PDU = 0;
  74     m_bw_max = 0;
  75     m_pdu_max = 0;
  76     m_max_record_retrieve = 0;
  77 }
  78
  79 Yaz_Proxy::~Yaz_Proxy()
  80 {
  81     yaz_log(LOG_LOG, "%s Closed %d/%d sent/recv bytes total", m_session_str,
  82             m_bytes_sent, m_bytes_recv);
  83     xfree (m_proxyTarget);
  84     xfree (m_default_target);
  85     xfree (m_proxy_authentication);
  86     xfree (m_optimize);
  87 }
  88
  89 int Yaz_Proxy::set_config(const char *config)
  90 {
  91     int r = m_config.read_xml(config);
  92     return r;
  93 }
  94
  95 void Yaz_Proxy::set_default_target(const char *target)
  96 {
  97     xfree (m_default_target);
  98     m_default_target = 0;
  99     if (target)
 100         m_default_target = (char *) xstrdup (target);
 101 }
 102
 103 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 104 {
 105     xfree (m_proxy_authentication);
 106     m_proxy_authentication = 0;
 107     if (auth)
 108         m_proxy_authentication = (char *) xstrdup (auth);
 109 }
 110
 111 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 112                                             *the_PDU_Observable, int fd)
 113 {
 114     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable);
 115     new_proxy->m_parent = this;
 116     new_proxy->m_config = m_config;
 117     new_proxy->timeout(m_client_idletime);
 118     new_proxy->m_target_idletime = m_target_idletime;
 119     new_proxy->set_default_target(m_default_target);
 120     new_proxy->set_APDU_log(get_APDU_log());
 121     new_proxy->set_proxy_authentication(m_proxy_authentication);
 122     sprintf(new_proxy->m_session_str, "%ld:%d", (long) time(0), m_session_no);
 123     m_session_no++;
 124     yaz_log (LOG_LOG, "%s New session %s", new_proxy->m_session_str,
 125              the_PDU_Observable->getpeername());
 126     return new_proxy;
 127 }
 128
 129 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 130 {
 131     int oid[OID_SIZE];
 132     Z_OtherInformationUnit *oi;
 133     struct oident ent;
 134     ent.proto = PROTO_Z3950;
 135     ent.oclass = CLASS_USERINFO;
 136     ent.value = (oid_value) VAL_COOKIE;
 137     assert (oid_ent_to_oid (&ent, oid));
 138
 139     if (oid_ent_to_oid (&ent, oid) &&
 140         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 141         oi->which == Z_OtherInfo_characterInfo)
 142         return oi->information.characterInfo;
 143     return 0;
 144 }
 145
 146 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 147 {
 148     int oid[OID_SIZE];
 149     Z_OtherInformationUnit *oi;
 150     struct oident ent;
 151     ent.proto = PROTO_Z3950;
 152     ent.oclass = CLASS_USERINFO;
 153     ent.value = (oid_value) VAL_PROXY;
 154     if (oid_ent_to_oid (&ent, oid) &&
 155         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 156         oi->which == Z_OtherInfo_characterInfo)
 157         return oi->information.characterInfo;
 158     return 0;
 159 }
 160
 161 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu)
 162 {
 163     assert (m_parent);
 164     Yaz_Proxy *parent = m_parent;
 165     Z_OtherInformation **oi;
 166     Yaz_ProxyClient *c = m_client;
 167
 168     get_otherInfoAPDU(apdu, &oi);
 169     char *cookie = get_cookie(oi);
 170
 171     if (!m_proxyTarget)
 172     {
 173         const char *proxy_host = get_proxy(oi);
 174         if (!proxy_host)
 175         {
 176             xfree(m_default_target);
 177             m_default_target = xstrdup(proxy_host);
 178             proxy_host = m_default_target;
 179         }
 180
 181         const char *url = 0;
 182         int client_idletime = -1;
 183         m_config.get_target_info(proxy_host, &url, &m_keepalive, &m_bw_max,
 184                                  &m_pdu_max, &m_max_record_retrieve,
 185                                  &m_target_idletime, &client_idletime,
 186                                  &parent->m_max_clients);
 187         if (client_idletime != -1)
 188         {
 189             m_client_idletime = client_idletime;
 190             timeout(m_client_idletime);
 191         }
 192         if (!url)
 193         {
 194             yaz_log(LOG_LOG, "%s No default target", m_session_str);
 195             return 0;
 196         }
 197         m_proxyTarget = (char*) xstrdup(url);
 198     }
 199     if (cookie && *cookie)
 200     {
 201         Yaz_ProxyClient *cc = 0;
 202
 203         for (c = parent->m_clientPool; c; c = c->m_next)
 204         {
 205             assert (c->m_prev);
 206             assert (*c->m_prev == c);
 207             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 208                 !strcmp(m_proxyTarget, c->get_hostname()))
 209             {
 210                 cc = c;
 211             }
 212         }
 213         if (cc)
 214         {
 215             // found it in cache
 216             c = cc;
 217             // The following handles "cancel"
 218             // If connection is busy (waiting for PDU) and
 219             // we have an initRequest we can safely do re-open
 220             if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 221             {
 222                 yaz_log (LOG_LOG, "%s REOPEN target=%s", m_session_str,
 223                          c->get_hostname());
 224                 c->close();
 225                 c->client(m_proxyTarget);
 226                 c->m_init_flag = 0;
 227
 228                 c->m_last_ok = 0;
 229                 c->m_cache.clear();
 230                 c->m_last_resultCount = 0;
 231                 c->m_sr_transform = 0;
 232                 c->m_waiting = 0;
 233                 c->m_resultSetStartPoint = 0;
 234                 c->timeout(m_target_idletime);
 235             }
 236             c->m_seqno = parent->m_seqno;
 237             if (c->m_server && c->m_server != this)
 238                 c->m_server->m_client = 0;
 239             c->m_server = this;
 240             (parent->m_seqno)++;
 241             yaz_log (LOG_DEBUG, "get_client 1 %p %p", this, c);
 242             return c;
 243         }
 244     }
 245     else if (!c)
 246     {
 247         Yaz_ProxyClient *cc = 0;
 248
 249         for (c = parent->m_clientPool; c; c = c->m_next)
 250         {
 251             assert (c->m_prev);
 252             assert (*c->m_prev == c);
 253             if (c->m_server == 0 && c->m_cookie == 0 &&
 254                 !strcmp(m_proxyTarget, c->get_hostname()))
 255             {
 256                 cc = c;
 257             }
 258         }
 259         if (cc)
 260         {
 261             // found it in cache
 262             c = cc;
 263
 264             yaz_log (LOG_LOG, "%s REUSE %d %d %s",
 265                      m_session_str,
 266                      c->m_seqno, parent->m_seqno, c->get_hostname());
 267
 268             c->m_seqno = parent->m_seqno;
 269             assert(c->m_server == 0);
 270             c->m_server = this;
 271
 272             (parent->m_seqno)++;
 273             return c;
 274         }
 275     }
 276     if (!m_client)
 277     {
 278         if (apdu->which != Z_APDU_initRequest)
 279         {
 280             yaz_log (LOG_LOG, "no first INIT!");
 281             return 0;
 282         }
 283         Z_InitRequest *initRequest = apdu->u.initRequest;
 284
 285         if (!initRequest->idAuthentication)
 286         {
 287             if (m_proxy_authentication)
 288             {
 289                 initRequest->idAuthentication =
 290                     (Z_IdAuthentication *)
 291                     odr_malloc (odr_encode(),
 292                                 sizeof(*initRequest->idAuthentication));
 293                 initRequest->idAuthentication->which =
 294                     Z_IdAuthentication_open;
 295                 initRequest->idAuthentication->u.open =
 296                     odr_strdup (odr_encode(), m_proxy_authentication);
 297             }
 298         }
 299
 300         // go through list of clients - and find the lowest/oldest one.
 301         Yaz_ProxyClient *c_min = 0;
 302         int min_seq = -1;
 303         int no_of_clients = 0;
 304         if (parent->m_clientPool)
 305             yaz_log (LOG_LOG, "Existing sessions");
 306         for (c = parent->m_clientPool; c; c = c->m_next)
 307         {
 308             yaz_log (LOG_LOG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 309                                c->m_waiting, c->get_hostname(),
 310                                c->m_cookie ? c->m_cookie : "");
 311             no_of_clients++;
 312             if (min_seq < 0 || c->m_seqno < min_seq)
 313             {
 314                 min_seq = c->m_seqno;
 315                 c_min = c;
 316             }
 317         }
 318         if (no_of_clients >= parent->m_max_clients)
 319         {
 320             c = c_min;
 321             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 322             {
 323                 yaz_log (LOG_LOG, "%s MAXCLIENTS Destroy %d",
 324                          m_session_str, c->m_seqno);
 325                 if (c->m_server && c->m_server != this)
 326                     delete c->m_server;
 327                 c->m_server = 0;
 328             }
 329             else
 330             {
 331                 yaz_log (LOG_LOG, "%s MAXCLIENTS Reuse %d %d %s",
 332                          m_session_str,
 333                          c->m_seqno, parent->m_seqno, c->get_hostname());
 334                 xfree (c->m_cookie);
 335                 c->m_cookie = 0;
 336                 if (cookie)
 337                     c->m_cookie = xstrdup(cookie);
 338                 c->m_seqno = parent->m_seqno;
 339                 if (c->m_server && c->m_server != this)
 340                 {
 341                     c->m_server->m_client = 0;
 342                     delete c->m_server;
 343                 }
 344                 (parent->m_seqno)++;
 345                 return c;
 346             }
 347         }
 348         else
 349         {
 350             yaz_log (LOG_LOG, "%s NEW %d %s",
 351                      m_session_str, parent->m_seqno, m_proxyTarget);
 352             c = new Yaz_ProxyClient(m_PDU_Observable->clone());
 353             c->m_next = parent->m_clientPool;
 354             if (c->m_next)
 355                 c->m_next->m_prev = &c->m_next;
 356             parent->m_clientPool = c;
 357             c->m_prev = &parent->m_clientPool;
 358         }
 359
 360         xfree (c->m_cookie);
 361         c->m_cookie = 0;
 362         if (cookie)
 363             c->m_cookie = xstrdup(cookie);
 364
 365         c->m_seqno = parent->m_seqno;
 366         c->client(m_proxyTarget);
 367         c->m_init_flag = 0;
 368         c->m_last_resultCount = 0;
 369         c->m_last_ok = 0;
 370         c->m_cache.clear();
 371         c->m_sr_transform = 0;
 372         c->m_waiting = 0;
 373         c->m_resultSetStartPoint = 0;
 374         c->timeout(30);
 375
 376         (parent->m_seqno)++;
 377     }
 378     yaz_log (LOG_DEBUG, "get_client 3 %p %p", this, c);
 379     return c;
 380 }
 381
 382 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 383 {
 384     int i;
 385     for (i = 0; i<num; i++)
 386     {
 387         oident *ent;
 388         Z_DefaultDiagFormat *r;
 389         Z_DiagRec *p = pp[i];
 390         if (p->which != Z_DiagRec_defaultFormat)
 391         {
 392             yaz_log(LOG_LOG, "%s Error no diagnostics", m_session_str);
 393             return;
 394         }
 395         else
 396             r = p->u.defaultFormat;
 397         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 398             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 399             yaz_log(LOG_LOG, "%s Error unknown diagnostic set", m_session_str);
 400         switch (r->which)
 401         {
 402         case Z_DefaultDiagFormat_v2Addinfo:
 403             yaz_log(LOG_LOG, "%s Error %d %s:%s",
 404                     m_session_str,
 405                     *r->condition, diagbib1_str(*r->condition),
 406                     r->u.v2Addinfo);
 407             break;
 408         case Z_DefaultDiagFormat_v3Addinfo:
 409             yaz_log(LOG_LOG, "%s Error %d %s:%s",
 410                     m_session_str,
 411                     *r->condition, diagbib1_str(*r->condition),
 412                     r->u.v3Addinfo);
 413             break;
 414         }
 415     }
 416 }
 417
 418 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 419 {
 420     int len = 0;
 421     if (apdu->which == Z_APDU_searchResponse)
 422     {
 423         Z_SearchResponse *sr = apdu->u.searchResponse;
 424         Z_Records *p = sr->records;
 425         if (p && p->which == Z_Records_NSD)
 426         {
 427             Z_DiagRec dr, *dr_p = &dr;
 428             dr.which = Z_DiagRec_defaultFormat;
 429             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 430
 431             display_diagrecs(&dr_p, 1);
 432         }
 433         else
 434         {
 435             if (sr->resultCount)
 436                 yaz_log(LOG_LOG, "%s %d hits", m_session_str,
 437                         *sr->resultCount);
 438         }
 439     }
 440     else if (apdu->which == Z_APDU_presentResponse)
 441     {
 442         Z_PresentResponse *sr = apdu->u.presentResponse;
 443         Z_Records *p = sr->records;
 444         if (p && p->which == Z_Records_NSD)
 445         {
 446             Z_DiagRec dr, *dr_p = &dr;
 447             dr.which = Z_DiagRec_defaultFormat;
 448             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 449
 450             display_diagrecs(&dr_p, 1);
 451         }
 452     }
 453     int r = send_Z_PDU(apdu, &len);
 454     yaz_log (LOG_LOG, "%s Sending %s to client %d bytes", m_session_str,
 455              apdu_name(apdu), len);
 456     m_bytes_sent += len;
 457     m_bw_stat.add_bytes(len);
 458     return r;
 459 }
 460
 461 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
 462 {
 463     int len = 0;
 464     int r = send_Z_PDU(apdu, &len);
 465     yaz_log (LOG_LOG, "%s Sending %s to %s %d bytes",
 466              get_session_str(),
 467              apdu_name(apdu), get_hostname(), len);
 468     m_bytes_sent += len;
 469     return r;
 470 }
 471
 472 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
 473 {
 474     if (*m_parent->m_optimize == '0')
 475         return apdu;      // don't optimize result sets..
 476     if (apdu->which == Z_APDU_presentRequest)
 477     {
 478         Z_PresentRequest *pr = apdu->u.presentRequest;
 479         Z_NamePlusRecordList *npr;
 480         int toget = *pr->numberOfRecordsRequested;
 481         int start = *pr->resultSetStartPoint;
 482
 483         if (m_client->m_last_resultSetId &&
 484             !strcmp(m_client->m_last_resultSetId, pr->resultSetId))
 485         {
 486             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
 487                                           pr->preferredRecordSyntax,
 488                                           pr->recordComposition))
 489             {
 490                 yaz_log (LOG_LOG, "%s Returned cache records for present request",
 491                          m_session_str);
 492                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
 493                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
 494
 495                 new_apdu->u.presentResponse->numberOfRecordsReturned
 496                     = odr_intdup(odr_encode(), toget);
 497
 498                 new_apdu->u.presentResponse->records = (Z_Records*)
 499                     odr_malloc(odr_encode(), sizeof(Z_Records));
 500                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
 501                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
 502                 new_apdu->u.presentResponse->nextResultSetPosition =
 503                     odr_intdup(odr_encode(), start+toget);
 504
 505                 send_to_client(new_apdu);
 506                 return 0;
 507             }
 508         }
 509     }
 510
 511     if (apdu->which != Z_APDU_searchRequest)
 512         return apdu;
 513     Z_SearchRequest *sr = apdu->u.searchRequest;
 514     Yaz_Z_Query *this_query = new Yaz_Z_Query;
 515     Yaz_Z_Databases this_databases;
 516
 517     this_databases.set(sr->num_databaseNames, (const char **)
 518                        sr->databaseNames);
 519
 520     this_query->set_Z_Query(sr->query);
 521
 522     char query_str[80];
 523     this_query->print(query_str, sizeof(query_str)-1);
 524     yaz_log(LOG_LOG, "%s Query %s", m_session_str, query_str);
 525
 526     if (m_client->m_last_ok && m_client->m_last_query &&
 527         m_client->m_last_query->match(this_query) &&
 528         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
 529         m_client->m_last_databases.match(this_databases))
 530     {
 531         delete this_query;
 532         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
 533             m_client->m_last_resultCount < *sr->largeSetLowerBound)
 534         {
 535             Z_NamePlusRecordList *npr;
 536             int toget = *sr->mediumSetPresentNumber;
 537             Z_RecordComposition *comp = 0;
 538
 539             if (toget > m_client->m_last_resultCount)
 540                 toget = m_client->m_last_resultCount;
 541
 542             if (sr->mediumSetElementSetNames)
 543             {
 544                 comp = (Z_RecordComposition *)
 545                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
 546                 comp->which = Z_RecordComp_simple;
 547                 comp->u.simple = sr->mediumSetElementSetNames;
 548             }
 549
 550             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
 551                                           sr->preferredRecordSyntax, comp))
 552             {
 553                 yaz_log (LOG_LOG, "%s Returned cache records for medium set",
 554                          m_session_str);
 555                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
 556                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
 557                 new_apdu->u.searchResponse->resultCount =
 558                     &m_client->m_last_resultCount;
 559
 560                 new_apdu->u.searchResponse->numberOfRecordsReturned
 561                     = odr_intdup(odr_encode(), toget);
 562
 563                 new_apdu->u.searchResponse->presentStatus =
 564                     odr_intdup(odr_encode(), Z_PresentStatus_success);
 565                 new_apdu->u.searchResponse->records = (Z_Records*)
 566                     odr_malloc(odr_encode(), sizeof(Z_Records));
 567                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
 568                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
 569                 new_apdu->u.searchResponse->nextResultSetPosition =
 570                     odr_intdup(odr_encode(), toget+1);
 571                 send_to_client(new_apdu);
 572                 return 0;
 573             }
 574             else
 575             {
 576                 // medium Set
 577                 // send present request (medium size)
 578                 yaz_log (LOG_LOG, "%s Optimizing search for medium set",
 579                          m_session_str);
 580
 581                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
 582                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
 583                 pr->referenceId = sr->referenceId;
 584                 pr->resultSetId = sr->resultSetName;
 585                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
 586                 *pr->numberOfRecordsRequested = toget;
 587                 pr->recordComposition = comp;
 588                 m_client->m_sr_transform = 1;
 589                 return new_apdu;
 590             }
 591         }
 592         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
 593             m_client->m_last_resultCount <= 0)
 594         {
 595             // large set. Return pseudo-search response immediately
 596             yaz_log (LOG_LOG, "%s Optimizing search for large set",
 597                      m_session_str);
 598             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
 599             new_apdu->u.searchResponse->referenceId = sr->referenceId;
 600             new_apdu->u.searchResponse->resultCount =
 601                 &m_client->m_last_resultCount;
 602             send_to_client(new_apdu);
 603             return 0;
 604         }
 605         else
 606         {
 607             Z_NamePlusRecordList *npr;
 608             int toget = m_client->m_last_resultCount;
 609             Z_RecordComposition *comp = 0;
 610             // small set
 611             // send a present request (small set)
 612
 613             if (sr->smallSetElementSetNames)
 614             {
 615                 comp = (Z_RecordComposition *)
 616                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
 617                 comp->which = Z_RecordComp_simple;
 618                 comp->u.simple = sr->smallSetElementSetNames;
 619             }
 620
 621             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
 622                                           sr->preferredRecordSyntax, comp))
 623             {
 624                 yaz_log (LOG_LOG, "%s Returned cache records for small set",
 625                          m_session_str);
 626                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
 627                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
 628                 new_apdu->u.searchResponse->resultCount =
 629                     &m_client->m_last_resultCount;
 630
 631                 new_apdu->u.searchResponse->numberOfRecordsReturned
 632                     = odr_intdup(odr_encode(), toget);
 633
 634                 new_apdu->u.searchResponse->presentStatus =
 635                     odr_intdup(odr_encode(), Z_PresentStatus_success);
 636                 new_apdu->u.searchResponse->records = (Z_Records*)
 637                     odr_malloc(odr_encode(), sizeof(Z_Records));
 638                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
 639                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
 640                 new_apdu->u.searchResponse->nextResultSetPosition =
 641                     odr_intdup(odr_encode(), toget+1);
 642                 send_to_client(new_apdu);
 643                 return 0;
 644             }
 645             else
 646             {
 647                 yaz_log (LOG_LOG, "%s Optimizing search for small set",
 648                          m_session_str);
 649                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
 650                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
 651                 pr->referenceId = sr->referenceId;
 652                 pr->resultSetId = sr->resultSetName;
 653                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
 654                 *pr->numberOfRecordsRequested = toget;
 655                 pr->recordComposition = comp;
 656                 m_client->m_sr_transform = 1;
 657                 return new_apdu;
 658             }
 659         }
 660     }
 661     else
 662     {
 663         delete m_client->m_last_query;
 664         m_client->m_last_query = this_query;
 665         m_client->m_last_ok = 0;
 666         m_client->m_cache.clear();
 667         m_client->m_resultSetStartPoint = 0;
 668
 669         xfree (m_client->m_last_resultSetId);
 670         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
 671
 672         m_client->m_last_databases.set(sr->num_databaseNames,
 673                                        (const char **) sr->databaseNames);
 674     }
 675     return apdu;
 676 }
 677
 678
 679 void Yaz_Proxy::recv_Z_PDU(Z_APDU *apdu, int len)
 680 {
 681     int reduce = 0;
 682     m_bytes_recv += len;
 683
 684     yaz_log (LOG_LOG, "%s Receiving %s from client %d bytes", m_session_str,
 685              apdu_name(apdu), len);
 686
 687     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
 688         shutdown();
 689
 690     m_bw_stat.add_bytes(len);
 691     m_pdu_stat.add_bytes(1);
 692
 693     int bw_total = m_bw_stat.get_total();
 694     int pdu_total = m_pdu_stat.get_total();
 695
 696     yaz_log(LOG_LOG, "%s stat bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
 697             m_session_str, bw_total, pdu_total, m_bw_max, m_pdu_max);
 698     if (m_bw_max)
 699     {
 700         if (bw_total > m_bw_max)
 701         {
 702             reduce = (bw_total/m_bw_max);
 703         }
 704     }
 705     if (m_pdu_max)
 706     {
 707         if (pdu_total > m_pdu_max)
 708         {
 709             int nreduce = (60/m_pdu_max);
 710             reduce = (reduce > nreduce) ? reduce : nreduce;
 711         }
 712     }
 713     if (reduce)
 714     {
 715         yaz_log(LOG_LOG, "%s Limit delay=%d", m_session_str, reduce);
 716         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
 717         timeout(reduce);       // call us reduce seconds later
 718     }
 719     else
 720         recv_Z_PDU_0(apdu);    // all fine. Proceed receive PDU as usual
 721 }
 722
 723 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
 724 {
 725     if (m_max_record_retrieve)
 726     {
 727         if (apdu->which == Z_APDU_presentRequest)
 728         {
 729             Z_PresentRequest *pr = apdu->u.presentRequest;
 730             if (pr->numberOfRecordsRequested &&
 731                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
 732                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
 733         }
 734     }
 735 }
 736
 737 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
 738                                                      int error,
 739                                                      const char *addinfo)
 740 {
 741     Z_Records *rec = (Z_Records *)
 742         odr_malloc (odr, sizeof(*rec));
 743     int *err = (int *)
 744         odr_malloc (odr, sizeof(*err));
 745     Z_DiagRec *drec = (Z_DiagRec *)
 746         odr_malloc (odr, sizeof(*drec));
 747     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
 748         odr_malloc (odr, sizeof(*dr));
 749     *err = error;
 750     rec->which = Z_Records_NSD;
 751     rec->u.nonSurrogateDiagnostic = dr;
 752     dr->diagnosticSetId =
 753         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
 754     dr->condition = err;
 755     dr->which = Z_DefaultDiagFormat_v2Addinfo;
 756     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
 757     return rec;
 758 }
 759
 760 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
 761 {
 762     if (apdu->which == Z_APDU_searchRequest)
 763     {
 764         Z_SearchRequest *sr = apdu->u.searchRequest;
 765         int err;
 766         char *addinfo = 0;
 767         err = m_config.check_query(odr_encode(), m_default_target, sr->query,
 768                                    &addinfo);
 769         if (err)
 770         {
 771             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
 772             int *nulint = odr_intdup (odr_encode(), 0);
 773
 774             new_apdu->u.searchResponse->referenceId = sr->referenceId;
 775             new_apdu->u.searchResponse->records =
 776                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
 777             new_apdu->u.searchResponse->searchStatus = nulint;
 778             new_apdu->u.searchResponse->resultCount = nulint;
 779
 780             send_to_client(new_apdu);
 781
 782             return 0;
 783         }
 784     }
 785     return apdu;
 786 }
 787
 788 void Yaz_Proxy::recv_Z_PDU_0(Z_APDU *apdu)
 789 {
 790     // Determine our client.
 791     m_client = get_client(apdu);
 792     if (!m_client)
 793     {
 794         delete this;
 795         return;
 796     }
 797     m_client->m_server = this;
 798
 799     if (apdu->which == Z_APDU_initRequest)
 800     {
 801         if (m_client->m_init_flag)
 802         {
 803             Z_APDU *apdu = m_client->m_initResponse;
 804             apdu->u.initResponse->otherInfo = 0;
 805             if (m_client->m_cookie && *m_client->m_cookie)
 806                 set_otherInformationString(apdu, VAL_COOKIE, 1,
 807                                            m_client->m_cookie);
 808             send_to_client(apdu);
 809             return;
 810         }
 811         m_client->m_init_flag = 1;
 812     }
 813     handle_max_record_retrieve(apdu);
 814
 815     if (apdu)
 816         apdu = handle_query_validation(apdu);
 817
 818     if (apdu)
 819         apdu = result_set_optimize(apdu);
 820     if (!apdu)
 821     {
 822         m_client->timeout(m_target_idletime);  // mark it active even
 823         // though we didn't use it
 824         return;
 825     }
 826
 827     // delete other info part from PDU before sending to target
 828     Z_OtherInformation **oi;
 829     get_otherInfoAPDU(apdu, &oi);
 830     if (oi)
 831         *oi = 0;
 832
 833     if (apdu->which == Z_APDU_presentRequest &&
 834         m_client->m_resultSetStartPoint == 0)
 835     {
 836         Z_PresentRequest *pr = apdu->u.presentRequest;
 837         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
 838         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
 839     } else {
 840         m_client->m_resultSetStartPoint = 0;
 841     }
 842     if (m_client->send_to_target(apdu) < 0)
 843     {
 844         delete m_client;
 845         m_client = 0;
 846         delete this;
 847     }
 848     else
 849         m_client->m_waiting = 1;
 850 }
 851
 852 void Yaz_Proxy::connectNotify()
 853 {
 854 }
 855
 856 void Yaz_Proxy::shutdown()
 857 {
 858     // only keep if keep_alive flag is set...
 859     if (m_keepalive && m_client && m_client->m_waiting == 0)
 860     {
 861         yaz_log (LOG_LOG, "%s Shutdown (client to proxy) keepalive %s",
 862                  m_session_str,
 863                  m_client->get_hostname());
 864         assert (m_client->m_waiting != 2);
 865         // Tell client (if any) that no server connection is there..
 866         m_client->m_server = 0;
 867     }
 868     else if (m_client)
 869     {
 870         yaz_log (LOG_LOG, "%s Shutdown (client to proxy) close %s",
 871                  m_session_str,
 872                  m_client->get_hostname());
 873         assert (m_client->m_waiting != 2);
 874         delete m_client;
 875     }
 876     else if (!m_parent)
 877     {
 878         yaz_log (LOG_LOG, "%s shutdown (client to proxy) bad state",
 879                  m_session_str);
 880         assert (m_parent);
 881     }
 882     else
 883     {
 884         yaz_log (LOG_LOG, "%s Shutdown (client to proxy)",
 885                  m_session_str);
 886     }
 887     delete this;
 888 }
 889
 890 const char *Yaz_ProxyClient::get_session_str()
 891 {
 892     if (!m_server)
 893         return "0";
 894     return m_server->get_session_str();
 895 }
 896
 897 void Yaz_ProxyClient::shutdown()
 898 {
 899     yaz_log (LOG_LOG, "%s Shutdown (proxy to target) %s", get_session_str(),
 900              get_hostname());
 901     delete m_server;
 902     delete this;
 903 }
 904
 905 void Yaz_Proxy::failNotify()
 906 {
 907     yaz_log (LOG_LOG, "%s Connection closed by client",
 908              get_session_str());
 909     shutdown();
 910 }
 911
 912 void Yaz_ProxyClient::failNotify()
 913 {
 914     yaz_log (LOG_LOG, "%s Connection closed by target %s",
 915              get_session_str(), get_hostname());
 916     shutdown();
 917 }
 918
 919 void Yaz_ProxyClient::connectNotify()
 920 {
 921     yaz_log (LOG_LOG, "%s Connection accepted by %s", get_session_str(),
 922              get_hostname());
 923     int to;
 924     if (m_server)
 925         to = m_server->get_target_idletime();
 926     else
 927         to = 600;
 928     timeout(to);
 929 }
 930
 931 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
 932                                                   *the_PDU_Observable, int fd)
 933 {
 934     return new Yaz_ProxyClient(the_PDU_Observable);
 935 }
 936
 937 Yaz_ProxyClient::~Yaz_ProxyClient()
 938 {
 939     if (m_prev)
 940         *m_prev = m_next;
 941     if (m_next)
 942         m_next->m_prev = m_prev;
 943     m_waiting = 2;     // for debugging purposes only.
 944     odr_destroy(m_init_odr);
 945     delete m_last_query;
 946     xfree (m_last_resultSetId);
 947     xfree (m_cookie);
 948 }
 949
 950 void Yaz_Proxy::timeoutNotify()
 951 {
 952     if (m_bw_hold_PDU)
 953     {
 954         timeout(m_client_idletime);
 955         Z_APDU *apdu = m_bw_hold_PDU;
 956         m_bw_hold_PDU = 0;
 957         recv_Z_PDU_0(apdu);
 958     }
 959     else
 960     {
 961         yaz_log (LOG_LOG, "%s Timeout (client to proxy)", m_session_str);
 962         shutdown();
 963     }
 964 }
 965
 966 void Yaz_ProxyClient::timeoutNotify()
 967 {
 968     yaz_log (LOG_LOG, "%s Timeout (proxy to target) %s", get_session_str(),
 969              get_hostname());
 970     shutdown();
 971 }
 972
 973 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable) :
 974     Yaz_Z_Assoc (the_PDU_Observable)
 975 {
 976     m_cookie = 0;
 977     m_next = 0;
 978     m_prev = 0;
 979     m_init_flag = 0;
 980     m_last_query = 0;
 981     m_last_resultSetId = 0;
 982     m_last_resultCount = 0;
 983     m_last_ok = 0;
 984     m_sr_transform = 0;
 985     m_waiting = 0;
 986     m_init_odr = odr_createmem (ODR_DECODE);
 987     m_initResponse = 0;
 988     m_resultSetStartPoint = 0;
 989     m_bytes_sent = m_bytes_recv = 0;
 990 }
 991
 992 const char *Yaz_Proxy::option(const char *name, const char *value)
 993 {
 994     if (!strcmp (name, "optimize")) {
 995         if (value) {
 996             xfree (m_optimize);
 997             m_optimize = xstrdup (value);
 998         }
 999         return m_optimize;
1000     }
1001     return 0;
1002 }
1003
1004 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
1005 {
1006     m_bytes_recv += len;
1007     m_waiting = 0;
1008     yaz_log (LOG_LOG, "%s Receiving %s from %s %d bytes", get_session_str(),
1009              apdu_name(apdu), get_hostname(), len);
1010     if (apdu->which == Z_APDU_initResponse)
1011     {
1012         NMEM nmem = odr_extract_mem (odr_decode());
1013         odr_reset (m_init_odr);
1014         nmem_transfer (m_init_odr->mem, nmem);
1015         m_initResponse = apdu;
1016
1017         Z_InitResponse *ir = apdu->u.initResponse;
1018         char *im0 = ir->implementationName;
1019
1020         char *im1 = (char*)
1021             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
1022         *im1 = '\0';
1023         if (im0)
1024         {
1025             strcat(im1, im0);
1026             strcat(im1, " ");
1027         }
1028         strcat(im1, "(YAZ Proxy)");
1029         ir->implementationName = im1;
1030
1031         nmem_destroy (nmem);
1032     }
1033     if (apdu->which == Z_APDU_searchResponse)
1034     {
1035         Z_SearchResponse *sr = apdu->u.searchResponse;
1036         m_last_resultCount = *sr->resultCount;
1037         int status = *sr->searchStatus;
1038         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
1039         {
1040             m_last_ok = 1;
1041
1042             if (sr->records && sr->records->which == Z_Records_DBOSD)
1043             {
1044                 m_cache.add(odr_decode(),
1045                             sr->records->u.databaseOrSurDiagnostics, 1,
1046                             *sr->resultCount);
1047             }
1048         }
1049     }
1050     if (apdu->which == Z_APDU_presentResponse)
1051     {
1052         Z_PresentResponse *pr = apdu->u.presentResponse;
1053         if (m_sr_transform)
1054         {
1055             m_sr_transform = 0;
1056             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1057             Z_SearchResponse *sr = new_apdu->u.searchResponse;
1058             sr->referenceId = pr->referenceId;
1059             *sr->resultCount = m_last_resultCount;
1060             sr->records = pr->records;
1061             sr->nextResultSetPosition = pr->nextResultSetPosition;
1062             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
1063             apdu = new_apdu;
1064         }
1065         if (pr->records &&
1066             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
1067         {
1068             m_cache.add(odr_decode(),
1069                         pr->records->u.databaseOrSurDiagnostics,
1070                         m_resultSetStartPoint, -1);
1071             m_resultSetStartPoint = 0;
1072         }
1073     }
1074     if (m_cookie)
1075         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
1076     if (m_server)
1077     {
1078         m_server->send_to_client(apdu);
1079     }
1080     if (apdu->which == Z_APDU_close)
1081     {
1082         shutdown();
1083     }
1084 }