src/yaz-proxy.cpp

   1 /*
   2  * Copyright (c) 1998-2003, Index Data.
   3  * See the file LICENSE for details.
   4  *
   5  * $Id: yaz-proxy.cpp,v 1.73 2003-12-22 15:16:23 adam Exp $
   6  */
   7
   8 #include <assert.h>
   9 #include <time.h>
  10
  11 #include <yaz/srw.h>
  12 #include <yaz/marcdisp.h>
  13 #include <yaz/yaz-iconv.h>
  14 #include <yaz/log.h>
  15 #include <yaz/diagbib1.h>
  16 #include <yaz++/proxy.h>
  17 #include <yaz/pquery.h>
  18
  19 static const char *apdu_name(Z_APDU *apdu)
  20 {
  21     switch (apdu->which)
  22     {
  23     case Z_APDU_initRequest:
  24         return "initRequest";
  25     case Z_APDU_initResponse:
  26         return "initResponse";
  27     case Z_APDU_searchRequest:
  28         return "searchRequest";
  29     case Z_APDU_searchResponse:
  30         return "searchResponse";
  31     case Z_APDU_presentRequest:
  32         return "presentRequest";
  33     case Z_APDU_presentResponse:
  34         return "presentResponse";
  35     case Z_APDU_deleteResultSetRequest:
  36         return "deleteResultSetRequest";
  37     case Z_APDU_deleteResultSetResponse:
  38         return "deleteResultSetResponse";
  39     case Z_APDU_scanRequest:
  40         return "scanRequest";
  41     case Z_APDU_scanResponse:
  42         return "scanResponse";
  43     case Z_APDU_sortRequest:
  44         return "sortRequest";
  45     case Z_APDU_sortResponse:
  46         return "sortResponse";
  47     case Z_APDU_extendedServicesRequest:
  48         return "extendedServicesRequest";
  49     case Z_APDU_extendedServicesResponse:
  50         return "extendedServicesResponse";
  51     case Z_APDU_close:
  52         return "close";
  53     }
  54     return "other";
  55 }
  56
  57 static const char *gdu_name(Z_GDU *gdu)
  58 {
  59     switch(gdu->which)
  60     {
  61     case Z_GDU_Z3950:
  62         return apdu_name(gdu->u.z3950);
  63     case Z_GDU_HTTP_Request:
  64         return "HTTP Request";
  65     case Z_GDU_HTTP_Response:
  66         return "HTTP Response";
  67     }
  68     return "Unknown request/response";
  69 }
  70
  71 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
  72                      Yaz_Proxy *parent) :
  73     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
  74 {
  75     m_PDU_Observable = the_PDU_Observable;
  76     m_client = 0;
  77     m_parent = parent;
  78     m_clientPool = 0;
  79     m_seqno = 1;
  80     m_keepalive_limit_bw = 500000;
  81     m_keepalive_limit_pdu = 1000;
  82     m_proxyTarget = 0;
  83     m_default_target = 0;
  84     m_proxy_authentication = 0;
  85     m_max_clients = 150;
  86     m_log_mask = 0;
  87     m_seed = time(0);
  88     m_client_idletime = 600;
  89     m_target_idletime = 600;
  90     m_optimize = xstrdup ("1");
  91     strcpy(m_session_str, "0 ");
  92     m_session_no=0;
  93     m_bytes_sent = m_bytes_recv = 0;
  94     m_bw_hold_PDU = 0;
  95     m_bw_max = 0;
  96     m_pdu_max = 0;
  97     m_max_record_retrieve = 0;
  98     m_reconfig_flag = 0;
  99     m_config_fname = 0;
 100     m_request_no = 0;
 101     m_invalid_session = 0;
 102     m_config = 0;
 103     m_marcxml_flag = 0;
 104     m_initRequest_apdu = 0;
 105     m_initRequest_mem = 0;
 106     m_apdu_invalid_session = 0;
 107     m_mem_invalid_session = 0;
 108     m_s2z_odr_init = 0;
 109     m_s2z_odr_search = 0;
 110     m_s2z_init_apdu = 0;
 111     m_s2z_search_apdu = 0;
 112     m_s2z_present_apdu = 0;
 113     m_http_keepalive = 0;
 114     m_http_version = 0;
 115     m_soap_ns = 0;
 116     m_s2z_packing = Z_SRW_recordPacking_string;
 117     m_time_tv.tv_sec = 0;
 118     m_time_tv.tv_usec = 0;
 119 }
 120
 121 Yaz_Proxy::~Yaz_Proxy()
 122 {
 123     yaz_log(LOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 124             m_bytes_sent, m_bytes_recv);
 125     nmem_destroy(m_initRequest_mem);
 126     nmem_destroy(m_mem_invalid_session);
 127     xfree (m_proxyTarget);
 128     xfree (m_default_target);
 129     xfree (m_proxy_authentication);
 130     xfree (m_optimize);
 131     if (m_s2z_odr_init)
 132         odr_destroy(m_s2z_odr_init);
 133     if (m_s2z_odr_search)
 134         odr_destroy(m_s2z_odr_search);
 135     delete m_config;
 136 }
 137
 138 int Yaz_Proxy::set_config(const char *config)
 139 {
 140     delete m_config;
 141     m_config = new Yaz_ProxyConfig();
 142     xfree(m_config_fname);
 143     m_config_fname = xstrdup(config);
 144     int r = m_config->read_xml(config);
 145     if (!r)
 146         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 147     return r;
 148 }
 149
 150 void Yaz_Proxy::set_default_target(const char *target)
 151 {
 152     xfree (m_default_target);
 153     m_default_target = 0;
 154     if (target)
 155         m_default_target = (char *) xstrdup (target);
 156 }
 157
 158 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 159 {
 160     xfree (m_proxy_authentication);
 161     m_proxy_authentication = 0;
 162     if (auth)
 163         m_proxy_authentication = (char *) xstrdup (auth);
 164 }
 165
 166 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 167 {
 168     if (m_parent)
 169         return m_parent->check_reconfigure();
 170
 171     Yaz_ProxyConfig *cfg = m_config;
 172     if (m_reconfig_flag)
 173     {
 174         yaz_log(LOG_LOG, "reconfigure");
 175         yaz_log_reopen();
 176         if (m_config_fname && cfg)
 177         {
 178             yaz_log(LOG_LOG, "reconfigure config %s", m_config_fname);
 179             int r = cfg->read_xml(m_config_fname);
 180             if (r)
 181                 yaz_log(LOG_WARN, "reconfigure failed");
 182             else
 183             {
 184                 m_log_mask = 0;
 185                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 186             }
 187         }
 188         else
 189             yaz_log(LOG_LOG, "reconfigure");
 190         m_reconfig_flag = 0;
 191     }
 192     return cfg;
 193 }
 194
 195 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 196                                             *the_PDU_Observable, int fd)
 197 {
 198     check_reconfigure();
 199     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 200     new_proxy->m_config = 0;
 201     new_proxy->m_config_fname = 0;
 202     new_proxy->timeout(m_client_idletime);
 203     new_proxy->m_target_idletime = m_target_idletime;
 204     new_proxy->set_default_target(m_default_target);
 205     new_proxy->m_max_clients = m_max_clients;
 206     new_proxy->m_log_mask = m_log_mask;
 207     new_proxy->set_APDU_log(get_APDU_log());
 208     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 209         new_proxy->set_APDU_yazlog(1);
 210     else
 211         new_proxy->set_APDU_yazlog(0);
 212     new_proxy->set_proxy_authentication(m_proxy_authentication);
 213     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 214     m_session_no++;
 215     yaz_log (LOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 216              the_PDU_Observable->getpeername());
 217     return new_proxy;
 218 }
 219
 220 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 221 {
 222     int oid[OID_SIZE];
 223     Z_OtherInformationUnit *oi;
 224     struct oident ent;
 225     ent.proto = PROTO_Z3950;
 226     ent.oclass = CLASS_USERINFO;
 227     ent.value = (oid_value) VAL_COOKIE;
 228     assert (oid_ent_to_oid (&ent, oid));
 229
 230     if (oid_ent_to_oid (&ent, oid) &&
 231         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 232         oi->which == Z_OtherInfo_characterInfo)
 233         return oi->information.characterInfo;
 234     return 0;
 235 }
 236
 237 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 238 {
 239     int oid[OID_SIZE];
 240     Z_OtherInformationUnit *oi;
 241     struct oident ent;
 242     ent.proto = PROTO_Z3950;
 243     ent.oclass = CLASS_USERINFO;
 244     ent.value = (oid_value) VAL_PROXY;
 245     if (oid_ent_to_oid (&ent, oid) &&
 246         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 247         oi->which == Z_OtherInfo_characterInfo)
 248         return oi->information.characterInfo;
 249     return 0;
 250 }
 251
 252 const char *Yaz_Proxy::load_balance(const char **url)
 253 {
 254     int zurl_in_use[MAX_ZURL_PLEX];
 255     Yaz_ProxyClient *c;
 256     int i;
 257
 258     for (i = 0; i<MAX_ZURL_PLEX; i++)
 259         zurl_in_use[i] = 0;
 260     for (c = m_parent->m_clientPool; c; c = c->m_next)
 261     {
 262         for (i = 0; url[i]; i++)
 263             if (!strcmp(url[i], c->get_hostname()))
 264                 zurl_in_use[i]++;
 265     }
 266     int min = 100000;
 267     const char *ret = 0;
 268     for (i = 0; url[i]; i++)
 269     {
 270         yaz_log(LOG_DEBUG, "%szurl=%s use=%d",
 271                 m_session_str, url[i], zurl_in_use[i]);
 272         if (min > zurl_in_use[i])
 273         {
 274             ret = url[i];
 275             min = zurl_in_use[i];
 276         }
 277     }
 278     return ret;
 279 }
 280
 281 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 282                                        const char *proxy_host)
 283 {
 284     assert (m_parent);
 285     Yaz_Proxy *parent = m_parent;
 286     Z_OtherInformation **oi;
 287     Yaz_ProxyClient *c = m_client;
 288
 289     if (!m_proxyTarget)
 290     {
 291         const char *url[MAX_ZURL_PLEX];
 292         Yaz_ProxyConfig *cfg = check_reconfigure();
 293         if (proxy_host)
 294         {
 295 #if 1
 296 /* only to be enabled for debugging... */
 297             if (!strcmp(proxy_host, "stop"))
 298                 exit(0);
 299 #endif
 300             xfree(m_default_target);
 301             m_default_target = xstrdup(proxy_host);
 302             proxy_host = m_default_target;
 303         }
 304         int client_idletime = -1;
 305         const char *cql2rpn_fname = 0;
 306         url[0] = m_default_target;
 307         url[1] = 0;
 308         if (cfg)
 309         {
 310             int pre_init = 0;
 311             cfg->get_target_info(proxy_host, url, &m_bw_max,
 312                                  &m_pdu_max, &m_max_record_retrieve,
 313                                  &m_target_idletime, &client_idletime,
 314                                  &parent->m_max_clients,
 315                                  &m_keepalive_limit_bw,
 316                                  &m_keepalive_limit_pdu,
 317                                  &pre_init,
 318                                  &cql2rpn_fname);
 319         }
 320         if (client_idletime != -1)
 321         {
 322             m_client_idletime = client_idletime;
 323             timeout(m_client_idletime);
 324         }
 325         if (cql2rpn_fname)
 326             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 327         if (!url[0])
 328         {
 329             yaz_log(LOG_LOG, "%sNo default target", m_session_str);
 330             return 0;
 331         }
 332         // we don't handle multiplexing for cookie session, so we just
 333         // pick the first one in this case (anonymous users will be able
 334         // to use any backend)
 335         if (cookie && *cookie)
 336             m_proxyTarget = (char*) xstrdup(url[0]);
 337         else
 338             m_proxyTarget = (char*) xstrdup(load_balance(url));
 339     }
 340     if (cookie && *cookie)
 341     {   // search in sessions with a cookie
 342         for (c = parent->m_clientPool; c; c = c->m_next)
 343         {
 344             assert (c->m_prev);
 345             assert (*c->m_prev == c);
 346             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 347                 !strcmp(m_proxyTarget, c->get_hostname()))
 348             {
 349                 // Found it in cache
 350                 // The following handles "cancel"
 351                 // If connection is busy (waiting for PDU) and
 352                 // we have an initRequest we can safely do re-open
 353                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 354                 {
 355                     yaz_log (LOG_LOG, "%s REOPEN target=%s", m_session_str,
 356                              c->get_hostname());
 357                     c->close();
 358                     c->m_init_flag = 0;
 359
 360                     c->m_last_ok = 0;
 361                     c->m_cache.clear();
 362                     c->m_last_resultCount = 0;
 363                     c->m_sr_transform = 0;
 364                     c->m_waiting = 0;
 365                     c->m_resultSetStartPoint = 0;
 366                     c->m_target_idletime = m_target_idletime;
 367                     if (c->client(m_proxyTarget))
 368                     {
 369                         delete c;
 370                         return 0;
 371                     }
 372                     c->timeout(30);
 373                 }
 374                 c->m_seqno = parent->m_seqno;
 375                 if (c->m_server && c->m_server != this)
 376                     c->m_server->m_client = 0;
 377                 c->m_server = this;
 378                 (parent->m_seqno)++;
 379                 yaz_log (LOG_DEBUG, "get_client 1 %p %p", this, c);
 380                 return c;
 381             }
 382         }
 383     }
 384     else if (!c)
 385     {
 386         // don't have a client session yet. Search in session w/o cookie
 387         for (c = parent->m_clientPool; c; c = c->m_next)
 388         {
 389             assert (c->m_prev);
 390             assert (*c->m_prev == c);
 391             if (c->m_server == 0 && c->m_cookie == 0 &&
 392                 c->m_waiting == 0 &&
 393                 !strcmp(m_proxyTarget, c->get_hostname()))
 394             {
 395                 // found it in cache
 396                 yaz_log (LOG_LOG, "%sREUSE %s",
 397                          m_session_str, c->get_hostname());
 398
 399                 c->m_seqno = parent->m_seqno;
 400                 assert(c->m_server == 0);
 401                 c->m_server = this;
 402
 403                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 404                     c->set_APDU_yazlog(1);
 405                 else
 406                     c->set_APDU_yazlog(0);
 407
 408                 (parent->m_seqno)++;
 409
 410                 parent->pre_init();
 411
 412                 return c;
 413             }
 414         }
 415     }
 416     if (!m_client)
 417     {
 418         if (apdu->which != Z_APDU_initRequest)
 419         {
 420             yaz_log (LOG_LOG, "no first INIT!");
 421             return 0;
 422         }
 423         Z_InitRequest *initRequest = apdu->u.initRequest;
 424
 425         if (!initRequest->idAuthentication)
 426         {
 427             if (m_proxy_authentication)
 428             {
 429                 initRequest->idAuthentication =
 430                     (Z_IdAuthentication *)
 431                     odr_malloc (odr_encode(),
 432                                 sizeof(*initRequest->idAuthentication));
 433                 initRequest->idAuthentication->which =
 434                     Z_IdAuthentication_open;
 435                 initRequest->idAuthentication->u.open =
 436                     odr_strdup (odr_encode(), m_proxy_authentication);
 437             }
 438         }
 439
 440         // go through list of clients - and find the lowest/oldest one.
 441         Yaz_ProxyClient *c_min = 0;
 442         int min_seq = -1;
 443         int no_of_clients = 0;
 444         if (parent->m_clientPool)
 445             yaz_log (LOG_DEBUG, "Existing sessions");
 446         for (c = parent->m_clientPool; c; c = c->m_next)
 447         {
 448             yaz_log (LOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 449                                c->m_waiting, c->get_hostname(),
 450                                c->m_cookie ? c->m_cookie : "");
 451             no_of_clients++;
 452             if (min_seq < 0 || c->m_seqno < min_seq)
 453             {
 454                 min_seq = c->m_seqno;
 455                 c_min = c;
 456             }
 457         }
 458         if (no_of_clients >= parent->m_max_clients)
 459         {
 460             c = c_min;
 461             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 462             {
 463                 yaz_log (LOG_LOG, "%sMAXCLIENTS Destroy %d",
 464                          m_session_str, c->m_seqno);
 465                 if (c->m_server && c->m_server != this)
 466                     delete c->m_server;
 467                 c->m_server = 0;
 468             }
 469             else
 470             {
 471                 yaz_log (LOG_LOG, "%sMAXCLIENTS Reuse %d %d %s",
 472                          m_session_str,
 473                          c->m_seqno, parent->m_seqno, c->get_hostname());
 474                 xfree (c->m_cookie);
 475                 c->m_cookie = 0;
 476                 if (cookie)
 477                     c->m_cookie = xstrdup(cookie);
 478                 c->m_seqno = parent->m_seqno;
 479                 if (c->m_server && c->m_server != this)
 480                 {
 481                     c->m_server->m_client = 0;
 482                     delete c->m_server;
 483                 }
 484                 (parent->m_seqno)++;
 485                 c->m_target_idletime = m_target_idletime;
 486                 c->timeout(m_target_idletime);
 487
 488                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 489                     c->set_APDU_yazlog(1);
 490                 else
 491                     c->set_APDU_yazlog(0);
 492
 493                 return c;
 494             }
 495         }
 496         else
 497         {
 498             yaz_log (LOG_LOG, "%sNEW %d %s",
 499                      m_session_str, parent->m_seqno, m_proxyTarget);
 500             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 501             c->m_next = parent->m_clientPool;
 502             if (c->m_next)
 503                 c->m_next->m_prev = &c->m_next;
 504             parent->m_clientPool = c;
 505             c->m_prev = &parent->m_clientPool;
 506         }
 507
 508         xfree (c->m_cookie);
 509         c->m_cookie = 0;
 510         if (cookie)
 511             c->m_cookie = xstrdup(cookie);
 512
 513         c->m_seqno = parent->m_seqno;
 514         c->m_init_flag = 0;
 515         c->m_last_resultCount = 0;
 516         c->m_last_ok = 0;
 517         c->m_cache.clear();
 518         c->m_sr_transform = 0;
 519         c->m_waiting = 0;
 520         c->m_resultSetStartPoint = 0;
 521         (parent->m_seqno)++;
 522         if (c->client(m_proxyTarget))
 523         {
 524             delete c;
 525             return 0;
 526         }
 527         c->m_target_idletime = m_target_idletime;
 528         c->timeout(30);
 529
 530         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 531             c->set_APDU_yazlog(1);
 532         else
 533             c->set_APDU_yazlog(0);
 534     }
 535     yaz_log (LOG_DEBUG, "get_client 3 %p %p", this, c);
 536     return c;
 537 }
 538
 539 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 540 {
 541     int i;
 542     for (i = 0; i<num; i++)
 543     {
 544         oident *ent;
 545         Z_DefaultDiagFormat *r;
 546         Z_DiagRec *p = pp[i];
 547         if (p->which != Z_DiagRec_defaultFormat)
 548         {
 549             yaz_log(LOG_LOG, "%sError no diagnostics", m_session_str);
 550             return;
 551         }
 552         else
 553             r = p->u.defaultFormat;
 554         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 555             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 556             yaz_log(LOG_LOG, "%sError unknown diagnostic set", m_session_str);
 557         switch (r->which)
 558         {
 559         case Z_DefaultDiagFormat_v2Addinfo:
 560             yaz_log(LOG_LOG, "%sError %d %s:%s",
 561                     m_session_str,
 562                     *r->condition, diagbib1_str(*r->condition),
 563                     r->u.v2Addinfo);
 564             break;
 565         case Z_DefaultDiagFormat_v3Addinfo:
 566             yaz_log(LOG_LOG, "%sError %d %s:%s",
 567                     m_session_str,
 568                     *r->condition, diagbib1_str(*r->condition),
 569                     r->u.v3Addinfo);
 570             break;
 571         }
 572     }
 573 }
 574
 575 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p)
 576 {
 577     int i;
 578
 579     yaz_marc_t mt = yaz_marc_create();
 580     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 581     for (i = 0; i < p->num_records; i++)
 582     {
 583         Z_NamePlusRecord *npr = p->records[i];
 584         if (npr->which == Z_NamePlusRecord_databaseRecord)
 585         {
 586             Z_External *r = npr->u.databaseRecord;
 587             if (r->which == Z_External_octet)
 588             {
 589                 int rlen;
 590                 char *result;
 591                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 592                                         r->u.octet_aligned->len,
 593                                         &result, &rlen))
 594                 {
 595                     yaz_iconv_t cd = yaz_iconv_open("UTF-8", "MARC-8");
 596                     WRBUF wrbuf = wrbuf_alloc();
 597
 598                     char outbuf[120];
 599                     size_t inbytesleft = rlen;
 600                     const char *inp = result;
 601                     while (cd && inbytesleft)
 602                     {
 603                         size_t outbytesleft = sizeof(outbuf);
 604                         char *outp = outbuf;
 605                         size_t r;
 606
 607                         r = yaz_iconv (cd, (char**) &inp,
 608                                        &inbytesleft,
 609                                        &outp, &outbytesleft);
 610                         if (r == (size_t) (-1))
 611                         {
 612                             int e = yaz_iconv_error(cd);
 613                             if (e != YAZ_ICONV_E2BIG)
 614                             {
 615                                 yaz_log(LOG_WARN, "conversion failure");
 616                                 break;
 617                             }
 618                         }
 619                         wrbuf_write(wrbuf, outbuf, outp - outbuf);
 620                     }
 621                     if (cd)
 622                         yaz_iconv_close(cd);
 623
 624                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 625                                                          VAL_TEXT_XML,
 626                                                          wrbuf_buf(wrbuf),
 627                                                          wrbuf_len(wrbuf));
 628                     wrbuf_free(wrbuf, 1);
 629                 }
 630             }
 631         }
 632     }
 633     yaz_marc_destroy(mt);
 634 }
 635
 636 void Yaz_Proxy::logtime()
 637 {
 638     if (m_time_tv.tv_sec)
 639     {
 640         struct timeval tv;
 641         gettimeofday(&tv, 0);
 642         long diff = (tv.tv_sec - m_time_tv.tv_sec)*1000000 +
 643             (tv.tv_usec - m_time_tv.tv_usec);
 644         if (diff >= 0)
 645             yaz_log(LOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 646                     diff/1000000, (diff/1000)%1000);
 647     }
 648     m_time_tv.tv_sec = 0;
 649     m_time_tv.tv_usec = 0;
 650 }
 651
 652 int Yaz_Proxy::send_http_response(int code)
 653 {
 654     ODR o = odr_encode();
 655     const char *ctype = "text/xml";
 656     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 657     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 658     if (m_http_version)
 659         hres->version = odr_strdup(o, m_http_version);
 660     m_http_keepalive = 0;
 661     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 662     {
 663         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 664                  gdu_name(gdu));
 665     }
 666     int len;
 667     int r = send_GDU(gdu, &len);
 668     logtime();
 669     return r;
 670 }
 671
 672 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 673 {
 674     ODR o = odr_encode();
 675     const char *ctype = "text/xml";
 676     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 677     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 678     if (m_http_version)
 679         hres->version = odr_strdup(o, m_http_version);
 680     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 681     if (m_http_keepalive)
 682         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 683
 684     static Z_SOAP_Handler soap_handlers[2] = {
 685 #if HAVE_XML2
 686         {"http://www.loc.gov/zing/srw/", 0,
 687          (Z_SOAP_fun) yaz_srw_codec},
 688 #endif
 689         {0, 0, 0}
 690     };
 691
 692     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 693     soap_package->which = Z_SOAP_generic;
 694     soap_package->u.generic =
 695         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 696     soap_package->u.generic->no = 0;
 697     soap_package->u.generic->ns = soap_handlers[0].ns;
 698     soap_package->u.generic->p = (void *) srw_pdu;
 699     soap_package->ns = m_soap_ns;
 700     int ret = z_soap_codec_enc(o, &soap_package,
 701                                &hres->content_buf, &hres->content_len,
 702                                soap_handlers, 0);
 703     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 704     {
 705         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 706                  gdu_name(gdu));
 707     }
 708     int len;
 709     int r = send_GDU(gdu, &len);
 710     logtime();
 711     return r;
 712 }
 713
 714 int Yaz_Proxy::send_to_srw_client_error(int srw_error)
 715 {
 716     ODR o = odr_encode();
 717     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 718     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 719
 720     srw_res->num_diagnostics = 1;
 721     srw_res->diagnostics = (Z_SRW_diagnostic *)
 722         odr_malloc(o, sizeof(*srw_res->diagnostics));
 723     srw_res->diagnostics[0].code =  odr_intdup(o, srw_error);
 724     srw_res->diagnostics[0].details = 0;
 725     return send_srw_response(srw_pdu);
 726 }
 727
 728 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 729                              Z_DefaultDiagFormat *ddf)
 730 {
 731     int bib1_code = *ddf->condition;
 732     if (bib1_code == 109)
 733         return 404;
 734     srw_res->num_diagnostics = 1;
 735     srw_res->diagnostics = (Z_SRW_diagnostic *)
 736         odr_malloc(o, sizeof(*srw_res->diagnostics));
 737     srw_res->diagnostics[0].code =
 738         odr_intdup(o, yaz_diag_bib1_to_srw(*ddf->condition));
 739     srw_res->diagnostics[0].details = ddf->u.v2Addinfo;
 740     return 0;
 741 }
 742
 743 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
 744 {
 745     ODR o = odr_encode();
 746     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 747     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 748
 749     srw_res->numberOfRecords = odr_intdup (o, hits);
 750     if (records && records->which == Z_Records_DBOSD)
 751     {
 752         srw_res->num_records =
 753             records->u.databaseOrSurDiagnostics->num_records;
 754         int i;
 755         srw_res->records = (Z_SRW_record *)
 756             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
 757         for (i = 0; i < srw_res->num_records; i++)
 758         {
 759             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
 760             if (npr->which != Z_NamePlusRecord_databaseRecord)
 761             {
 762                 srw_res->records[i].recordSchema = "diagnostic";
 763                 srw_res->records[i].recordPacking = m_s2z_packing;
 764                 srw_res->records[i].recordData_buf = "67";
 765                 srw_res->records[i].recordData_len = 2;
 766                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 767                 continue;
 768             }
 769             Z_External *r = npr->u.databaseRecord;
 770             oident *ent = oid_getentbyoid(r->direct_reference);
 771             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
 772             {
 773                 srw_res->records[i].recordSchema = "http://www.loc.gov/marcxml/";
 774                 srw_res->records[i].recordPacking = m_s2z_packing;
 775                 srw_res->records[i].recordData_buf = (char*)
 776                     r->u.octet_aligned->buf;
 777                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
 778                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 779             }
 780             else
 781             {
 782                 srw_res->records[i].recordSchema = "diagnostic";
 783                 srw_res->records[i].recordPacking = m_s2z_packing;
 784                 srw_res->records[i].recordData_buf = "67";
 785                 srw_res->records[i].recordData_len = 2;
 786                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 787             }
 788         }
 789     }
 790     if (records && records->which == Z_Records_NSD)
 791     {
 792         int http_code;
 793         http_code = z_to_srw_diag(odr_encode(), srw_res,
 794                                    records->u.nonSurrogateDiagnostic);
 795         if (http_code)
 796             return send_http_response(http_code);
 797     }
 798     return send_srw_response(srw_pdu);
 799
 800 }
 801
 802 int Yaz_Proxy::send_srw_explain()
 803 {
 804     Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
 805     Z_SRW_explainResponse *er = res->u.explain_response;
 806
 807     Yaz_ProxyConfig *cfg = check_reconfigure();
 808     if (cfg)
 809     {
 810         int len;
 811         assert (m_proxyTarget);
 812         char *b = cfg->get_explain(odr_encode(), 0 /* target */,
 813                                    0 /* db */, &len);
 814         if (b)
 815         {
 816             er->record.recordData_buf = b;
 817             er->record.recordData_len = len;
 818             er->record.recordPacking = m_s2z_packing;
 819         }
 820     }
 821     return send_srw_response(res);
 822 }
 823
 824 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu, int *len)
 825 {
 826     if (m_http_version)
 827     {
 828         if (apdu->which == Z_APDU_initResponse)
 829         {
 830             Z_InitResponse *res = apdu->u.initResponse;
 831             if (*res->result == 0)
 832             {
 833                 send_to_srw_client_error(3);
 834             }
 835             else if (!m_s2z_search_apdu)
 836             {
 837                 send_srw_explain();
 838             }
 839             else
 840             {
 841                 handle_incoming_Z_PDU(m_s2z_search_apdu);
 842             }
 843         }
 844         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
 845         {
 846             m_s2z_search_apdu = 0;
 847             Z_SearchResponse *res = apdu->u.searchResponse;
 848             m_s2z_hit_count = *res->resultCount;
 849             if (res->records && res->records->which == Z_Records_NSD)
 850             {
 851                 send_to_srw_client_ok(0, res->records, 1);
 852             }
 853             else if (m_s2z_present_apdu)
 854             {
 855                 handle_incoming_Z_PDU(m_s2z_present_apdu);
 856             }
 857             else
 858             {
 859                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
 860             }
 861         }
 862         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
 863         {
 864             int start =
 865                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
 866
 867             m_s2z_present_apdu = 0;
 868             Z_PresentResponse *res = apdu->u.presentResponse;
 869             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
 870         }
 871     }
 872     else
 873     {
 874         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 875             yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 876                      apdu_name(apdu));
 877         int r = send_Z_PDU(apdu, len);
 878         logtime();
 879         return r;
 880     }
 881     return 0;
 882 }
 883
 884 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 885 {
 886     int len = 0;
 887     int kill_session = 0;
 888     if (apdu->which == Z_APDU_searchResponse)
 889     {
 890         Z_SearchResponse *sr = apdu->u.searchResponse;
 891         Z_Records *p = sr->records;
 892         if (p && p->which == Z_Records_NSD)
 893         {
 894             Z_DiagRec dr, *dr_p = &dr;
 895             dr.which = Z_DiagRec_defaultFormat;
 896             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 897
 898             *sr->searchStatus = 0;
 899             display_diagrecs(&dr_p, 1);
 900         }
 901         else
 902         {
 903             if (m_marcxml_flag && p && p->which == Z_Records_DBOSD)
 904                 convert_to_marcxml(p->u.databaseOrSurDiagnostics);
 905             if (sr->resultCount)
 906             {
 907                 yaz_log(LOG_LOG, "%s%d hits", m_session_str,
 908                         *sr->resultCount);
 909                 if (*sr->resultCount < 0)
 910                 {
 911                     m_invalid_session = 1;
 912                     kill_session = 1;
 913
 914                     *sr->searchStatus = 0;
 915                     sr->records =
 916                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
 917                     *sr->resultCount = 0;
 918                 }
 919             }
 920         }
 921     }
 922     else if (apdu->which == Z_APDU_presentResponse)
 923     {
 924         Z_PresentResponse *sr = apdu->u.presentResponse;
 925         Z_Records *p = sr->records;
 926         if (p && p->which == Z_Records_NSD)
 927         {
 928             Z_DiagRec dr, *dr_p = &dr;
 929             dr.which = Z_DiagRec_defaultFormat;
 930             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 931             if (*sr->presentStatus == Z_PresentStatus_success)
 932                 *sr->presentStatus = Z_PresentStatus_failure;
 933             display_diagrecs(&dr_p, 1);
 934         }
 935         if (m_marcxml_flag && p && p->which == Z_Records_DBOSD)
 936             convert_to_marcxml(p->u.databaseOrSurDiagnostics);
 937     }
 938     int r = send_PDU_convert(apdu, &len);
 939     if (r)
 940         return r;
 941     m_bytes_sent += len;
 942     m_bw_stat.add_bytes(len);
 943     if (kill_session)
 944     {
 945         delete m_client;
 946         m_client = 0;
 947         m_parent->pre_init();
 948     }
 949     if (m_http_version)
 950     {
 951         if (!m_http_keepalive)
 952         {
 953 #if 1
 954             timeout(1);
 955 #else
 956             shutdown();
 957             return -1;
 958 #endif
 959         }
 960     }
 961     return r;
 962 }
 963
 964 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
 965 {
 966     int len = 0;
 967     int r = send_Z_PDU(apdu, &len);
 968     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
 969         yaz_log (LOG_LOG, "%sSending %s to %s %d bytes",
 970                  get_session_str(),
 971                  apdu_name(apdu), get_hostname(), len);
 972     m_bytes_sent += len;
 973     return r;
 974 }
 975
 976 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
 977 {
 978     if (apdu->which == Z_APDU_presentRequest)
 979     {
 980         Z_PresentRequest *pr = apdu->u.presentRequest;
 981         int toget = *pr->numberOfRecordsRequested;
 982         int start = *pr->resultSetStartPoint;
 983
 984         yaz_log(LOG_LOG, "%sPresent %s %d+%d", m_session_str,
 985                 pr->resultSetId, start, toget);
 986
 987         if (*m_parent->m_optimize == '0')
 988             return apdu;
 989
 990         if (!m_client->m_last_resultSetId)
 991         {
 992             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
 993             new_apdu->u.presentResponse->records =
 994                 create_nonSurrogateDiagnostics(odr_encode(), 30,
 995                                                pr->resultSetId);
 996             send_to_client(new_apdu);
 997             return 0;
 998         }
 999         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1000         {
1001             if (start+toget-1 > m_client->m_last_resultCount)
1002             {
1003                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1004                 new_apdu->u.presentResponse->records =
1005                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1006                 send_to_client(new_apdu);
1007                 return 0;
1008             }
1009             Z_NamePlusRecordList *npr;
1010             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1011                                           pr->preferredRecordSyntax,
1012                                           pr->recordComposition))
1013             {
1014                 yaz_log (LOG_LOG, "%sReturned cached records for present request",
1015                          m_session_str);
1016                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1017                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1018
1019                 new_apdu->u.presentResponse->numberOfRecordsReturned
1020                     = odr_intdup(odr_encode(), toget);
1021
1022                 new_apdu->u.presentResponse->records = (Z_Records*)
1023                     odr_malloc(odr_encode(), sizeof(Z_Records));
1024                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1025                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1026                 new_apdu->u.presentResponse->nextResultSetPosition =
1027                     odr_intdup(odr_encode(), start+toget);
1028
1029                 send_to_client(new_apdu);
1030                 return 0;
1031             }
1032         }
1033     }
1034
1035     if (apdu->which != Z_APDU_searchRequest)
1036         return apdu;
1037     Z_SearchRequest *sr = apdu->u.searchRequest;
1038     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1039     Yaz_Z_Databases this_databases;
1040
1041     this_databases.set(sr->num_databaseNames, (const char **)
1042                        sr->databaseNames);
1043
1044     this_query->set_Z_Query(sr->query);
1045
1046     char query_str[120];
1047     this_query->print(query_str, sizeof(query_str)-1);
1048     yaz_log(LOG_LOG, "%sSearch %s", m_session_str, query_str);
1049
1050     if (*m_parent->m_optimize != '0' &&
1051         m_client->m_last_ok && m_client->m_last_query &&
1052         m_client->m_last_query->match(this_query) &&
1053         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1054         m_client->m_last_databases.match(this_databases))
1055     {
1056         delete this_query;
1057         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1058             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1059         {
1060             Z_NamePlusRecordList *npr;
1061             int toget = *sr->mediumSetPresentNumber;
1062             Z_RecordComposition *comp = 0;
1063
1064             if (toget > m_client->m_last_resultCount)
1065                 toget = m_client->m_last_resultCount;
1066
1067             if (sr->mediumSetElementSetNames)
1068             {
1069                 comp = (Z_RecordComposition *)
1070                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1071                 comp->which = Z_RecordComp_simple;
1072                 comp->u.simple = sr->mediumSetElementSetNames;
1073             }
1074
1075             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1076                                           sr->preferredRecordSyntax, comp))
1077             {
1078                 yaz_log (LOG_LOG, "%sReturned cached records for medium set",
1079                          m_session_str);
1080                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1081                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1082                 new_apdu->u.searchResponse->resultCount =
1083                     &m_client->m_last_resultCount;
1084
1085                 new_apdu->u.searchResponse->numberOfRecordsReturned
1086                     = odr_intdup(odr_encode(), toget);
1087
1088                 new_apdu->u.searchResponse->presentStatus =
1089                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1090                 new_apdu->u.searchResponse->records = (Z_Records*)
1091                     odr_malloc(odr_encode(), sizeof(Z_Records));
1092                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1093                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1094                 new_apdu->u.searchResponse->nextResultSetPosition =
1095                     odr_intdup(odr_encode(), toget+1);
1096                 send_to_client(new_apdu);
1097                 return 0;
1098             }
1099             else
1100             {
1101                 // medium Set
1102                 // send present request (medium size)
1103                 yaz_log (LOG_LOG, "%sOptimizing search for medium set",
1104                          m_session_str);
1105
1106                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1107                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1108                 pr->referenceId = sr->referenceId;
1109                 pr->resultSetId = sr->resultSetName;
1110                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1111                 *pr->numberOfRecordsRequested = toget;
1112                 pr->recordComposition = comp;
1113                 m_client->m_sr_transform = 1;
1114                 return new_apdu;
1115             }
1116         }
1117         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1118             m_client->m_last_resultCount <= 0)
1119         {
1120             // large set. Return pseudo-search response immediately
1121             yaz_log (LOG_LOG, "%sOptimizing search for large set",
1122                      m_session_str);
1123             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1124             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1125             new_apdu->u.searchResponse->resultCount =
1126                 &m_client->m_last_resultCount;
1127             send_to_client(new_apdu);
1128             return 0;
1129         }
1130         else
1131         {
1132             Z_NamePlusRecordList *npr;
1133             int toget = m_client->m_last_resultCount;
1134             Z_RecordComposition *comp = 0;
1135             // small set
1136             // send a present request (small set)
1137
1138             if (sr->smallSetElementSetNames)
1139             {
1140                 comp = (Z_RecordComposition *)
1141                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1142                 comp->which = Z_RecordComp_simple;
1143                 comp->u.simple = sr->smallSetElementSetNames;
1144             }
1145
1146             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1147                                           sr->preferredRecordSyntax, comp))
1148             {
1149                 yaz_log (LOG_LOG, "%sReturned cached records for small set",
1150                          m_session_str);
1151                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1152                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1153                 new_apdu->u.searchResponse->resultCount =
1154                     &m_client->m_last_resultCount;
1155
1156                 new_apdu->u.searchResponse->numberOfRecordsReturned
1157                     = odr_intdup(odr_encode(), toget);
1158
1159                 new_apdu->u.searchResponse->presentStatus =
1160                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1161                 new_apdu->u.searchResponse->records = (Z_Records*)
1162                     odr_malloc(odr_encode(), sizeof(Z_Records));
1163                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1164                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1165                 new_apdu->u.searchResponse->nextResultSetPosition =
1166                     odr_intdup(odr_encode(), toget+1);
1167                 send_to_client(new_apdu);
1168                 return 0;
1169             }
1170             else
1171             {
1172                 yaz_log (LOG_LOG, "%sOptimizing search for small set",
1173                          m_session_str);
1174                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1175                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1176                 pr->referenceId = sr->referenceId;
1177                 pr->resultSetId = sr->resultSetName;
1178                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1179                 *pr->numberOfRecordsRequested = toget;
1180                 pr->recordComposition = comp;
1181                 m_client->m_sr_transform = 1;
1182                 return new_apdu;
1183             }
1184         }
1185     }
1186     else  // query doesn't match
1187     {
1188         delete m_client->m_last_query;
1189         m_client->m_last_query = this_query;
1190         m_client->m_last_ok = 0;
1191         m_client->m_cache.clear();
1192         m_client->m_resultSetStartPoint = 0;
1193
1194         xfree (m_client->m_last_resultSetId);
1195         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1196
1197         m_client->m_last_databases.set(sr->num_databaseNames,
1198                                        (const char **) sr->databaseNames);
1199     }
1200     return apdu;
1201 }
1202
1203
1204 void Yaz_Proxy::inc_request_no()
1205 {
1206     char *cp = strchr(m_session_str, ' ');
1207     m_request_no++;
1208     if (cp)
1209         sprintf(cp+1, "%d ", m_request_no);
1210 }
1211
1212 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1213 {
1214     inc_request_no();
1215
1216     m_bytes_recv += len;
1217
1218     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
1219         yaz_log (LOG_DEBUG, "%sReceiving %s from client %d bytes",
1220                  m_session_str, gdu_name(apdu), len);
1221
1222     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1223         shutdown();
1224
1225     m_bw_stat.add_bytes(len);
1226     m_pdu_stat.add_bytes(1);
1227
1228     gettimeofday(&m_time_tv, 0);
1229
1230     int bw_total = m_bw_stat.get_total();
1231     int pdu_total = m_pdu_stat.get_total();
1232
1233     int reduce = 0;
1234     if (m_bw_max)
1235     {
1236         if (bw_total > m_bw_max)
1237         {
1238             reduce = (bw_total/m_bw_max);
1239         }
1240     }
1241     if (m_pdu_max)
1242     {
1243         if (pdu_total > m_pdu_max)
1244         {
1245             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1246             reduce = (reduce > nreduce) ? reduce : nreduce;
1247         }
1248     }
1249     if (reduce)
1250     {
1251         yaz_log(LOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1252                 m_session_str, reduce, bw_total, pdu_total,
1253                 m_bw_max, m_pdu_max);
1254
1255         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1256         timeout(reduce);       // call us reduce seconds later
1257     }
1258     else if (apdu->which == Z_GDU_Z3950)
1259         handle_incoming_Z_PDU(apdu->u.z3950);
1260     else if (apdu->which == Z_GDU_HTTP_Request)
1261         handle_incoming_HTTP(apdu->u.HTTP_Request);
1262 }
1263
1264 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1265 {
1266     if (m_max_record_retrieve)
1267     {
1268         if (apdu->which == Z_APDU_presentRequest)
1269         {
1270             Z_PresentRequest *pr = apdu->u.presentRequest;
1271             if (pr->numberOfRecordsRequested &&
1272                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1273                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1274         }
1275     }
1276 }
1277
1278 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1279                                                      int error,
1280                                                      const char *addinfo)
1281 {
1282     Z_Records *rec = (Z_Records *)
1283         odr_malloc (odr, sizeof(*rec));
1284     int *err = (int *)
1285         odr_malloc (odr, sizeof(*err));
1286     Z_DiagRec *drec = (Z_DiagRec *)
1287         odr_malloc (odr, sizeof(*drec));
1288     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1289         odr_malloc (odr, sizeof(*dr));
1290     *err = error;
1291     rec->which = Z_Records_NSD;
1292     rec->u.nonSurrogateDiagnostic = dr;
1293     dr->diagnosticSetId =
1294         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1295     dr->condition = err;
1296     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1297     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1298     return rec;
1299 }
1300
1301 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1302 {
1303     if (apdu->which == Z_APDU_searchRequest &&
1304         apdu->u.searchRequest->query &&
1305         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1306         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1307     {
1308         Z_RPNQuery *rpnquery = 0;
1309         Z_SearchRequest *sr = apdu->u.searchRequest;
1310
1311         yaz_log(LOG_LOG, "%sCQL: %s", m_session_str,
1312                 sr->query->u.type_104->u.cql);
1313
1314         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1315                                           &rpnquery, odr_encode());
1316         if (r == -3)
1317             yaz_log(LOG_LOG, "%sNo CQL to RPN table", m_session_str);
1318         else if (r)
1319             yaz_log(LOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1320         else
1321         {
1322             sr->query->which = Z_Query_type_1;
1323             sr->query->u.type_1 = rpnquery;
1324         }
1325         return apdu;
1326     }
1327     return apdu;
1328 }
1329
1330 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1331 {
1332     if (apdu->which == Z_APDU_searchRequest)
1333     {
1334         Z_SearchRequest *sr = apdu->u.searchRequest;
1335         int err = 0;
1336         char *addinfo = 0;
1337
1338         Yaz_ProxyConfig *cfg = check_reconfigure();
1339         if (cfg)
1340             err = cfg->check_query(odr_encode(), m_default_target,
1341                                    sr->query, &addinfo);
1342         if (err)
1343         {
1344             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1345
1346             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1347             new_apdu->u.searchResponse->records =
1348                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1349             *new_apdu->u.searchResponse->searchStatus = 0;
1350
1351             send_to_client(new_apdu);
1352
1353             return 0;
1354         }
1355     }
1356     return apdu;
1357 }
1358
1359 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1360 {
1361     m_marcxml_flag = 0;
1362     if (apdu->which == Z_APDU_searchRequest)
1363     {
1364         Z_SearchRequest *sr = apdu->u.searchRequest;
1365         int err = 0;
1366         char *addinfo = 0;
1367         Yaz_ProxyConfig *cfg = check_reconfigure();
1368
1369         if (cfg)
1370             err = cfg->check_syntax(odr_encode(),
1371                                     m_default_target,
1372                                     sr->preferredRecordSyntax,
1373                                     &addinfo);
1374         if (err == -1)
1375         {
1376             sr->preferredRecordSyntax =
1377                 yaz_oidval_to_z3950oid(odr_decode(), CLASS_RECSYN,
1378                                        VAL_USMARC);
1379             m_marcxml_flag = 1;
1380         }
1381         else if (err)
1382         {
1383             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1384
1385             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1386             new_apdu->u.searchResponse->records =
1387                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1388             *new_apdu->u.searchResponse->searchStatus = 0;
1389
1390             send_to_client(new_apdu);
1391
1392             return 0;
1393         }
1394     }
1395     else if (apdu->which == Z_APDU_presentRequest)
1396     {
1397         Z_PresentRequest *pr = apdu->u.presentRequest;
1398         int err = 0;
1399         char *addinfo = 0;
1400         Yaz_ProxyConfig *cfg = check_reconfigure();
1401
1402         if (cfg)
1403             err = cfg->check_syntax(odr_encode(), m_default_target,
1404                                     pr->preferredRecordSyntax,
1405                                     &addinfo);
1406         if (err == -1)
1407         {
1408             pr->preferredRecordSyntax =
1409                 yaz_oidval_to_z3950oid(odr_decode(), CLASS_RECSYN,
1410                                        VAL_USMARC);
1411             m_marcxml_flag = 1;
1412         }
1413         else if (err)
1414         {
1415             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1416
1417             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1418             new_apdu->u.presentResponse->records =
1419                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1420             *new_apdu->u.presentResponse->presentStatus =
1421                 Z_PresentStatus_failure;
1422
1423             send_to_client(new_apdu);
1424
1425             return 0;
1426         }
1427     }
1428     return apdu;
1429 }
1430
1431 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1432 {
1433     Z_SRW_PDU *srw_pdu = 0;
1434     char *soap_ns = 0;
1435     if (m_s2z_odr_init)
1436     {
1437         odr_destroy(m_s2z_odr_init);
1438         m_s2z_odr_init = 0;
1439     }
1440     if (m_s2z_odr_search)
1441     {
1442         odr_destroy(m_s2z_odr_search);
1443         m_s2z_odr_search = 0;
1444     }
1445
1446     m_http_keepalive = 0;
1447     m_http_version = 0;
1448     if (!strcmp(hreq->version, "1.0"))
1449     {
1450         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1451         if (v && !strcmp(v, "Keep-Alive"))
1452             m_http_keepalive = 1;
1453         else
1454             m_http_keepalive = 0;
1455         m_http_version = "1.0";
1456     }
1457     else
1458     {
1459         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1460         if (v && !strcmp(v, "close"))
1461             m_http_keepalive = 0;
1462         else
1463             m_http_keepalive = 1;
1464         m_http_version = "1.1";
1465     }
1466
1467     if (yaz_check_for_srw(hreq, &srw_pdu, &soap_ns, odr_decode()) == 0
1468         || yaz_check_for_sru(hreq, &srw_pdu, &soap_ns, odr_decode()) == 0)
1469     {
1470         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
1471         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
1472         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_ns);
1473         m_s2z_init_apdu = 0;
1474         m_s2z_search_apdu = 0;
1475         m_s2z_present_apdu = 0;
1476         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
1477         {
1478             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
1479
1480             // set packing for response records ..
1481             if (srw_req->recordPacking &&
1482                 !strcmp(srw_req->recordPacking, "xml"))
1483                 m_s2z_packing = Z_SRW_recordPacking_XML;
1484             else
1485                 m_s2z_packing = Z_SRW_recordPacking_string;
1486
1487             // prepare search PDU
1488             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
1489                                           Z_APDU_searchRequest);
1490             Z_SearchRequest *z_searchRequest =
1491                 m_s2z_search_apdu->u.searchRequest;
1492
1493             z_searchRequest->num_databaseNames = 1;
1494             z_searchRequest->databaseNames = (char**)
1495                 odr_malloc(m_s2z_odr_search, sizeof(char *));
1496             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
1497                                                            srw_req->database);
1498
1499             // query transformation
1500             Z_Query *query = (Z_Query *)
1501                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
1502             z_searchRequest->query = query;
1503
1504             if (srw_req->query_type == Z_SRW_query_type_cql)
1505             {
1506                 Z_External *ext = (Z_External *)
1507                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
1508                 ext->direct_reference =
1509                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
1510                 ext->indirect_reference = 0;
1511                 ext->descriptor = 0;
1512                 ext->which = Z_External_CQL;
1513                 ext->u.cql = srw_req->query.cql;
1514
1515                 query->which = Z_Query_type_104;
1516                 query->u.type_104 =  ext;
1517             }
1518             else if (srw_req->query_type == Z_SRW_query_type_pqf)
1519             {
1520                 Z_RPNQuery *RPNquery;
1521                 YAZ_PQF_Parser pqf_parser;
1522
1523                 pqf_parser = yaz_pqf_create ();
1524
1525                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
1526                                           srw_req->query.pqf);
1527                 if (!RPNquery)
1528                 {
1529                     const char *pqf_msg;
1530                     size_t off;
1531                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
1532                     yaz_log(LOG_LOG, "%*s^\n", off+4, "");
1533                     yaz_log(LOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
1534
1535                     send_to_srw_client_error(10);
1536                     return;
1537                 }
1538                 query->which = Z_Query_type_1;
1539                 query->u.type_1 =  RPNquery;
1540
1541                 yaz_pqf_destroy (pqf_parser);
1542             }
1543             else
1544             {
1545                 send_to_srw_client_error(11);
1546                 return;
1547             }
1548
1549             // present
1550             m_s2z_present_apdu = 0;
1551             int max = 0;
1552             if (srw_req->maximumRecords)
1553                 max = *srw_req->maximumRecords;
1554             int start = 1;
1555             if (srw_req->startRecord)
1556                 start = *srw_req->startRecord;
1557             if (max > 0)
1558             {
1559                 if (start <= 1)  // Z39.50 piggyback
1560                 {
1561                     *z_searchRequest->smallSetUpperBound = max;
1562                     *z_searchRequest->mediumSetPresentNumber = max;
1563                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
1564                     z_searchRequest->preferredRecordSyntax =
1565                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1566                                                VAL_TEXT_XML);
1567                 }
1568                 else   // Z39.50 present
1569                 {
1570                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
1571                                                    Z_APDU_presentRequest);
1572                     Z_PresentRequest *z_presentRequest =
1573                         m_s2z_present_apdu->u.presentRequest;
1574                     *z_presentRequest->resultSetStartPoint = start;
1575                     *z_presentRequest->numberOfRecordsRequested = max;
1576                     z_presentRequest->preferredRecordSyntax =
1577                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1578                                                VAL_TEXT_XML);
1579                 }
1580             }
1581             if (!m_client)
1582             {
1583                 yaz_log(LOG_LOG, "handle_incoming: initRequest");
1584                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1585                                             Z_APDU_initRequest);
1586
1587                 // prevent m_initRequest_apdu memory from being grabbed
1588                 // in Yaz_Proxy::handle_incoming_Z_PDU
1589                 m_initRequest_apdu = m_s2z_init_apdu;
1590                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1591                 return;
1592             }
1593             else
1594             {
1595                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1596                 return;
1597             }
1598         }
1599         else if (srw_pdu->which == Z_SRW_explain_request)
1600         {
1601             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
1602
1603             if (srw_req->recordPacking &&
1604                 !strcmp(srw_req->recordPacking, "xml"))
1605                 m_s2z_packing = Z_SRW_recordPacking_XML;
1606             else
1607                 m_s2z_packing = Z_SRW_recordPacking_string;
1608
1609             if (!m_client)
1610             {
1611                 yaz_log(LOG_LOG, "handle_incoming: initRequest");
1612                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1613                                             Z_APDU_initRequest);
1614
1615                 // prevent m_initRequest_apdu memory from being grabbed
1616                 // in Yaz_Proxy::handle_incoming_Z_PDU
1617                 m_initRequest_apdu = m_s2z_init_apdu;
1618                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1619             }
1620             else
1621                 send_srw_explain();
1622             return;
1623         }
1624     }
1625     int len = 0;
1626     Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
1627     send_GDU(p, &len);
1628     timeout(1);
1629 }
1630
1631 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
1632 {
1633     if (!m_client && m_invalid_session)
1634     {
1635         m_apdu_invalid_session = apdu;
1636         m_mem_invalid_session = odr_extract_mem(odr_decode());
1637         apdu = m_initRequest_apdu;
1638     }
1639
1640     // Determine our client.
1641     Z_OtherInformation **oi;
1642     get_otherInfoAPDU(apdu, &oi);
1643     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
1644     if (!m_client)
1645     {
1646         delete this;
1647         return;
1648     }
1649     m_client->m_server = this;
1650
1651     if (apdu->which == Z_APDU_initRequest)
1652     {
1653         if (apdu->u.initRequest->implementationId)
1654             yaz_log(LOG_LOG, "%simplementationId: %s",
1655                     m_session_str, apdu->u.initRequest->implementationId);
1656         if (apdu->u.initRequest->implementationName)
1657             yaz_log(LOG_LOG, "%simplementationName: %s",
1658                     m_session_str, apdu->u.initRequest->implementationName);
1659         if (apdu->u.initRequest->implementationVersion)
1660             yaz_log(LOG_LOG, "%simplementationVersion: %s",
1661                     m_session_str, apdu->u.initRequest->implementationVersion);
1662         if (m_initRequest_apdu == 0)
1663         {
1664             if (m_initRequest_mem)
1665                 nmem_destroy(m_initRequest_mem);
1666             m_initRequest_apdu = apdu;
1667             m_initRequest_mem = odr_extract_mem(odr_decode());
1668         }
1669         if (m_client->m_init_flag)
1670         {
1671             if (handle_init_response_for_invalid_session(apdu))
1672                 return;
1673             Z_APDU *apdu2 = m_client->m_initResponse;
1674             apdu2->u.initResponse->otherInfo = 0;
1675             if (m_client->m_cookie && *m_client->m_cookie)
1676                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
1677                                            m_client->m_cookie);
1678             apdu2->u.initResponse->referenceId =
1679                 apdu->u.initRequest->referenceId;
1680             send_to_client(apdu2);
1681             return;
1682         }
1683         m_client->m_init_flag = 1;
1684     }
1685     handle_max_record_retrieve(apdu);
1686
1687     if (apdu)
1688         apdu = handle_syntax_validation(apdu);
1689
1690     if (apdu)
1691         apdu = handle_query_transformation(apdu);
1692
1693     if (apdu)
1694         apdu = handle_query_validation(apdu);
1695
1696     if (apdu)
1697         apdu = result_set_optimize(apdu);
1698     if (!apdu)
1699     {
1700         m_client->timeout(m_target_idletime);  // mark it active even
1701         // though we didn't use it
1702         return;
1703     }
1704
1705     // delete other info part from PDU before sending to target
1706     get_otherInfoAPDU(apdu, &oi);
1707     if (oi)
1708         *oi = 0;
1709
1710     if (apdu->which == Z_APDU_presentRequest &&
1711         m_client->m_resultSetStartPoint == 0)
1712     {
1713         Z_PresentRequest *pr = apdu->u.presentRequest;
1714         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
1715         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
1716     } else {
1717         m_client->m_resultSetStartPoint = 0;
1718     }
1719     if (m_client->send_to_target(apdu) < 0)
1720     {
1721         delete m_client;
1722         m_client = 0;
1723         delete this;
1724     }
1725     else
1726         m_client->m_waiting = 1;
1727 }
1728
1729 void Yaz_Proxy::connectNotify()
1730 {
1731 }
1732
1733 void Yaz_Proxy::shutdown()
1734 {
1735     m_invalid_session = 0;
1736     // only keep if keep_alive flag is set...
1737     if (m_client &&
1738         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
1739         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
1740         m_client->m_waiting == 0)
1741     {
1742         yaz_log(LOG_LOG, "%sShutdown (client to proxy) keepalive %s",
1743                  m_session_str,
1744                  m_client->get_hostname());
1745         yaz_log(LOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1746                 m_session_str, m_client->m_pdu_recv,
1747                 m_client->m_bytes_sent + m_client->m_bytes_recv,
1748                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
1749         assert (m_client->m_waiting != 2);
1750         // Tell client (if any) that no server connection is there..
1751         m_client->m_server = 0;
1752         m_invalid_session = 0;
1753     }
1754     else if (m_client)
1755     {
1756         yaz_log (LOG_LOG, "%sShutdown (client to proxy) close %s",
1757                  m_session_str,
1758                  m_client->get_hostname());
1759         assert (m_client->m_waiting != 2);
1760         delete m_client;
1761     }
1762     else if (!m_parent)
1763     {
1764         yaz_log (LOG_LOG, "%sshutdown (client to proxy) bad state",
1765                  m_session_str);
1766         assert (m_parent);
1767     }
1768     else
1769     {
1770         yaz_log (LOG_LOG, "%sShutdown (client to proxy)",
1771                  m_session_str);
1772     }
1773     if (m_parent)
1774         m_parent->pre_init();
1775     delete this;
1776 }
1777
1778 const char *Yaz_ProxyClient::get_session_str()
1779 {
1780     if (!m_server)
1781         return "0 ";
1782     return m_server->get_session_str();
1783 }
1784
1785 void Yaz_ProxyClient::shutdown()
1786 {
1787     yaz_log (LOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
1788              get_hostname());
1789     delete m_server;
1790     delete this;
1791 }
1792
1793 void Yaz_Proxy::failNotify()
1794 {
1795     inc_request_no();
1796     yaz_log (LOG_LOG, "%sConnection closed by client",
1797              get_session_str());
1798     shutdown();
1799 }
1800
1801 void Yaz_ProxyClient::failNotify()
1802 {
1803     if (m_server)
1804         m_server->inc_request_no();
1805     yaz_log (LOG_LOG, "%sConnection closed by target %s",
1806              get_session_str(), get_hostname());
1807     shutdown();
1808 }
1809
1810 void Yaz_ProxyClient::connectNotify()
1811 {
1812     const char *s = get_session_str();
1813     const char *h = get_hostname();
1814     yaz_log (LOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
1815              m_target_idletime);
1816     timeout(m_target_idletime);
1817     if (!m_server)
1818         pre_init_client();
1819 }
1820
1821 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
1822                                                   *the_PDU_Observable, int fd)
1823 {
1824     return new Yaz_ProxyClient(the_PDU_Observable, 0);
1825 }
1826
1827 Yaz_ProxyClient::~Yaz_ProxyClient()
1828 {
1829     if (m_prev)
1830         *m_prev = m_next;
1831     if (m_next)
1832         m_next->m_prev = m_prev;
1833     m_waiting = 2;     // for debugging purposes only.
1834     odr_destroy(m_init_odr);
1835     delete m_last_query;
1836     xfree (m_last_resultSetId);
1837     xfree (m_cookie);
1838 }
1839
1840 void Yaz_ProxyClient::pre_init_client()
1841 {
1842     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
1843     Z_InitRequest *req = apdu->u.initRequest;
1844
1845     ODR_MASK_SET(req->options, Z_Options_search);
1846     ODR_MASK_SET(req->options, Z_Options_present);
1847     ODR_MASK_SET(req->options, Z_Options_namedResultSets);
1848     ODR_MASK_SET(req->options, Z_Options_triggerResourceCtrl);
1849     ODR_MASK_SET(req->options, Z_Options_scan);
1850     ODR_MASK_SET(req->options, Z_Options_sort);
1851     ODR_MASK_SET(req->options, Z_Options_extendedServices);
1852     ODR_MASK_SET(req->options, Z_Options_delSet);
1853
1854     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
1855     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
1856     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3);
1857
1858     if (send_to_target(apdu) < 0)
1859     {
1860         delete this;
1861     }
1862     else
1863     {
1864         m_waiting = 1;
1865         m_init_flag = 1;
1866     }
1867 }
1868
1869 void Yaz_Proxy::pre_init()
1870 {
1871     int i;
1872     const char *name = 0;
1873     const char *zurl_in_use[MAX_ZURL_PLEX];
1874     int limit_bw, limit_pdu, limit_req;
1875     int target_idletime, client_idletime;
1876     int max_clients;
1877     int keepalive_limit_bw, keepalive_limit_pdu;
1878     int pre_init;
1879     const char *cql2rpn = 0;
1880     const char *zeerex = 0;
1881
1882     Yaz_ProxyConfig *cfg = check_reconfigure();
1883
1884     zurl_in_use[0] = 0;
1885
1886     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
1887         set_APDU_yazlog(1);
1888     else
1889         set_APDU_yazlog(0);
1890
1891     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
1892                                           &limit_bw, &limit_pdu, &limit_req,
1893                                           &target_idletime, &client_idletime,
1894                                           &max_clients,
1895                                           &keepalive_limit_bw,
1896                                           &keepalive_limit_pdu,
1897                                           &pre_init,
1898                                           &cql2rpn) ; i++)
1899     {
1900         if (pre_init)
1901         {
1902             int j;
1903             for (j = 0; zurl_in_use[j]; j++)
1904             {
1905                 Yaz_ProxyClient *c;
1906                 int spare = 0;
1907                 int in_use = 0;
1908                 int other = 0;
1909                 for (c = m_clientPool; c; c = c->m_next)
1910                 {
1911                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
1912                     {
1913                         if (c->m_cookie == 0)
1914                         {
1915                             if (c->m_server == 0)
1916                                 spare++;
1917                             else
1918                                 in_use++;
1919                         }
1920                         else
1921                             other++;
1922                     }
1923                 }
1924                 yaz_log(LOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
1925                         "preinit=%d",m_session_str,
1926                         name, zurl_in_use[j], in_use, other, spare, pre_init);
1927                 if (spare < pre_init)
1928                 {
1929                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
1930                     c->m_next = m_clientPool;
1931                     if (c->m_next)
1932                         c->m_next->m_prev = &c->m_next;
1933                     m_clientPool = c;
1934                     c->m_prev = &m_clientPool;
1935
1936                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
1937                         c->set_APDU_yazlog(1);
1938                     else
1939                         c->set_APDU_yazlog(0);
1940
1941                     if (c->client(zurl_in_use[j]))
1942                     {
1943                         timeout(60);
1944                         delete c;
1945                         return;
1946                     }
1947                     c->timeout(30);
1948                     c->m_waiting = 1;
1949                     c->m_target_idletime = target_idletime;
1950                     c->m_seqno = m_seqno++;
1951                 }
1952             }
1953         }
1954     }
1955 }
1956
1957 void Yaz_Proxy::timeoutNotify()
1958 {
1959     if (m_parent)
1960     {
1961         if (m_bw_hold_PDU)
1962         {
1963             timeout(m_client_idletime);
1964             Z_GDU *apdu = m_bw_hold_PDU;
1965             m_bw_hold_PDU = 0;
1966
1967             if (apdu->which == Z_GDU_Z3950)
1968                 handle_incoming_Z_PDU(apdu->u.z3950);
1969             else if (apdu->which == Z_GDU_HTTP_Request)
1970                 handle_incoming_HTTP(apdu->u.HTTP_Request);
1971         }
1972         else
1973         {
1974             inc_request_no();
1975
1976             yaz_log (LOG_LOG, "%sTimeout (client to proxy)", m_session_str);
1977             shutdown();
1978         }
1979     }
1980     else
1981     {
1982         timeout(600);
1983         pre_init();
1984     }
1985 }
1986
1987 void Yaz_ProxyClient::timeoutNotify()
1988 {
1989     if (m_server)
1990         m_server->inc_request_no();
1991
1992     yaz_log (LOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
1993              get_hostname());
1994     m_waiting = 1;
1995     m_root->pre_init();
1996     shutdown();
1997 }
1998
1999 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2000                                  Yaz_Proxy *parent) :
2001     Yaz_Z_Assoc (the_PDU_Observable)
2002 {
2003     m_cookie = 0;
2004     m_next = 0;
2005     m_prev = 0;
2006     m_init_flag = 0;
2007     m_last_query = 0;
2008     m_last_resultSetId = 0;
2009     m_last_resultCount = 0;
2010     m_last_ok = 0;
2011     m_sr_transform = 0;
2012     m_waiting = 0;
2013     m_init_odr = odr_createmem (ODR_DECODE);
2014     m_initResponse = 0;
2015     m_resultSetStartPoint = 0;
2016     m_bytes_sent = m_bytes_recv = 0;
2017     m_pdu_recv = 0;
2018     m_server = 0;
2019     m_seqno = 0;
2020     m_target_idletime = 600;
2021     m_root = parent;
2022 }
2023
2024 const char *Yaz_Proxy::option(const char *name, const char *value)
2025 {
2026     if (!strcmp (name, "optimize")) {
2027         if (value) {
2028             xfree (m_optimize);
2029             m_optimize = xstrdup (value);
2030         }
2031         return m_optimize;
2032     }
2033     return 0;
2034 }
2035
2036 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2037 {
2038
2039 }
2040
2041 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2042 {
2043     if (apdu->which == Z_GDU_Z3950)
2044         recv_Z_PDU(apdu->u.z3950, len);
2045     else if (apdu->which == Z_GDU_HTTP_Response)
2046         recv_HTTP_response(apdu->u.HTTP_Response, len);
2047     else
2048         shutdown();
2049 }
2050
2051 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2052 {
2053     if (!m_invalid_session)
2054         return 0;
2055     m_invalid_session = 0;
2056     handle_incoming_Z_PDU(m_apdu_invalid_session);
2057     assert (m_mem_invalid_session);
2058     nmem_destroy(m_mem_invalid_session);
2059     m_mem_invalid_session = 0;
2060     return 1;
2061 }
2062
2063 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2064 {
2065     m_bytes_recv += len;
2066     m_pdu_recv++;
2067     m_waiting = 0;
2068     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2069         yaz_log (LOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2070                  apdu_name(apdu), get_hostname(), len);
2071     if (apdu->which == Z_APDU_initResponse)
2072     {
2073         if (!m_server)  // if this is a pre init session , check for more
2074             m_root->pre_init();
2075         NMEM nmem = odr_extract_mem (odr_decode());
2076         odr_reset (m_init_odr);
2077         nmem_transfer (m_init_odr->mem, nmem);
2078         m_initResponse = apdu;
2079
2080         Z_InitResponse *ir = apdu->u.initResponse;
2081         char *im0 = ir->implementationName;
2082
2083         char *im1 = (char*)
2084             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2085         *im1 = '\0';
2086         if (im0)
2087         {
2088             strcat(im1, im0);
2089             strcat(im1, " ");
2090         }
2091         strcat(im1, "(YAZ Proxy)");
2092         ir->implementationName = im1;
2093
2094         nmem_destroy (nmem);
2095
2096         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2097             return;
2098     }
2099     if (apdu->which == Z_APDU_searchResponse)
2100     {
2101         Z_SearchResponse *sr = apdu->u.searchResponse;
2102         m_last_resultCount = *sr->resultCount;
2103         int status = *sr->searchStatus;
2104         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2105         {
2106             m_last_ok = 1;
2107
2108             if (sr->records && sr->records->which == Z_Records_DBOSD)
2109             {
2110                 m_cache.add(odr_decode(),
2111                             sr->records->u.databaseOrSurDiagnostics, 1,
2112                             *sr->resultCount);
2113             }
2114         }
2115     }
2116     if (apdu->which == Z_APDU_presentResponse)
2117     {
2118         Z_PresentResponse *pr = apdu->u.presentResponse;
2119         if (m_sr_transform)
2120         {
2121             m_sr_transform = 0;
2122             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2123             Z_SearchResponse *sr = new_apdu->u.searchResponse;
2124             sr->referenceId = pr->referenceId;
2125             *sr->resultCount = m_last_resultCount;
2126             sr->records = pr->records;
2127             sr->nextResultSetPosition = pr->nextResultSetPosition;
2128             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2129             apdu = new_apdu;
2130         }
2131         if (pr->records &&
2132             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2133         {
2134             m_cache.add(odr_decode(),
2135                         pr->records->u.databaseOrSurDiagnostics,
2136                         m_resultSetStartPoint, -1);
2137             m_resultSetStartPoint = 0;
2138         }
2139     }
2140     if (m_cookie)
2141         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2142     if (m_server)
2143     {
2144         m_server->send_to_client(apdu);
2145     }
2146     if (apdu->which == Z_APDU_close)
2147     {
2148         shutdown();
2149     }
2150 }
2151
2152 int Yaz_Proxy::server(const char *addr)
2153 {
2154     int r = Yaz_Z_Assoc::server(addr);
2155     if (!r)
2156     {
2157         yaz_log(LOG_LOG, "%sStarted listener on %s", m_session_str, addr);
2158         timeout(1);
2159     }
2160     return r;
2161 }
2162