src/yaz-proxy.cpp

   1 /*
   2  * Copyright (c) 1998-2003, Index Data.
   3  * See the file LICENSE for details.
   4  *
   5  * $Id: yaz-proxy.cpp,v 1.72 2003-12-20 22:44:30 adam Exp $
   6  */
   7
   8 #include <assert.h>
   9 #include <time.h>
  10
  11 #include <yaz/srw.h>
  12 #include <yaz/marcdisp.h>
  13 #include <yaz/yaz-iconv.h>
  14 #include <yaz/log.h>
  15 #include <yaz/diagbib1.h>
  16 #include <yaz++/proxy.h>
  17 #include <yaz/pquery.h>
  18
  19 static const char *apdu_name(Z_APDU *apdu)
  20 {
  21     switch (apdu->which)
  22     {
  23     case Z_APDU_initRequest:
  24         return "initRequest";
  25     case Z_APDU_initResponse:
  26         return "initResponse";
  27     case Z_APDU_searchRequest:
  28         return "searchRequest";
  29     case Z_APDU_searchResponse:
  30         return "searchResponse";
  31     case Z_APDU_presentRequest:
  32         return "presentRequest";
  33     case Z_APDU_presentResponse:
  34         return "presentResponse";
  35     case Z_APDU_deleteResultSetRequest:
  36         return "deleteResultSetRequest";
  37     case Z_APDU_deleteResultSetResponse:
  38         return "deleteResultSetResponse";
  39     case Z_APDU_scanRequest:
  40         return "scanRequest";
  41     case Z_APDU_scanResponse:
  42         return "scanResponse";
  43     case Z_APDU_sortRequest:
  44         return "sortRequest";
  45     case Z_APDU_sortResponse:
  46         return "sortResponse";
  47     case Z_APDU_extendedServicesRequest:
  48         return "extendedServicesRequest";
  49     case Z_APDU_extendedServicesResponse:
  50         return "extendedServicesResponse";
  51     case Z_APDU_close:
  52         return "close";
  53     }
  54     return "other";
  55 }
  56
  57 static const char *gdu_name(Z_GDU *gdu)
  58 {
  59     switch(gdu->which)
  60     {
  61     case Z_GDU_Z3950:
  62         return apdu_name(gdu->u.z3950);
  63     case Z_GDU_HTTP_Request:
  64         return "HTTP Request";
  65     case Z_GDU_HTTP_Response:
  66         return "HTTP Response";
  67     }
  68     return "Unknown request/response";
  69 }
  70
  71 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
  72                      Yaz_Proxy *parent) :
  73     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
  74 {
  75     m_PDU_Observable = the_PDU_Observable;
  76     m_client = 0;
  77     m_parent = parent;
  78     m_clientPool = 0;
  79     m_seqno = 1;
  80     m_keepalive_limit_bw = 500000;
  81     m_keepalive_limit_pdu = 1000;
  82     m_proxyTarget = 0;
  83     m_default_target = 0;
  84     m_proxy_authentication = 0;
  85     m_max_clients = 150;
  86     m_log_mask = 0;
  87     m_seed = time(0);
  88     m_client_idletime = 600;
  89     m_target_idletime = 600;
  90     m_optimize = xstrdup ("1");
  91     strcpy(m_session_str, "0 ");
  92     m_session_no=0;
  93     m_bytes_sent = m_bytes_recv = 0;
  94     m_bw_hold_PDU = 0;
  95     m_bw_max = 0;
  96     m_pdu_max = 0;
  97     m_max_record_retrieve = 0;
  98     m_reconfig_flag = 0;
  99     m_config_fname = 0;
 100     m_request_no = 0;
 101     m_invalid_session = 0;
 102     m_config = 0;
 103     m_marcxml_flag = 0;
 104     m_initRequest_apdu = 0;
 105     m_initRequest_mem = 0;
 106     m_apdu_invalid_session = 0;
 107     m_mem_invalid_session = 0;
 108     m_s2z_odr_init = 0;
 109     m_s2z_odr_search = 0;
 110     m_s2z_init_apdu = 0;
 111     m_s2z_search_apdu = 0;
 112     m_s2z_present_apdu = 0;
 113     m_http_keepalive = 0;
 114     m_http_version = 0;
 115     m_soap_ns = 0;
 116     m_s2z_packing = Z_SRW_recordPacking_string;
 117     m_zeerex_fname = 0;
 118 }
 119
 120 Yaz_Proxy::~Yaz_Proxy()
 121 {
 122     yaz_log(LOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 123             m_bytes_sent, m_bytes_recv);
 124     nmem_destroy(m_initRequest_mem);
 125     nmem_destroy(m_mem_invalid_session);
 126     xfree (m_proxyTarget);
 127     xfree (m_default_target);
 128     xfree (m_proxy_authentication);
 129     xfree (m_optimize);
 130     if (m_s2z_odr_init)
 131         odr_destroy(m_s2z_odr_init);
 132     if (m_s2z_odr_search)
 133         odr_destroy(m_s2z_odr_search);
 134     delete m_config;
 135 }
 136
 137 int Yaz_Proxy::set_config(const char *config)
 138 {
 139     delete m_config;
 140     m_config = new Yaz_ProxyConfig();
 141     xfree(m_config_fname);
 142     m_config_fname = xstrdup(config);
 143     int r = m_config->read_xml(config);
 144     if (!r)
 145         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 146     return r;
 147 }
 148
 149 void Yaz_Proxy::set_default_target(const char *target)
 150 {
 151     xfree (m_default_target);
 152     m_default_target = 0;
 153     if (target)
 154         m_default_target = (char *) xstrdup (target);
 155 }
 156
 157 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 158 {
 159     xfree (m_proxy_authentication);
 160     m_proxy_authentication = 0;
 161     if (auth)
 162         m_proxy_authentication = (char *) xstrdup (auth);
 163 }
 164
 165 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 166 {
 167     if (m_parent)
 168         return m_parent->check_reconfigure();
 169
 170     Yaz_ProxyConfig *cfg = m_config;
 171     if (m_reconfig_flag)
 172     {
 173         yaz_log(LOG_LOG, "reconfigure");
 174         yaz_log_reopen();
 175         if (m_config_fname && cfg)
 176         {
 177             yaz_log(LOG_LOG, "reconfigure config %s", m_config_fname);
 178             int r = cfg->read_xml(m_config_fname);
 179             if (r)
 180                 yaz_log(LOG_WARN, "reconfigure failed");
 181             else
 182             {
 183                 m_log_mask = 0;
 184                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 185             }
 186         }
 187         else
 188             yaz_log(LOG_LOG, "reconfigure");
 189         m_reconfig_flag = 0;
 190     }
 191     return cfg;
 192 }
 193
 194 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 195                                             *the_PDU_Observable, int fd)
 196 {
 197     check_reconfigure();
 198     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 199     new_proxy->m_config = 0;
 200     new_proxy->m_config_fname = 0;
 201     new_proxy->timeout(m_client_idletime);
 202     new_proxy->m_target_idletime = m_target_idletime;
 203     new_proxy->set_default_target(m_default_target);
 204     new_proxy->m_max_clients = m_max_clients;
 205     new_proxy->m_log_mask = m_log_mask;
 206     new_proxy->set_APDU_log(get_APDU_log());
 207     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 208         new_proxy->set_APDU_yazlog(1);
 209     else
 210         new_proxy->set_APDU_yazlog(0);
 211     new_proxy->set_proxy_authentication(m_proxy_authentication);
 212     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 213     m_session_no++;
 214     yaz_log (LOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 215              the_PDU_Observable->getpeername());
 216     return new_proxy;
 217 }
 218
 219 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 220 {
 221     int oid[OID_SIZE];
 222     Z_OtherInformationUnit *oi;
 223     struct oident ent;
 224     ent.proto = PROTO_Z3950;
 225     ent.oclass = CLASS_USERINFO;
 226     ent.value = (oid_value) VAL_COOKIE;
 227     assert (oid_ent_to_oid (&ent, oid));
 228
 229     if (oid_ent_to_oid (&ent, oid) &&
 230         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 231         oi->which == Z_OtherInfo_characterInfo)
 232         return oi->information.characterInfo;
 233     return 0;
 234 }
 235
 236 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 237 {
 238     int oid[OID_SIZE];
 239     Z_OtherInformationUnit *oi;
 240     struct oident ent;
 241     ent.proto = PROTO_Z3950;
 242     ent.oclass = CLASS_USERINFO;
 243     ent.value = (oid_value) VAL_PROXY;
 244     if (oid_ent_to_oid (&ent, oid) &&
 245         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 246         oi->which == Z_OtherInfo_characterInfo)
 247         return oi->information.characterInfo;
 248     return 0;
 249 }
 250
 251 const char *Yaz_Proxy::load_balance(const char **url)
 252 {
 253     int zurl_in_use[MAX_ZURL_PLEX];
 254     Yaz_ProxyClient *c;
 255     int i;
 256
 257     for (i = 0; i<MAX_ZURL_PLEX; i++)
 258         zurl_in_use[i] = 0;
 259     for (c = m_parent->m_clientPool; c; c = c->m_next)
 260     {
 261         for (i = 0; url[i]; i++)
 262             if (!strcmp(url[i], c->get_hostname()))
 263                 zurl_in_use[i]++;
 264     }
 265     int min = 100000;
 266     const char *ret = 0;
 267     for (i = 0; url[i]; i++)
 268     {
 269         yaz_log(LOG_DEBUG, "%szurl=%s use=%d",
 270                 m_session_str, url[i], zurl_in_use[i]);
 271         if (min > zurl_in_use[i])
 272         {
 273             ret = url[i];
 274             min = zurl_in_use[i];
 275         }
 276     }
 277     return ret;
 278 }
 279
 280 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 281                                        const char *proxy_host)
 282 {
 283     assert (m_parent);
 284     Yaz_Proxy *parent = m_parent;
 285     Z_OtherInformation **oi;
 286     Yaz_ProxyClient *c = m_client;
 287
 288     if (!m_proxyTarget)
 289     {
 290         const char *url[MAX_ZURL_PLEX];
 291         Yaz_ProxyConfig *cfg = check_reconfigure();
 292         if (proxy_host)
 293         {
 294 #if 1
 295 /* only to be enabled for debugging... */
 296             if (!strcmp(proxy_host, "stop"))
 297                 exit(0);
 298 #endif
 299             xfree(m_default_target);
 300             m_default_target = xstrdup(proxy_host);
 301             proxy_host = m_default_target;
 302         }
 303         int client_idletime = -1;
 304         const char *cql2rpn_fname = 0;
 305         url[0] = m_default_target;
 306         url[1] = 0;
 307         if (cfg)
 308         {
 309             int pre_init = 0;
 310             cfg->get_target_info(proxy_host, url, &m_bw_max,
 311                                  &m_pdu_max, &m_max_record_retrieve,
 312                                  &m_target_idletime, &client_idletime,
 313                                  &parent->m_max_clients,
 314                                  &m_keepalive_limit_bw,
 315                                  &m_keepalive_limit_pdu,
 316                                  &pre_init,
 317                                  &cql2rpn_fname,
 318                                  &m_zeerex_fname);
 319         }
 320         if (client_idletime != -1)
 321         {
 322             m_client_idletime = client_idletime;
 323             timeout(m_client_idletime);
 324         }
 325         if (cql2rpn_fname)
 326             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 327         if (!url[0])
 328         {
 329             yaz_log(LOG_LOG, "%sNo default target", m_session_str);
 330             return 0;
 331         }
 332         // we don't handle multiplexing for cookie session, so we just
 333         // pick the first one in this case (anonymous users will be able
 334         // to use any backend)
 335         if (cookie && *cookie)
 336             m_proxyTarget = (char*) xstrdup(url[0]);
 337         else
 338             m_proxyTarget = (char*) xstrdup(load_balance(url));
 339     }
 340     if (cookie && *cookie)
 341     {   // search in sessions with a cookie
 342         for (c = parent->m_clientPool; c; c = c->m_next)
 343         {
 344             assert (c->m_prev);
 345             assert (*c->m_prev == c);
 346             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 347                 !strcmp(m_proxyTarget, c->get_hostname()))
 348             {
 349                 // Found it in cache
 350                 // The following handles "cancel"
 351                 // If connection is busy (waiting for PDU) and
 352                 // we have an initRequest we can safely do re-open
 353                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 354                 {
 355                     yaz_log (LOG_LOG, "%s REOPEN target=%s", m_session_str,
 356                              c->get_hostname());
 357                     c->close();
 358                     c->m_init_flag = 0;
 359
 360                     c->m_last_ok = 0;
 361                     c->m_cache.clear();
 362                     c->m_last_resultCount = 0;
 363                     c->m_sr_transform = 0;
 364                     c->m_waiting = 0;
 365                     c->m_resultSetStartPoint = 0;
 366                     c->m_target_idletime = m_target_idletime;
 367                     if (c->client(m_proxyTarget))
 368                     {
 369                         delete c;
 370                         return 0;
 371                     }
 372                     c->timeout(30);
 373                 }
 374                 c->m_seqno = parent->m_seqno;
 375                 if (c->m_server && c->m_server != this)
 376                     c->m_server->m_client = 0;
 377                 c->m_server = this;
 378                 (parent->m_seqno)++;
 379                 yaz_log (LOG_DEBUG, "get_client 1 %p %p", this, c);
 380                 return c;
 381             }
 382         }
 383     }
 384     else if (!c)
 385     {
 386         // don't have a client session yet. Search in session w/o cookie
 387         for (c = parent->m_clientPool; c; c = c->m_next)
 388         {
 389             assert (c->m_prev);
 390             assert (*c->m_prev == c);
 391             if (c->m_server == 0 && c->m_cookie == 0 &&
 392                 c->m_waiting == 0 &&
 393                 !strcmp(m_proxyTarget, c->get_hostname()))
 394             {
 395                 // found it in cache
 396                 yaz_log (LOG_LOG, "%sREUSE %s",
 397                          m_session_str, c->get_hostname());
 398
 399                 c->m_seqno = parent->m_seqno;
 400                 assert(c->m_server == 0);
 401                 c->m_server = this;
 402
 403                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 404                     c->set_APDU_yazlog(1);
 405                 else
 406                     c->set_APDU_yazlog(0);
 407
 408                 (parent->m_seqno)++;
 409
 410                 parent->pre_init();
 411
 412                 return c;
 413             }
 414         }
 415     }
 416     if (!m_client)
 417     {
 418         if (apdu->which != Z_APDU_initRequest)
 419         {
 420             yaz_log (LOG_LOG, "no first INIT!");
 421             return 0;
 422         }
 423         Z_InitRequest *initRequest = apdu->u.initRequest;
 424
 425         if (!initRequest->idAuthentication)
 426         {
 427             if (m_proxy_authentication)
 428             {
 429                 initRequest->idAuthentication =
 430                     (Z_IdAuthentication *)
 431                     odr_malloc (odr_encode(),
 432                                 sizeof(*initRequest->idAuthentication));
 433                 initRequest->idAuthentication->which =
 434                     Z_IdAuthentication_open;
 435                 initRequest->idAuthentication->u.open =
 436                     odr_strdup (odr_encode(), m_proxy_authentication);
 437             }
 438         }
 439
 440         // go through list of clients - and find the lowest/oldest one.
 441         Yaz_ProxyClient *c_min = 0;
 442         int min_seq = -1;
 443         int no_of_clients = 0;
 444         if (parent->m_clientPool)
 445             yaz_log (LOG_DEBUG, "Existing sessions");
 446         for (c = parent->m_clientPool; c; c = c->m_next)
 447         {
 448             yaz_log (LOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 449                                c->m_waiting, c->get_hostname(),
 450                                c->m_cookie ? c->m_cookie : "");
 451             no_of_clients++;
 452             if (min_seq < 0 || c->m_seqno < min_seq)
 453             {
 454                 min_seq = c->m_seqno;
 455                 c_min = c;
 456             }
 457         }
 458         if (no_of_clients >= parent->m_max_clients)
 459         {
 460             c = c_min;
 461             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 462             {
 463                 yaz_log (LOG_LOG, "%sMAXCLIENTS Destroy %d",
 464                          m_session_str, c->m_seqno);
 465                 if (c->m_server && c->m_server != this)
 466                     delete c->m_server;
 467                 c->m_server = 0;
 468             }
 469             else
 470             {
 471                 yaz_log (LOG_LOG, "%sMAXCLIENTS Reuse %d %d %s",
 472                          m_session_str,
 473                          c->m_seqno, parent->m_seqno, c->get_hostname());
 474                 xfree (c->m_cookie);
 475                 c->m_cookie = 0;
 476                 if (cookie)
 477                     c->m_cookie = xstrdup(cookie);
 478                 c->m_seqno = parent->m_seqno;
 479                 if (c->m_server && c->m_server != this)
 480                 {
 481                     c->m_server->m_client = 0;
 482                     delete c->m_server;
 483                 }
 484                 (parent->m_seqno)++;
 485                 c->m_target_idletime = m_target_idletime;
 486                 c->timeout(m_target_idletime);
 487
 488                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 489                     c->set_APDU_yazlog(1);
 490                 else
 491                     c->set_APDU_yazlog(0);
 492
 493                 return c;
 494             }
 495         }
 496         else
 497         {
 498             yaz_log (LOG_LOG, "%sNEW %d %s",
 499                      m_session_str, parent->m_seqno, m_proxyTarget);
 500             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 501             c->m_next = parent->m_clientPool;
 502             if (c->m_next)
 503                 c->m_next->m_prev = &c->m_next;
 504             parent->m_clientPool = c;
 505             c->m_prev = &parent->m_clientPool;
 506         }
 507
 508         xfree (c->m_cookie);
 509         c->m_cookie = 0;
 510         if (cookie)
 511             c->m_cookie = xstrdup(cookie);
 512
 513         c->m_seqno = parent->m_seqno;
 514         c->m_init_flag = 0;
 515         c->m_last_resultCount = 0;
 516         c->m_last_ok = 0;
 517         c->m_cache.clear();
 518         c->m_sr_transform = 0;
 519         c->m_waiting = 0;
 520         c->m_resultSetStartPoint = 0;
 521         (parent->m_seqno)++;
 522         if (c->client(m_proxyTarget))
 523         {
 524             delete c;
 525             return 0;
 526         }
 527         c->m_target_idletime = m_target_idletime;
 528         c->timeout(30);
 529
 530         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 531             c->set_APDU_yazlog(1);
 532         else
 533             c->set_APDU_yazlog(0);
 534     }
 535     yaz_log (LOG_DEBUG, "get_client 3 %p %p", this, c);
 536     return c;
 537 }
 538
 539 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 540 {
 541     int i;
 542     for (i = 0; i<num; i++)
 543     {
 544         oident *ent;
 545         Z_DefaultDiagFormat *r;
 546         Z_DiagRec *p = pp[i];
 547         if (p->which != Z_DiagRec_defaultFormat)
 548         {
 549             yaz_log(LOG_LOG, "%sError no diagnostics", m_session_str);
 550             return;
 551         }
 552         else
 553             r = p->u.defaultFormat;
 554         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 555             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 556             yaz_log(LOG_LOG, "%sError unknown diagnostic set", m_session_str);
 557         switch (r->which)
 558         {
 559         case Z_DefaultDiagFormat_v2Addinfo:
 560             yaz_log(LOG_LOG, "%sError %d %s:%s",
 561                     m_session_str,
 562                     *r->condition, diagbib1_str(*r->condition),
 563                     r->u.v2Addinfo);
 564             break;
 565         case Z_DefaultDiagFormat_v3Addinfo:
 566             yaz_log(LOG_LOG, "%sError %d %s:%s",
 567                     m_session_str,
 568                     *r->condition, diagbib1_str(*r->condition),
 569                     r->u.v3Addinfo);
 570             break;
 571         }
 572     }
 573 }
 574
 575 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p)
 576 {
 577     int i;
 578
 579     yaz_marc_t mt = yaz_marc_create();
 580     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 581     for (i = 0; i < p->num_records; i++)
 582     {
 583         Z_NamePlusRecord *npr = p->records[i];
 584         if (npr->which == Z_NamePlusRecord_databaseRecord)
 585         {
 586             Z_External *r = npr->u.databaseRecord;
 587             if (r->which == Z_External_octet)
 588             {
 589                 int rlen;
 590                 char *result;
 591                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 592                                         r->u.octet_aligned->len,
 593                                         &result, &rlen))
 594                 {
 595                     yaz_iconv_t cd = yaz_iconv_open("UTF-8", "MARC-8");
 596                     WRBUF wrbuf = wrbuf_alloc();
 597
 598                     char outbuf[120];
 599                     size_t inbytesleft = rlen;
 600                     const char *inp = result;
 601                     while (cd && inbytesleft)
 602                     {
 603                         size_t outbytesleft = sizeof(outbuf);
 604                         char *outp = outbuf;
 605                         size_t r;
 606
 607                         r = yaz_iconv (cd, (char**) &inp,
 608                                        &inbytesleft,
 609                                        &outp, &outbytesleft);
 610                         if (r == (size_t) (-1))
 611                         {
 612                             int e = yaz_iconv_error(cd);
 613                             if (e != YAZ_ICONV_E2BIG)
 614                             {
 615                                 yaz_log(LOG_WARN, "conversion failure");
 616                                 break;
 617                             }
 618                         }
 619                         wrbuf_write(wrbuf, outbuf, outp - outbuf);
 620                     }
 621                     if (cd)
 622                         yaz_iconv_close(cd);
 623
 624                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 625                                                          VAL_TEXT_XML,
 626                                                          wrbuf_buf(wrbuf),
 627                                                          wrbuf_len(wrbuf));
 628                     wrbuf_free(wrbuf, 1);
 629                 }
 630             }
 631         }
 632     }
 633     yaz_marc_destroy(mt);
 634 }
 635
 636 int Yaz_Proxy::send_http_response(int code)
 637 {
 638     ODR o = odr_encode();
 639     const char *ctype = "text/xml";
 640     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 641     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 642     if (m_http_version)
 643         hres->version = odr_strdup(o, m_http_version);
 644     m_http_keepalive = 0;
 645     int len;
 646     return send_GDU(gdu, &len);
 647 }
 648
 649 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 650 {
 651     ODR o = odr_encode();
 652     const char *ctype = "text/xml";
 653     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 654     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 655     if (m_http_version)
 656         hres->version = odr_strdup(o, m_http_version);
 657     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 658     if (m_http_keepalive)
 659         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 660
 661     static Z_SOAP_Handler soap_handlers[2] = {
 662 #if HAVE_XML2
 663         {"http://www.loc.gov/zing/srw/", 0,
 664          (Z_SOAP_fun) yaz_srw_codec},
 665 #endif
 666         {0, 0, 0}
 667     };
 668
 669     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 670     soap_package->which = Z_SOAP_generic;
 671     soap_package->u.generic =
 672         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 673     soap_package->u.generic->no = 0;
 674     soap_package->u.generic->ns = soap_handlers[0].ns;
 675     soap_package->u.generic->p = (void *) srw_pdu;
 676     soap_package->ns = m_soap_ns;
 677     int ret = z_soap_codec_enc(o, &soap_package,
 678                                &hres->content_buf, &hres->content_len,
 679                                soap_handlers, 0);
 680     int len;
 681     return send_GDU(gdu, &len);
 682 }
 683
 684 int Yaz_Proxy::send_to_srw_client_error(int srw_error)
 685 {
 686     ODR o = odr_encode();
 687     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 688     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 689
 690     srw_res->num_diagnostics = 1;
 691     srw_res->diagnostics = (Z_SRW_diagnostic *)
 692         odr_malloc(o, sizeof(*srw_res->diagnostics));
 693     srw_res->diagnostics[0].code =  odr_intdup(o, srw_error);
 694     srw_res->diagnostics[0].details = 0;
 695     return send_srw_response(srw_pdu);
 696 }
 697
 698 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 699                              Z_DefaultDiagFormat *ddf)
 700 {
 701     int bib1_code = *ddf->condition;
 702     if (bib1_code == 109)
 703         return 404;
 704     srw_res->num_diagnostics = 1;
 705     srw_res->diagnostics = (Z_SRW_diagnostic *)
 706         odr_malloc(o, sizeof(*srw_res->diagnostics));
 707     srw_res->diagnostics[0].code =
 708         odr_intdup(o, yaz_diag_bib1_to_srw(*ddf->condition));
 709     srw_res->diagnostics[0].details = ddf->u.v2Addinfo;
 710     return 0;
 711 }
 712
 713 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
 714 {
 715     ODR o = odr_encode();
 716     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 717     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 718
 719     srw_res->numberOfRecords = odr_intdup (o, hits);
 720     if (records && records->which == Z_Records_DBOSD)
 721     {
 722         srw_res->num_records =
 723             records->u.databaseOrSurDiagnostics->num_records;
 724         int i;
 725         srw_res->records = (Z_SRW_record *)
 726             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
 727         for (i = 0; i < srw_res->num_records; i++)
 728         {
 729             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
 730             if (npr->which != Z_NamePlusRecord_databaseRecord)
 731             {
 732                 srw_res->records[i].recordSchema = "diagnostic";
 733                 srw_res->records[i].recordPacking = m_s2z_packing;
 734                 srw_res->records[i].recordData_buf = "67";
 735                 srw_res->records[i].recordData_len = 2;
 736                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 737                 continue;
 738             }
 739             Z_External *r = npr->u.databaseRecord;
 740             oident *ent = oid_getentbyoid(r->direct_reference);
 741             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
 742             {
 743                 srw_res->records[i].recordSchema = "http://www.loc.gov/marcxml/";
 744                 srw_res->records[i].recordPacking = m_s2z_packing;
 745                 srw_res->records[i].recordData_buf = (char*)
 746                     r->u.octet_aligned->buf;
 747                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
 748                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 749             }
 750             else
 751             {
 752                 srw_res->records[i].recordSchema = "diagnostic";
 753                 srw_res->records[i].recordPacking = m_s2z_packing;
 754                 srw_res->records[i].recordData_buf = "67";
 755                 srw_res->records[i].recordData_len = 2;
 756                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 757             }
 758         }
 759     }
 760     if (records && records->which == Z_Records_NSD)
 761     {
 762         int http_code;
 763         http_code = z_to_srw_diag(odr_encode(), srw_res,
 764                                    records->u.nonSurrogateDiagnostic);
 765         if (http_code)
 766             return send_http_response(http_code);
 767     }
 768     return send_srw_response(srw_pdu);
 769
 770 }
 771
 772 int Yaz_Proxy::send_srw_explain()
 773 {
 774     Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
 775     Z_SRW_explainResponse *er = res->u.explain_response;
 776
 777     if (m_zeerex_fname)
 778     {
 779         FILE *inf = fopen(m_zeerex_fname, "rb");
 780         if (inf)
 781         {
 782             fseek(inf, 0L, SEEK_END);
 783             long sz = ftell(inf);
 784             fseek(inf, 0L, SEEK_SET);
 785             if (sz > 0)
 786             {
 787                 er->record.recordData_buf =
 788                     (char*) odr_malloc(odr_encode(), sz);
 789                 size_t s = fread(er->record.recordData_buf, 1,sz, inf);
 790                 if (s > 0)
 791                     er->record.recordData_len = s;
 792             }
 793             else
 794                 yaz_log(LOG_WARN|LOG_ERRNO, "zeerex file: ftell");
 795             fclose(inf);
 796         }
 797         else
 798             yaz_log(LOG_WARN|LOG_ERRNO, "zeerex file: fopen");
 799     }
 800     else
 801         yaz_log(LOG_LOG, "zeerex file: not defined");
 802     return send_srw_response(res);
 803 }
 804
 805 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu, int *len)
 806 {
 807     if (m_http_version)
 808     {
 809         if (apdu->which == Z_APDU_initResponse)
 810         {
 811             Z_InitResponse *res = apdu->u.initResponse;
 812             if (*res->result == 0)
 813             {
 814                 send_to_srw_client_error(3);
 815             }
 816             else if (!m_s2z_search_apdu)
 817             {
 818                 send_srw_explain();
 819             }
 820             else
 821             {
 822                 handle_incoming_Z_PDU(m_s2z_search_apdu);
 823             }
 824         }
 825         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
 826         {
 827             m_s2z_search_apdu = 0;
 828             Z_SearchResponse *res = apdu->u.searchResponse;
 829             m_s2z_hit_count = *res->resultCount;
 830             if (res->records && res->records->which == Z_Records_NSD)
 831             {
 832                 send_to_srw_client_ok(0, res->records, 1);
 833             }
 834             else if (m_s2z_present_apdu)
 835             {
 836                 handle_incoming_Z_PDU(m_s2z_present_apdu);
 837             }
 838             else
 839             {
 840                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
 841             }
 842         }
 843         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
 844         {
 845             int start =
 846                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
 847
 848             m_s2z_present_apdu = 0;
 849             Z_PresentResponse *res = apdu->u.presentResponse;
 850             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
 851         }
 852     }
 853     else
 854         return send_Z_PDU(apdu, len);
 855     return 0;
 856 }
 857
 858 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 859 {
 860     int len = 0;
 861     int kill_session = 0;
 862     if (apdu->which == Z_APDU_searchResponse)
 863     {
 864         Z_SearchResponse *sr = apdu->u.searchResponse;
 865         Z_Records *p = sr->records;
 866         if (p && p->which == Z_Records_NSD)
 867         {
 868             Z_DiagRec dr, *dr_p = &dr;
 869             dr.which = Z_DiagRec_defaultFormat;
 870             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 871
 872             *sr->searchStatus = 0;
 873             display_diagrecs(&dr_p, 1);
 874         }
 875         else
 876         {
 877             if (m_marcxml_flag && p && p->which == Z_Records_DBOSD)
 878                 convert_to_marcxml(p->u.databaseOrSurDiagnostics);
 879             if (sr->resultCount)
 880             {
 881                 yaz_log(LOG_LOG, "%s%d hits", m_session_str,
 882                         *sr->resultCount);
 883                 if (*sr->resultCount < 0)
 884                 {
 885                     m_invalid_session = 1;
 886                     kill_session = 1;
 887
 888                     *sr->searchStatus = 0;
 889                     sr->records =
 890                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
 891                     *sr->resultCount = 0;
 892                 }
 893             }
 894         }
 895     }
 896     else if (apdu->which == Z_APDU_presentResponse)
 897     {
 898         Z_PresentResponse *sr = apdu->u.presentResponse;
 899         Z_Records *p = sr->records;
 900         if (p && p->which == Z_Records_NSD)
 901         {
 902             Z_DiagRec dr, *dr_p = &dr;
 903             dr.which = Z_DiagRec_defaultFormat;
 904             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 905             if (*sr->presentStatus == Z_PresentStatus_success)
 906                 *sr->presentStatus = Z_PresentStatus_failure;
 907             display_diagrecs(&dr_p, 1);
 908         }
 909         if (m_marcxml_flag && p && p->which == Z_Records_DBOSD)
 910             convert_to_marcxml(p->u.databaseOrSurDiagnostics);
 911     }
 912     int r = send_PDU_convert(apdu, &len);
 913     if (r)
 914         return r;
 915     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 916         yaz_log (LOG_DEBUG, "%sSending %s to client %d bytes", m_session_str,
 917                  apdu_name(apdu), len);
 918     m_bytes_sent += len;
 919     m_bw_stat.add_bytes(len);
 920     if (kill_session)
 921     {
 922         delete m_client;
 923         m_client = 0;
 924         m_parent->pre_init();
 925     }
 926     if (m_http_version)
 927     {
 928         if (!m_http_keepalive)
 929         {
 930 #if 1
 931             timeout(1);
 932 #else
 933             shutdown();
 934             return -1;
 935 #endif
 936         }
 937     }
 938     return r;
 939 }
 940
 941 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
 942 {
 943     int len = 0;
 944     int r = send_Z_PDU(apdu, &len);
 945     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
 946         yaz_log (LOG_LOG, "%sSending %s to %s %d bytes",
 947                  get_session_str(),
 948                  apdu_name(apdu), get_hostname(), len);
 949     m_bytes_sent += len;
 950     return r;
 951 }
 952
 953 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
 954 {
 955     if (apdu->which == Z_APDU_presentRequest)
 956     {
 957         Z_PresentRequest *pr = apdu->u.presentRequest;
 958         int toget = *pr->numberOfRecordsRequested;
 959         int start = *pr->resultSetStartPoint;
 960
 961         yaz_log(LOG_LOG, "%sPresent %s %d+%d", m_session_str,
 962                 pr->resultSetId, start, toget);
 963
 964         if (*m_parent->m_optimize == '0')
 965             return apdu;
 966
 967         if (!m_client->m_last_resultSetId)
 968         {
 969             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
 970             new_apdu->u.presentResponse->records =
 971                 create_nonSurrogateDiagnostics(odr_encode(), 30,
 972                                                pr->resultSetId);
 973             send_to_client(new_apdu);
 974             return 0;
 975         }
 976         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
 977         {
 978             if (start+toget-1 > m_client->m_last_resultCount)
 979             {
 980                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
 981                 new_apdu->u.presentResponse->records =
 982                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
 983                 send_to_client(new_apdu);
 984                 return 0;
 985             }
 986             Z_NamePlusRecordList *npr;
 987             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
 988                                           pr->preferredRecordSyntax,
 989                                           pr->recordComposition))
 990             {
 991                 yaz_log (LOG_LOG, "%sReturned cached records for present request",
 992                          m_session_str);
 993                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
 994                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
 995
 996                 new_apdu->u.presentResponse->numberOfRecordsReturned
 997                     = odr_intdup(odr_encode(), toget);
 998
 999                 new_apdu->u.presentResponse->records = (Z_Records*)
1000                     odr_malloc(odr_encode(), sizeof(Z_Records));
1001                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1002                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1003                 new_apdu->u.presentResponse->nextResultSetPosition =
1004                     odr_intdup(odr_encode(), start+toget);
1005
1006                 send_to_client(new_apdu);
1007                 return 0;
1008             }
1009         }
1010     }
1011
1012     if (apdu->which != Z_APDU_searchRequest)
1013         return apdu;
1014     Z_SearchRequest *sr = apdu->u.searchRequest;
1015     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1016     Yaz_Z_Databases this_databases;
1017
1018     this_databases.set(sr->num_databaseNames, (const char **)
1019                        sr->databaseNames);
1020
1021     this_query->set_Z_Query(sr->query);
1022
1023     char query_str[120];
1024     this_query->print(query_str, sizeof(query_str)-1);
1025     yaz_log(LOG_LOG, "%sSearch %s", m_session_str, query_str);
1026
1027     if (*m_parent->m_optimize != '0' &&
1028         m_client->m_last_ok && m_client->m_last_query &&
1029         m_client->m_last_query->match(this_query) &&
1030         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1031         m_client->m_last_databases.match(this_databases))
1032     {
1033         delete this_query;
1034         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1035             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1036         {
1037             Z_NamePlusRecordList *npr;
1038             int toget = *sr->mediumSetPresentNumber;
1039             Z_RecordComposition *comp = 0;
1040
1041             if (toget > m_client->m_last_resultCount)
1042                 toget = m_client->m_last_resultCount;
1043
1044             if (sr->mediumSetElementSetNames)
1045             {
1046                 comp = (Z_RecordComposition *)
1047                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1048                 comp->which = Z_RecordComp_simple;
1049                 comp->u.simple = sr->mediumSetElementSetNames;
1050             }
1051
1052             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1053                                           sr->preferredRecordSyntax, comp))
1054             {
1055                 yaz_log (LOG_LOG, "%sReturned cached records for medium set",
1056                          m_session_str);
1057                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1058                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1059                 new_apdu->u.searchResponse->resultCount =
1060                     &m_client->m_last_resultCount;
1061
1062                 new_apdu->u.searchResponse->numberOfRecordsReturned
1063                     = odr_intdup(odr_encode(), toget);
1064
1065                 new_apdu->u.searchResponse->presentStatus =
1066                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1067                 new_apdu->u.searchResponse->records = (Z_Records*)
1068                     odr_malloc(odr_encode(), sizeof(Z_Records));
1069                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1070                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1071                 new_apdu->u.searchResponse->nextResultSetPosition =
1072                     odr_intdup(odr_encode(), toget+1);
1073                 send_to_client(new_apdu);
1074                 return 0;
1075             }
1076             else
1077             {
1078                 // medium Set
1079                 // send present request (medium size)
1080                 yaz_log (LOG_LOG, "%sOptimizing search for medium set",
1081                          m_session_str);
1082
1083                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1084                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1085                 pr->referenceId = sr->referenceId;
1086                 pr->resultSetId = sr->resultSetName;
1087                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1088                 *pr->numberOfRecordsRequested = toget;
1089                 pr->recordComposition = comp;
1090                 m_client->m_sr_transform = 1;
1091                 return new_apdu;
1092             }
1093         }
1094         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1095             m_client->m_last_resultCount <= 0)
1096         {
1097             // large set. Return pseudo-search response immediately
1098             yaz_log (LOG_LOG, "%sOptimizing search for large set",
1099                      m_session_str);
1100             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1101             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1102             new_apdu->u.searchResponse->resultCount =
1103                 &m_client->m_last_resultCount;
1104             send_to_client(new_apdu);
1105             return 0;
1106         }
1107         else
1108         {
1109             Z_NamePlusRecordList *npr;
1110             int toget = m_client->m_last_resultCount;
1111             Z_RecordComposition *comp = 0;
1112             // small set
1113             // send a present request (small set)
1114
1115             if (sr->smallSetElementSetNames)
1116             {
1117                 comp = (Z_RecordComposition *)
1118                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1119                 comp->which = Z_RecordComp_simple;
1120                 comp->u.simple = sr->smallSetElementSetNames;
1121             }
1122
1123             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1124                                           sr->preferredRecordSyntax, comp))
1125             {
1126                 yaz_log (LOG_LOG, "%sReturned cached records for small set",
1127                          m_session_str);
1128                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1129                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1130                 new_apdu->u.searchResponse->resultCount =
1131                     &m_client->m_last_resultCount;
1132
1133                 new_apdu->u.searchResponse->numberOfRecordsReturned
1134                     = odr_intdup(odr_encode(), toget);
1135
1136                 new_apdu->u.searchResponse->presentStatus =
1137                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1138                 new_apdu->u.searchResponse->records = (Z_Records*)
1139                     odr_malloc(odr_encode(), sizeof(Z_Records));
1140                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1141                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1142                 new_apdu->u.searchResponse->nextResultSetPosition =
1143                     odr_intdup(odr_encode(), toget+1);
1144                 send_to_client(new_apdu);
1145                 return 0;
1146             }
1147             else
1148             {
1149                 yaz_log (LOG_LOG, "%sOptimizing search for small set",
1150                          m_session_str);
1151                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1152                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1153                 pr->referenceId = sr->referenceId;
1154                 pr->resultSetId = sr->resultSetName;
1155                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1156                 *pr->numberOfRecordsRequested = toget;
1157                 pr->recordComposition = comp;
1158                 m_client->m_sr_transform = 1;
1159                 return new_apdu;
1160             }
1161         }
1162     }
1163     else  // query doesn't match
1164     {
1165         delete m_client->m_last_query;
1166         m_client->m_last_query = this_query;
1167         m_client->m_last_ok = 0;
1168         m_client->m_cache.clear();
1169         m_client->m_resultSetStartPoint = 0;
1170
1171         xfree (m_client->m_last_resultSetId);
1172         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1173
1174         m_client->m_last_databases.set(sr->num_databaseNames,
1175                                        (const char **) sr->databaseNames);
1176     }
1177     return apdu;
1178 }
1179
1180
1181 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1182 {
1183     char *cp = strchr(m_session_str, ' ');
1184     m_request_no++;
1185     if (cp)
1186         sprintf(cp+1, "%d ", m_request_no);
1187
1188     m_bytes_recv += len;
1189
1190     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
1191         yaz_log (LOG_DEBUG, "%sReceiving %s from client %d bytes",
1192                  m_session_str, gdu_name(apdu), len);
1193
1194     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1195         shutdown();
1196
1197     m_bw_stat.add_bytes(len);
1198     m_pdu_stat.add_bytes(1);
1199
1200     int bw_total = m_bw_stat.get_total();
1201     int pdu_total = m_pdu_stat.get_total();
1202
1203     int reduce = 0;
1204     if (m_bw_max)
1205     {
1206         if (bw_total > m_bw_max)
1207         {
1208             reduce = (bw_total/m_bw_max);
1209         }
1210     }
1211     if (m_pdu_max)
1212     {
1213         if (pdu_total > m_pdu_max)
1214         {
1215             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1216             reduce = (reduce > nreduce) ? reduce : nreduce;
1217         }
1218     }
1219     if (reduce)
1220     {
1221         yaz_log(LOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1222                 m_session_str, reduce, bw_total, pdu_total,
1223                 m_bw_max, m_pdu_max);
1224
1225         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1226         timeout(reduce);       // call us reduce seconds later
1227     }
1228     else if (apdu->which == Z_GDU_Z3950)
1229         handle_incoming_Z_PDU(apdu->u.z3950);
1230     else if (apdu->which == Z_GDU_HTTP_Request)
1231         handle_incoming_HTTP(apdu->u.HTTP_Request);
1232 }
1233
1234 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1235 {
1236     if (m_max_record_retrieve)
1237     {
1238         if (apdu->which == Z_APDU_presentRequest)
1239         {
1240             Z_PresentRequest *pr = apdu->u.presentRequest;
1241             if (pr->numberOfRecordsRequested &&
1242                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1243                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1244         }
1245     }
1246 }
1247
1248 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1249                                                      int error,
1250                                                      const char *addinfo)
1251 {
1252     Z_Records *rec = (Z_Records *)
1253         odr_malloc (odr, sizeof(*rec));
1254     int *err = (int *)
1255         odr_malloc (odr, sizeof(*err));
1256     Z_DiagRec *drec = (Z_DiagRec *)
1257         odr_malloc (odr, sizeof(*drec));
1258     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1259         odr_malloc (odr, sizeof(*dr));
1260     *err = error;
1261     rec->which = Z_Records_NSD;
1262     rec->u.nonSurrogateDiagnostic = dr;
1263     dr->diagnosticSetId =
1264         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1265     dr->condition = err;
1266     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1267     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1268     return rec;
1269 }
1270
1271 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1272 {
1273     if (apdu->which == Z_APDU_searchRequest &&
1274         apdu->u.searchRequest->query &&
1275         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1276         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1277     {
1278         Z_RPNQuery *rpnquery = 0;
1279         Z_SearchRequest *sr = apdu->u.searchRequest;
1280
1281         yaz_log(LOG_LOG, "%sCQL: %s", m_session_str,
1282                 sr->query->u.type_104->u.cql);
1283
1284         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1285                                           &rpnquery, odr_encode());
1286         if (r == -3)
1287             yaz_log(LOG_LOG, "%sNo CQL to RPN table", m_session_str);
1288         else if (r)
1289             yaz_log(LOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1290         else
1291         {
1292             sr->query->which = Z_Query_type_1;
1293             sr->query->u.type_1 = rpnquery;
1294         }
1295         return apdu;
1296     }
1297     return apdu;
1298 }
1299
1300 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1301 {
1302     if (apdu->which == Z_APDU_searchRequest)
1303     {
1304         Z_SearchRequest *sr = apdu->u.searchRequest;
1305         int err = 0;
1306         char *addinfo = 0;
1307
1308         Yaz_ProxyConfig *cfg = check_reconfigure();
1309         if (cfg)
1310             err = cfg->check_query(odr_encode(), m_default_target,
1311                                    sr->query, &addinfo);
1312         if (err)
1313         {
1314             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1315
1316             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1317             new_apdu->u.searchResponse->records =
1318                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1319             *new_apdu->u.searchResponse->searchStatus = 0;
1320
1321             send_to_client(new_apdu);
1322
1323             return 0;
1324         }
1325     }
1326     return apdu;
1327 }
1328
1329 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1330 {
1331     m_marcxml_flag = 0;
1332     if (apdu->which == Z_APDU_searchRequest)
1333     {
1334         Z_SearchRequest *sr = apdu->u.searchRequest;
1335         int err = 0;
1336         char *addinfo = 0;
1337         Yaz_ProxyConfig *cfg = check_reconfigure();
1338
1339         if (cfg)
1340             err = cfg->check_syntax(odr_encode(),
1341                                     m_default_target,
1342                                     sr->preferredRecordSyntax,
1343                                     &addinfo);
1344         if (err == -1)
1345         {
1346             sr->preferredRecordSyntax =
1347                 yaz_oidval_to_z3950oid(odr_decode(), CLASS_RECSYN,
1348                                        VAL_USMARC);
1349             m_marcxml_flag = 1;
1350         }
1351         else if (err)
1352         {
1353             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1354
1355             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1356             new_apdu->u.searchResponse->records =
1357                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1358             *new_apdu->u.searchResponse->searchStatus = 0;
1359
1360             send_to_client(new_apdu);
1361
1362             return 0;
1363         }
1364     }
1365     else if (apdu->which == Z_APDU_presentRequest)
1366     {
1367         Z_PresentRequest *pr = apdu->u.presentRequest;
1368         int err = 0;
1369         char *addinfo = 0;
1370         Yaz_ProxyConfig *cfg = check_reconfigure();
1371
1372         if (cfg)
1373             err = cfg->check_syntax(odr_encode(), m_default_target,
1374                                     pr->preferredRecordSyntax,
1375                                     &addinfo);
1376         if (err == -1)
1377         {
1378             pr->preferredRecordSyntax =
1379                 yaz_oidval_to_z3950oid(odr_decode(), CLASS_RECSYN,
1380                                        VAL_USMARC);
1381             m_marcxml_flag = 1;
1382         }
1383         else if (err)
1384         {
1385             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1386
1387             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1388             new_apdu->u.presentResponse->records =
1389                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1390             *new_apdu->u.presentResponse->presentStatus =
1391                 Z_PresentStatus_failure;
1392
1393             send_to_client(new_apdu);
1394
1395             return 0;
1396         }
1397     }
1398     return apdu;
1399 }
1400
1401 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1402 {
1403     Z_SRW_PDU *srw_pdu = 0;
1404     char *soap_ns = 0;
1405     if (m_s2z_odr_init)
1406     {
1407         odr_destroy(m_s2z_odr_init);
1408         m_s2z_odr_init = 0;
1409     }
1410     if (m_s2z_odr_search)
1411     {
1412         odr_destroy(m_s2z_odr_search);
1413         m_s2z_odr_search = 0;
1414     }
1415
1416     m_http_keepalive = 0;
1417     m_http_version = 0;
1418     if (!strcmp(hreq->version, "1.0"))
1419     {
1420         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1421         if (v && !strcmp(v, "Keep-Alive"))
1422             m_http_keepalive = 1;
1423         else
1424             m_http_keepalive = 0;
1425         m_http_version = "1.0";
1426     }
1427     else
1428     {
1429         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1430         if (v && !strcmp(v, "close"))
1431             m_http_keepalive = 0;
1432         else
1433             m_http_keepalive = 1;
1434         m_http_version = "1.1";
1435     }
1436
1437     if (yaz_check_for_srw(hreq, &srw_pdu, &soap_ns, odr_decode()) == 0
1438         || yaz_check_for_sru(hreq, &srw_pdu, &soap_ns, odr_decode()) == 0)
1439     {
1440         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
1441         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
1442         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_ns);
1443         m_s2z_init_apdu = 0;
1444         m_s2z_search_apdu = 0;
1445         m_s2z_present_apdu = 0;
1446         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
1447         {
1448             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
1449
1450             // set packing for response records ..
1451             if (srw_req->recordPacking &&
1452                 !strcmp(srw_req->recordPacking, "xml"))
1453                 m_s2z_packing = Z_SRW_recordPacking_XML;
1454             else
1455                 m_s2z_packing = Z_SRW_recordPacking_string;
1456
1457             // prepare search PDU
1458             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
1459                                           Z_APDU_searchRequest);
1460             Z_SearchRequest *z_searchRequest =
1461                 m_s2z_search_apdu->u.searchRequest;
1462
1463             z_searchRequest->num_databaseNames = 1;
1464             z_searchRequest->databaseNames = (char**)
1465                 odr_malloc(m_s2z_odr_search, sizeof(char *));
1466             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
1467                                                            srw_req->database);
1468
1469             // query transformation
1470             Z_Query *query = (Z_Query *)
1471                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
1472             z_searchRequest->query = query;
1473
1474             if (srw_req->query_type == Z_SRW_query_type_cql)
1475             {
1476                 Z_External *ext = (Z_External *)
1477                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
1478                 ext->direct_reference =
1479                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
1480                 ext->indirect_reference = 0;
1481                 ext->descriptor = 0;
1482                 ext->which = Z_External_CQL;
1483                 ext->u.cql = srw_req->query.cql;
1484
1485                 query->which = Z_Query_type_104;
1486                 query->u.type_104 =  ext;
1487             }
1488             else if (srw_req->query_type == Z_SRW_query_type_pqf)
1489             {
1490                 Z_RPNQuery *RPNquery;
1491                 YAZ_PQF_Parser pqf_parser;
1492
1493                 pqf_parser = yaz_pqf_create ();
1494
1495                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
1496                                           srw_req->query.pqf);
1497                 if (!RPNquery)
1498                 {
1499                     const char *pqf_msg;
1500                     size_t off;
1501                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
1502                     yaz_log(LOG_LOG, "%*s^\n", off+4, "");
1503                     yaz_log(LOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
1504
1505                     send_to_srw_client_error(10);
1506                     return;
1507                 }
1508                 query->which = Z_Query_type_1;
1509                 query->u.type_1 =  RPNquery;
1510
1511                 yaz_pqf_destroy (pqf_parser);
1512             }
1513             else
1514             {
1515                 send_to_srw_client_error(11);
1516                 return;
1517             }
1518
1519             // present
1520             m_s2z_present_apdu = 0;
1521             int max = 0;
1522             if (srw_req->maximumRecords)
1523                 max = *srw_req->maximumRecords;
1524             int start = 1;
1525             if (srw_req->startRecord)
1526                 start = *srw_req->startRecord;
1527             if (max > 0)
1528             {
1529                 if (start <= 1)  // Z39.50 piggyback
1530                 {
1531                     *z_searchRequest->smallSetUpperBound = max;
1532                     *z_searchRequest->mediumSetPresentNumber = max;
1533                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
1534                     z_searchRequest->preferredRecordSyntax =
1535                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1536                                                VAL_TEXT_XML);
1537                 }
1538                 else   // Z39.50 present
1539                 {
1540                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
1541                                                    Z_APDU_presentRequest);
1542                     Z_PresentRequest *z_presentRequest =
1543                         m_s2z_present_apdu->u.presentRequest;
1544                     *z_presentRequest->resultSetStartPoint = start;
1545                     *z_presentRequest->numberOfRecordsRequested = max;
1546                     z_presentRequest->preferredRecordSyntax =
1547                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1548                                                VAL_TEXT_XML);
1549                 }
1550             }
1551             if (!m_client)
1552             {
1553                 yaz_log(LOG_LOG, "handle_incoming: initRequest");
1554                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1555                                             Z_APDU_initRequest);
1556
1557                 // prevent m_initRequest_apdu memory from being grabbed
1558                 // in Yaz_Proxy::handle_incoming_Z_PDU
1559                 m_initRequest_apdu = m_s2z_init_apdu;
1560                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1561                 return;
1562             }
1563             else
1564             {
1565                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1566                 return;
1567             }
1568         }
1569         else if (srw_pdu->which == Z_SRW_explain_request)
1570         {
1571             if (!m_client)
1572             {
1573                 yaz_log(LOG_LOG, "handle_incoming: initRequest");
1574                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1575                                             Z_APDU_initRequest);
1576
1577                 // prevent m_initRequest_apdu memory from being grabbed
1578                 // in Yaz_Proxy::handle_incoming_Z_PDU
1579                 m_initRequest_apdu = m_s2z_init_apdu;
1580                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1581             }
1582             else
1583                 send_srw_explain();
1584             return;
1585         }
1586     }
1587     int len = 0;
1588     Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
1589     send_GDU(p, &len);
1590     timeout(1);
1591 }
1592
1593 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
1594 {
1595     if (!m_client && m_invalid_session)
1596     {
1597         m_apdu_invalid_session = apdu;
1598         m_mem_invalid_session = odr_extract_mem(odr_decode());
1599         apdu = m_initRequest_apdu;
1600     }
1601
1602     // Determine our client.
1603     Z_OtherInformation **oi;
1604     get_otherInfoAPDU(apdu, &oi);
1605     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
1606     if (!m_client)
1607     {
1608         delete this;
1609         return;
1610     }
1611     m_client->m_server = this;
1612
1613     if (apdu->which == Z_APDU_initRequest)
1614     {
1615         if (apdu->u.initRequest->implementationId)
1616             yaz_log(LOG_LOG, "%simplementationId: %s",
1617                     m_session_str, apdu->u.initRequest->implementationId);
1618         if (apdu->u.initRequest->implementationName)
1619             yaz_log(LOG_LOG, "%simplementationName: %s",
1620                     m_session_str, apdu->u.initRequest->implementationName);
1621         if (apdu->u.initRequest->implementationVersion)
1622             yaz_log(LOG_LOG, "%simplementationVersion: %s",
1623                     m_session_str, apdu->u.initRequest->implementationVersion);
1624         if (m_initRequest_apdu == 0)
1625         {
1626             if (m_initRequest_mem)
1627                 nmem_destroy(m_initRequest_mem);
1628             m_initRequest_apdu = apdu;
1629             m_initRequest_mem = odr_extract_mem(odr_decode());
1630         }
1631         if (m_client->m_init_flag)
1632         {
1633             if (handle_init_response_for_invalid_session(apdu))
1634                 return;
1635             Z_APDU *apdu2 = m_client->m_initResponse;
1636             apdu2->u.initResponse->otherInfo = 0;
1637             if (m_client->m_cookie && *m_client->m_cookie)
1638                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
1639                                            m_client->m_cookie);
1640             apdu2->u.initResponse->referenceId =
1641                 apdu->u.initRequest->referenceId;
1642             send_to_client(apdu2);
1643             return;
1644         }
1645         m_client->m_init_flag = 1;
1646     }
1647     handle_max_record_retrieve(apdu);
1648
1649     if (apdu)
1650         apdu = handle_syntax_validation(apdu);
1651
1652     if (apdu)
1653         apdu = handle_query_transformation(apdu);
1654
1655     if (apdu)
1656         apdu = handle_query_validation(apdu);
1657
1658     if (apdu)
1659         apdu = result_set_optimize(apdu);
1660     if (!apdu)
1661     {
1662         m_client->timeout(m_target_idletime);  // mark it active even
1663         // though we didn't use it
1664         return;
1665     }
1666
1667     // delete other info part from PDU before sending to target
1668     get_otherInfoAPDU(apdu, &oi);
1669     if (oi)
1670         *oi = 0;
1671
1672     if (apdu->which == Z_APDU_presentRequest &&
1673         m_client->m_resultSetStartPoint == 0)
1674     {
1675         Z_PresentRequest *pr = apdu->u.presentRequest;
1676         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
1677         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
1678     } else {
1679         m_client->m_resultSetStartPoint = 0;
1680     }
1681     if (m_client->send_to_target(apdu) < 0)
1682     {
1683         delete m_client;
1684         m_client = 0;
1685         delete this;
1686     }
1687     else
1688         m_client->m_waiting = 1;
1689 }
1690
1691 void Yaz_Proxy::connectNotify()
1692 {
1693 }
1694
1695 void Yaz_Proxy::shutdown()
1696 {
1697     m_invalid_session = 0;
1698     // only keep if keep_alive flag is set...
1699     if (m_client &&
1700         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
1701         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
1702         m_client->m_waiting == 0)
1703     {
1704         yaz_log(LOG_LOG, "%sShutdown (client to proxy) keepalive %s",
1705                  m_session_str,
1706                  m_client->get_hostname());
1707         yaz_log(LOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1708                 m_session_str, m_client->m_pdu_recv,
1709                 m_client->m_bytes_sent + m_client->m_bytes_recv,
1710                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
1711         assert (m_client->m_waiting != 2);
1712         // Tell client (if any) that no server connection is there..
1713         m_client->m_server = 0;
1714         m_invalid_session = 0;
1715     }
1716     else if (m_client)
1717     {
1718         yaz_log (LOG_LOG, "%sShutdown (client to proxy) close %s",
1719                  m_session_str,
1720                  m_client->get_hostname());
1721         assert (m_client->m_waiting != 2);
1722         delete m_client;
1723     }
1724     else if (!m_parent)
1725     {
1726         yaz_log (LOG_LOG, "%sshutdown (client to proxy) bad state",
1727                  m_session_str);
1728         assert (m_parent);
1729     }
1730     else
1731     {
1732         yaz_log (LOG_LOG, "%sShutdown (client to proxy)",
1733                  m_session_str);
1734     }
1735     if (m_parent)
1736         m_parent->pre_init();
1737     delete this;
1738 }
1739
1740 const char *Yaz_ProxyClient::get_session_str()
1741 {
1742     if (!m_server)
1743         return "0 ";
1744     return m_server->get_session_str();
1745 }
1746
1747 void Yaz_ProxyClient::shutdown()
1748 {
1749     yaz_log (LOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
1750              get_hostname());
1751     delete m_server;
1752     delete this;
1753 }
1754
1755 void Yaz_Proxy::failNotify()
1756 {
1757     yaz_log (LOG_LOG, "%sConnection closed by client",
1758              get_session_str());
1759     shutdown();
1760 }
1761
1762 void Yaz_ProxyClient::failNotify()
1763 {
1764     yaz_log (LOG_LOG, "%sConnection closed by target %s",
1765              get_session_str(), get_hostname());
1766     shutdown();
1767 }
1768
1769 void Yaz_ProxyClient::connectNotify()
1770 {
1771     const char *s = get_session_str();
1772     const char *h = get_hostname();
1773     yaz_log (LOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
1774              m_target_idletime);
1775     timeout(m_target_idletime);
1776     if (!m_server)
1777         pre_init_client();
1778 }
1779
1780 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
1781                                                   *the_PDU_Observable, int fd)
1782 {
1783     return new Yaz_ProxyClient(the_PDU_Observable, 0);
1784 }
1785
1786 Yaz_ProxyClient::~Yaz_ProxyClient()
1787 {
1788     if (m_prev)
1789         *m_prev = m_next;
1790     if (m_next)
1791         m_next->m_prev = m_prev;
1792     m_waiting = 2;     // for debugging purposes only.
1793     odr_destroy(m_init_odr);
1794     delete m_last_query;
1795     xfree (m_last_resultSetId);
1796     xfree (m_cookie);
1797 }
1798
1799 void Yaz_ProxyClient::pre_init_client()
1800 {
1801     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
1802     Z_InitRequest *req = apdu->u.initRequest;
1803
1804     ODR_MASK_SET(req->options, Z_Options_search);
1805     ODR_MASK_SET(req->options, Z_Options_present);
1806     ODR_MASK_SET(req->options, Z_Options_namedResultSets);
1807     ODR_MASK_SET(req->options, Z_Options_triggerResourceCtrl);
1808     ODR_MASK_SET(req->options, Z_Options_scan);
1809     ODR_MASK_SET(req->options, Z_Options_sort);
1810     ODR_MASK_SET(req->options, Z_Options_extendedServices);
1811     ODR_MASK_SET(req->options, Z_Options_delSet);
1812
1813     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
1814     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
1815     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3);
1816
1817     if (send_to_target(apdu) < 0)
1818     {
1819         delete this;
1820     }
1821     else
1822     {
1823         m_waiting = 1;
1824         m_init_flag = 1;
1825     }
1826 }
1827
1828 void Yaz_Proxy::pre_init()
1829 {
1830     int i;
1831     const char *name = 0;
1832     const char *zurl_in_use[MAX_ZURL_PLEX];
1833     int limit_bw, limit_pdu, limit_req;
1834     int target_idletime, client_idletime;
1835     int max_clients;
1836     int keepalive_limit_bw, keepalive_limit_pdu;
1837     int pre_init;
1838     const char *cql2rpn = 0;
1839     const char *zeerex = 0;
1840
1841     Yaz_ProxyConfig *cfg = check_reconfigure();
1842
1843     zurl_in_use[0] = 0;
1844
1845     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
1846         set_APDU_yazlog(1);
1847     else
1848         set_APDU_yazlog(0);
1849
1850     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
1851                                           &limit_bw, &limit_pdu, &limit_req,
1852                                           &target_idletime, &client_idletime,
1853                                           &max_clients,
1854                                           &keepalive_limit_bw,
1855                                           &keepalive_limit_pdu,
1856                                           &pre_init,
1857                                           &cql2rpn,
1858                                           &zeerex) ; i++)
1859     {
1860         if (pre_init)
1861         {
1862             int j;
1863             for (j = 0; zurl_in_use[j]; j++)
1864             {
1865                 Yaz_ProxyClient *c;
1866                 int spare = 0;
1867                 int in_use = 0;
1868                 int other = 0;
1869                 for (c = m_clientPool; c; c = c->m_next)
1870                 {
1871                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
1872                     {
1873                         if (c->m_cookie == 0)
1874                         {
1875                             if (c->m_server == 0)
1876                                 spare++;
1877                             else
1878                                 in_use++;
1879                         }
1880                         else
1881                             other++;
1882                     }
1883                 }
1884                 yaz_log(LOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
1885                         "preinit=%d",m_session_str,
1886                         name, zurl_in_use[j], in_use, other, spare, pre_init);
1887                 if (spare < pre_init)
1888                 {
1889                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
1890                     c->m_next = m_clientPool;
1891                     if (c->m_next)
1892                         c->m_next->m_prev = &c->m_next;
1893                     m_clientPool = c;
1894                     c->m_prev = &m_clientPool;
1895
1896                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
1897                         c->set_APDU_yazlog(1);
1898                     else
1899                         c->set_APDU_yazlog(0);
1900
1901                     if (c->client(zurl_in_use[j]))
1902                     {
1903                         timeout(60);
1904                         delete c;
1905                         return;
1906                     }
1907                     c->timeout(30);
1908                     c->m_waiting = 1;
1909                     c->m_target_idletime = target_idletime;
1910                     c->m_seqno = m_seqno++;
1911                 }
1912             }
1913         }
1914     }
1915 }
1916
1917 void Yaz_Proxy::timeoutNotify()
1918 {
1919     if (m_parent)
1920     {
1921         if (m_bw_hold_PDU)
1922         {
1923             timeout(m_client_idletime);
1924             Z_GDU *apdu = m_bw_hold_PDU;
1925             m_bw_hold_PDU = 0;
1926
1927             if (apdu->which == Z_GDU_Z3950)
1928                 handle_incoming_Z_PDU(apdu->u.z3950);
1929             else if (apdu->which == Z_GDU_HTTP_Request)
1930                 handle_incoming_HTTP(apdu->u.HTTP_Request);
1931         }
1932         else
1933         {
1934             yaz_log (LOG_LOG, "%sTimeout (client to proxy)", m_session_str);
1935             shutdown();
1936         }
1937     }
1938     else
1939     {
1940         timeout(600);
1941         pre_init();
1942     }
1943 }
1944
1945 void Yaz_ProxyClient::timeoutNotify()
1946 {
1947     yaz_log (LOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
1948              get_hostname());
1949     m_waiting = 1;
1950     m_root->pre_init();
1951     shutdown();
1952 }
1953
1954 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
1955                                  Yaz_Proxy *parent) :
1956     Yaz_Z_Assoc (the_PDU_Observable)
1957 {
1958     m_cookie = 0;
1959     m_next = 0;
1960     m_prev = 0;
1961     m_init_flag = 0;
1962     m_last_query = 0;
1963     m_last_resultSetId = 0;
1964     m_last_resultCount = 0;
1965     m_last_ok = 0;
1966     m_sr_transform = 0;
1967     m_waiting = 0;
1968     m_init_odr = odr_createmem (ODR_DECODE);
1969     m_initResponse = 0;
1970     m_resultSetStartPoint = 0;
1971     m_bytes_sent = m_bytes_recv = 0;
1972     m_pdu_recv = 0;
1973     m_server = 0;
1974     m_seqno = 0;
1975     m_target_idletime = 600;
1976     m_root = parent;
1977 }
1978
1979 const char *Yaz_Proxy::option(const char *name, const char *value)
1980 {
1981     if (!strcmp (name, "optimize")) {
1982         if (value) {
1983             xfree (m_optimize);
1984             m_optimize = xstrdup (value);
1985         }
1986         return m_optimize;
1987     }
1988     return 0;
1989 }
1990
1991 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
1992 {
1993
1994 }
1995
1996 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
1997 {
1998     if (apdu->which == Z_GDU_Z3950)
1999         recv_Z_PDU(apdu->u.z3950, len);
2000     else if (apdu->which == Z_GDU_HTTP_Response)
2001         recv_HTTP_response(apdu->u.HTTP_Response, len);
2002     else
2003         shutdown();
2004 }
2005
2006 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2007 {
2008     if (!m_invalid_session)
2009         return 0;
2010     m_invalid_session = 0;
2011     handle_incoming_Z_PDU(m_apdu_invalid_session);
2012     assert (m_mem_invalid_session);
2013     nmem_destroy(m_mem_invalid_session);
2014     m_mem_invalid_session = 0;
2015     return 1;
2016 }
2017
2018 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2019 {
2020     m_bytes_recv += len;
2021     m_pdu_recv++;
2022     m_waiting = 0;
2023     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2024         yaz_log (LOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2025                  apdu_name(apdu), get_hostname(), len);
2026     if (apdu->which == Z_APDU_initResponse)
2027     {
2028         if (!m_server)  // if this is a pre init session , check for more
2029             m_root->pre_init();
2030         NMEM nmem = odr_extract_mem (odr_decode());
2031         odr_reset (m_init_odr);
2032         nmem_transfer (m_init_odr->mem, nmem);
2033         m_initResponse = apdu;
2034
2035         Z_InitResponse *ir = apdu->u.initResponse;
2036         char *im0 = ir->implementationName;
2037
2038         char *im1 = (char*)
2039             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2040         *im1 = '\0';
2041         if (im0)
2042         {
2043             strcat(im1, im0);
2044             strcat(im1, " ");
2045         }
2046         strcat(im1, "(YAZ Proxy)");
2047         ir->implementationName = im1;
2048
2049         nmem_destroy (nmem);
2050
2051         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2052             return;
2053     }
2054     if (apdu->which == Z_APDU_searchResponse)
2055     {
2056         Z_SearchResponse *sr = apdu->u.searchResponse;
2057         m_last_resultCount = *sr->resultCount;
2058         int status = *sr->searchStatus;
2059         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2060         {
2061             m_last_ok = 1;
2062
2063             if (sr->records && sr->records->which == Z_Records_DBOSD)
2064             {
2065                 m_cache.add(odr_decode(),
2066                             sr->records->u.databaseOrSurDiagnostics, 1,
2067                             *sr->resultCount);
2068             }
2069         }
2070     }
2071     if (apdu->which == Z_APDU_presentResponse)
2072     {
2073         Z_PresentResponse *pr = apdu->u.presentResponse;
2074         if (m_sr_transform)
2075         {
2076             m_sr_transform = 0;
2077             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2078             Z_SearchResponse *sr = new_apdu->u.searchResponse;
2079             sr->referenceId = pr->referenceId;
2080             *sr->resultCount = m_last_resultCount;
2081             sr->records = pr->records;
2082             sr->nextResultSetPosition = pr->nextResultSetPosition;
2083             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2084             apdu = new_apdu;
2085         }
2086         if (pr->records &&
2087             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2088         {
2089             m_cache.add(odr_decode(),
2090                         pr->records->u.databaseOrSurDiagnostics,
2091                         m_resultSetStartPoint, -1);
2092             m_resultSetStartPoint = 0;
2093         }
2094     }
2095     if (m_cookie)
2096         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2097     if (m_server)
2098     {
2099         m_server->send_to_client(apdu);
2100     }
2101     if (apdu->which == Z_APDU_close)
2102     {
2103         shutdown();
2104     }
2105 }
2106
2107 int Yaz_Proxy::server(const char *addr)
2108 {
2109     int r = Yaz_Z_Assoc::server(addr);
2110     if (!r)
2111     {
2112         yaz_log(LOG_LOG, "%sStarted listener on %s", m_session_str, addr);
2113         timeout(1);
2114     }
2115     return r;
2116 }
2117