src/yaz-proxy.cpp

   1 /*
   2  * Copyright (c) 1998-2004, Index Data.
   3  * See the file LICENSE for details.
   4  *
   5  * $Id: yaz-proxy.cpp,v 1.76 2004-01-05 11:31:04 adam Exp $
   6  */
   7
   8 #include <assert.h>
   9 #include <time.h>
  10
  11 #include <yaz/srw.h>
  12 #include <yaz/marcdisp.h>
  13 #include <yaz/yaz-iconv.h>
  14 #include <yaz/log.h>
  15 #include <yaz/diagbib1.h>
  16 #include <yaz++/proxy.h>
  17 #include <yaz/pquery.h>
  18
  19 #if HAVE_XSLT
  20 #include <libxslt/xsltutils.h>
  21 #include <libxslt/transform.h>
  22 #endif
  23
  24 static const char *apdu_name(Z_APDU *apdu)
  25 {
  26     switch (apdu->which)
  27     {
  28     case Z_APDU_initRequest:
  29         return "initRequest";
  30     case Z_APDU_initResponse:
  31         return "initResponse";
  32     case Z_APDU_searchRequest:
  33         return "searchRequest";
  34     case Z_APDU_searchResponse:
  35         return "searchResponse";
  36     case Z_APDU_presentRequest:
  37         return "presentRequest";
  38     case Z_APDU_presentResponse:
  39         return "presentResponse";
  40     case Z_APDU_deleteResultSetRequest:
  41         return "deleteResultSetRequest";
  42     case Z_APDU_deleteResultSetResponse:
  43         return "deleteResultSetResponse";
  44     case Z_APDU_scanRequest:
  45         return "scanRequest";
  46     case Z_APDU_scanResponse:
  47         return "scanResponse";
  48     case Z_APDU_sortRequest:
  49         return "sortRequest";
  50     case Z_APDU_sortResponse:
  51         return "sortResponse";
  52     case Z_APDU_extendedServicesRequest:
  53         return "extendedServicesRequest";
  54     case Z_APDU_extendedServicesResponse:
  55         return "extendedServicesResponse";
  56     case Z_APDU_close:
  57         return "close";
  58     }
  59     return "other";
  60 }
  61
  62 static const char *gdu_name(Z_GDU *gdu)
  63 {
  64     switch(gdu->which)
  65     {
  66     case Z_GDU_Z3950:
  67         return apdu_name(gdu->u.z3950);
  68     case Z_GDU_HTTP_Request:
  69         return "HTTP Request";
  70     case Z_GDU_HTTP_Response:
  71         return "HTTP Response";
  72     }
  73     return "Unknown request/response";
  74 }
  75
  76 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
  77                      Yaz_Proxy *parent) :
  78     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
  79 {
  80     m_PDU_Observable = the_PDU_Observable;
  81     m_client = 0;
  82     m_parent = parent;
  83     m_clientPool = 0;
  84     m_seqno = 1;
  85     m_keepalive_limit_bw = 500000;
  86     m_keepalive_limit_pdu = 1000;
  87     m_proxyTarget = 0;
  88     m_default_target = 0;
  89     m_proxy_authentication = 0;
  90     m_max_clients = 150;
  91     m_log_mask = 0;
  92     m_seed = time(0);
  93     m_client_idletime = 600;
  94     m_target_idletime = 600;
  95     m_optimize = xstrdup ("1");
  96     strcpy(m_session_str, "0 ");
  97     m_session_no=0;
  98     m_bytes_sent = m_bytes_recv = 0;
  99     m_bw_hold_PDU = 0;
 100     m_bw_max = 0;
 101     m_pdu_max = 0;
 102     m_max_record_retrieve = 0;
 103     m_reconfig_flag = 0;
 104     m_config_fname = 0;
 105     m_request_no = 0;
 106     m_invalid_session = 0;
 107     m_config = 0;
 108     m_marcxml_flag = 0;
 109     m_stylesheet = 0;
 110     m_initRequest_apdu = 0;
 111     m_initRequest_mem = 0;
 112     m_apdu_invalid_session = 0;
 113     m_mem_invalid_session = 0;
 114     m_s2z_odr_init = 0;
 115     m_s2z_odr_search = 0;
 116     m_s2z_init_apdu = 0;
 117     m_s2z_search_apdu = 0;
 118     m_s2z_present_apdu = 0;
 119     m_http_keepalive = 0;
 120     m_http_version = 0;
 121     m_soap_ns = 0;
 122     m_s2z_packing = Z_SRW_recordPacking_string;
 123     m_time_tv.tv_sec = 0;
 124     m_time_tv.tv_usec = 0;
 125 }
 126
 127 Yaz_Proxy::~Yaz_Proxy()
 128 {
 129     yaz_log(LOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 130             m_bytes_sent, m_bytes_recv);
 131     nmem_destroy(m_initRequest_mem);
 132     nmem_destroy(m_mem_invalid_session);
 133     xfree (m_proxyTarget);
 134     xfree (m_default_target);
 135     xfree (m_proxy_authentication);
 136     xfree (m_optimize);
 137     xfree (m_stylesheet);
 138     if (m_s2z_odr_init)
 139         odr_destroy(m_s2z_odr_init);
 140     if (m_s2z_odr_search)
 141         odr_destroy(m_s2z_odr_search);
 142     delete m_config;
 143 }
 144
 145 int Yaz_Proxy::set_config(const char *config)
 146 {
 147     delete m_config;
 148     m_config = new Yaz_ProxyConfig();
 149     xfree(m_config_fname);
 150     m_config_fname = xstrdup(config);
 151     int r = m_config->read_xml(config);
 152     if (!r)
 153         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 154     return r;
 155 }
 156
 157 void Yaz_Proxy::set_default_target(const char *target)
 158 {
 159     xfree (m_default_target);
 160     m_default_target = 0;
 161     if (target)
 162         m_default_target = (char *) xstrdup (target);
 163 }
 164
 165 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 166 {
 167     xfree (m_proxy_authentication);
 168     m_proxy_authentication = 0;
 169     if (auth)
 170         m_proxy_authentication = (char *) xstrdup (auth);
 171 }
 172
 173 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 174 {
 175     if (m_parent)
 176         return m_parent->check_reconfigure();
 177
 178     Yaz_ProxyConfig *cfg = m_config;
 179     if (m_reconfig_flag)
 180     {
 181         yaz_log(LOG_LOG, "reconfigure");
 182         yaz_log_reopen();
 183         if (m_config_fname && cfg)
 184         {
 185             yaz_log(LOG_LOG, "reconfigure config %s", m_config_fname);
 186             int r = cfg->read_xml(m_config_fname);
 187             if (r)
 188                 yaz_log(LOG_WARN, "reconfigure failed");
 189             else
 190             {
 191                 m_log_mask = 0;
 192                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 193             }
 194         }
 195         else
 196             yaz_log(LOG_LOG, "reconfigure");
 197         m_reconfig_flag = 0;
 198     }
 199     return cfg;
 200 }
 201
 202 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 203                                             *the_PDU_Observable, int fd)
 204 {
 205     check_reconfigure();
 206     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 207     new_proxy->m_config = 0;
 208     new_proxy->m_config_fname = 0;
 209     new_proxy->timeout(m_client_idletime);
 210     new_proxy->m_target_idletime = m_target_idletime;
 211     new_proxy->set_default_target(m_default_target);
 212     new_proxy->m_max_clients = m_max_clients;
 213     new_proxy->m_log_mask = m_log_mask;
 214     new_proxy->set_APDU_log(get_APDU_log());
 215     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 216         new_proxy->set_APDU_yazlog(1);
 217     else
 218         new_proxy->set_APDU_yazlog(0);
 219     new_proxy->set_proxy_authentication(m_proxy_authentication);
 220     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 221     m_session_no++;
 222     yaz_log (LOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 223              the_PDU_Observable->getpeername());
 224     return new_proxy;
 225 }
 226
 227 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 228 {
 229     int oid[OID_SIZE];
 230     Z_OtherInformationUnit *oi;
 231     struct oident ent;
 232     ent.proto = PROTO_Z3950;
 233     ent.oclass = CLASS_USERINFO;
 234     ent.value = (oid_value) VAL_COOKIE;
 235     assert (oid_ent_to_oid (&ent, oid));
 236
 237     if (oid_ent_to_oid (&ent, oid) &&
 238         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 239         oi->which == Z_OtherInfo_characterInfo)
 240         return oi->information.characterInfo;
 241     return 0;
 242 }
 243
 244 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 245 {
 246     int oid[OID_SIZE];
 247     Z_OtherInformationUnit *oi;
 248     struct oident ent;
 249     ent.proto = PROTO_Z3950;
 250     ent.oclass = CLASS_USERINFO;
 251     ent.value = (oid_value) VAL_PROXY;
 252     if (oid_ent_to_oid (&ent, oid) &&
 253         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 254         oi->which == Z_OtherInfo_characterInfo)
 255         return oi->information.characterInfo;
 256     return 0;
 257 }
 258
 259 const char *Yaz_Proxy::load_balance(const char **url)
 260 {
 261     int zurl_in_use[MAX_ZURL_PLEX];
 262     Yaz_ProxyClient *c;
 263     int i;
 264
 265     for (i = 0; i<MAX_ZURL_PLEX; i++)
 266         zurl_in_use[i] = 0;
 267     for (c = m_parent->m_clientPool; c; c = c->m_next)
 268     {
 269         for (i = 0; url[i]; i++)
 270             if (!strcmp(url[i], c->get_hostname()))
 271                 zurl_in_use[i]++;
 272     }
 273     int min = 100000;
 274     const char *ret = 0;
 275     for (i = 0; url[i]; i++)
 276     {
 277         yaz_log(LOG_DEBUG, "%szurl=%s use=%d",
 278                 m_session_str, url[i], zurl_in_use[i]);
 279         if (min > zurl_in_use[i])
 280         {
 281             ret = url[i];
 282             min = zurl_in_use[i];
 283         }
 284     }
 285     return ret;
 286 }
 287
 288 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 289                                        const char *proxy_host)
 290 {
 291     assert (m_parent);
 292     Yaz_Proxy *parent = m_parent;
 293     Z_OtherInformation **oi;
 294     Yaz_ProxyClient *c = m_client;
 295
 296     if (!m_proxyTarget)
 297     {
 298         const char *url[MAX_ZURL_PLEX];
 299         Yaz_ProxyConfig *cfg = check_reconfigure();
 300         if (proxy_host)
 301         {
 302 #if 1
 303 /* only to be enabled for debugging... */
 304             if (!strcmp(proxy_host, "stop"))
 305                 exit(0);
 306 #endif
 307             xfree(m_default_target);
 308             m_default_target = xstrdup(proxy_host);
 309             proxy_host = m_default_target;
 310         }
 311         int client_idletime = -1;
 312         const char *cql2rpn_fname = 0;
 313         url[0] = m_default_target;
 314         url[1] = 0;
 315         if (cfg)
 316         {
 317             int pre_init = 0;
 318             cfg->get_target_info(proxy_host, url, &m_bw_max,
 319                                  &m_pdu_max, &m_max_record_retrieve,
 320                                  &m_target_idletime, &client_idletime,
 321                                  &parent->m_max_clients,
 322                                  &m_keepalive_limit_bw,
 323                                  &m_keepalive_limit_pdu,
 324                                  &pre_init,
 325                                  &cql2rpn_fname);
 326         }
 327         if (client_idletime != -1)
 328         {
 329             m_client_idletime = client_idletime;
 330             timeout(m_client_idletime);
 331         }
 332         if (cql2rpn_fname)
 333             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 334         if (!url[0])
 335         {
 336             yaz_log(LOG_LOG, "%sNo default target", m_session_str);
 337             return 0;
 338         }
 339         // we don't handle multiplexing for cookie session, so we just
 340         // pick the first one in this case (anonymous users will be able
 341         // to use any backend)
 342         if (cookie && *cookie)
 343             m_proxyTarget = (char*) xstrdup(url[0]);
 344         else
 345             m_proxyTarget = (char*) xstrdup(load_balance(url));
 346     }
 347     if (cookie && *cookie)
 348     {   // search in sessions with a cookie
 349         for (c = parent->m_clientPool; c; c = c->m_next)
 350         {
 351             assert (c->m_prev);
 352             assert (*c->m_prev == c);
 353             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 354                 !strcmp(m_proxyTarget, c->get_hostname()))
 355             {
 356                 // Found it in cache
 357                 // The following handles "cancel"
 358                 // If connection is busy (waiting for PDU) and
 359                 // we have an initRequest we can safely do re-open
 360                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 361                 {
 362                     yaz_log (LOG_LOG, "%s REOPEN target=%s", m_session_str,
 363                              c->get_hostname());
 364                     c->close();
 365                     c->m_init_flag = 0;
 366
 367                     c->m_last_ok = 0;
 368                     c->m_cache.clear();
 369                     c->m_last_resultCount = 0;
 370                     c->m_sr_transform = 0;
 371                     c->m_waiting = 0;
 372                     c->m_resultSetStartPoint = 0;
 373                     c->m_target_idletime = m_target_idletime;
 374                     if (c->client(m_proxyTarget))
 375                     {
 376                         delete c;
 377                         return 0;
 378                     }
 379                     c->timeout(30);
 380                 }
 381                 c->m_seqno = parent->m_seqno;
 382                 if (c->m_server && c->m_server != this)
 383                     c->m_server->m_client = 0;
 384                 c->m_server = this;
 385                 (parent->m_seqno)++;
 386                 yaz_log (LOG_DEBUG, "get_client 1 %p %p", this, c);
 387                 return c;
 388             }
 389         }
 390     }
 391     else if (!c)
 392     {
 393         // don't have a client session yet. Search in session w/o cookie
 394         for (c = parent->m_clientPool; c; c = c->m_next)
 395         {
 396             assert (c->m_prev);
 397             assert (*c->m_prev == c);
 398             if (c->m_server == 0 && c->m_cookie == 0 &&
 399                 c->m_waiting == 0 &&
 400                 !strcmp(m_proxyTarget, c->get_hostname()))
 401             {
 402                 // found it in cache
 403                 yaz_log (LOG_LOG, "%sREUSE %s",
 404                          m_session_str, c->get_hostname());
 405
 406                 c->m_seqno = parent->m_seqno;
 407                 assert(c->m_server == 0);
 408                 c->m_server = this;
 409
 410                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 411                     c->set_APDU_yazlog(1);
 412                 else
 413                     c->set_APDU_yazlog(0);
 414
 415                 (parent->m_seqno)++;
 416
 417                 parent->pre_init();
 418
 419                 return c;
 420             }
 421         }
 422     }
 423     if (!m_client)
 424     {
 425         if (apdu->which != Z_APDU_initRequest)
 426         {
 427             yaz_log (LOG_LOG, "no first INIT!");
 428             return 0;
 429         }
 430         Z_InitRequest *initRequest = apdu->u.initRequest;
 431
 432         if (!initRequest->idAuthentication)
 433         {
 434             if (m_proxy_authentication)
 435             {
 436                 initRequest->idAuthentication =
 437                     (Z_IdAuthentication *)
 438                     odr_malloc (odr_encode(),
 439                                 sizeof(*initRequest->idAuthentication));
 440                 initRequest->idAuthentication->which =
 441                     Z_IdAuthentication_open;
 442                 initRequest->idAuthentication->u.open =
 443                     odr_strdup (odr_encode(), m_proxy_authentication);
 444             }
 445         }
 446
 447         // go through list of clients - and find the lowest/oldest one.
 448         Yaz_ProxyClient *c_min = 0;
 449         int min_seq = -1;
 450         int no_of_clients = 0;
 451         if (parent->m_clientPool)
 452             yaz_log (LOG_DEBUG, "Existing sessions");
 453         for (c = parent->m_clientPool; c; c = c->m_next)
 454         {
 455             yaz_log (LOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 456                                c->m_waiting, c->get_hostname(),
 457                                c->m_cookie ? c->m_cookie : "");
 458             no_of_clients++;
 459             if (min_seq < 0 || c->m_seqno < min_seq)
 460             {
 461                 min_seq = c->m_seqno;
 462                 c_min = c;
 463             }
 464         }
 465         if (no_of_clients >= parent->m_max_clients)
 466         {
 467             c = c_min;
 468             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 469             {
 470                 yaz_log (LOG_LOG, "%sMAXCLIENTS Destroy %d",
 471                          m_session_str, c->m_seqno);
 472                 if (c->m_server && c->m_server != this)
 473                     delete c->m_server;
 474                 c->m_server = 0;
 475             }
 476             else
 477             {
 478                 yaz_log (LOG_LOG, "%sMAXCLIENTS Reuse %d %d %s",
 479                          m_session_str,
 480                          c->m_seqno, parent->m_seqno, c->get_hostname());
 481                 xfree (c->m_cookie);
 482                 c->m_cookie = 0;
 483                 if (cookie)
 484                     c->m_cookie = xstrdup(cookie);
 485                 c->m_seqno = parent->m_seqno;
 486                 if (c->m_server && c->m_server != this)
 487                 {
 488                     c->m_server->m_client = 0;
 489                     delete c->m_server;
 490                 }
 491                 (parent->m_seqno)++;
 492                 c->m_target_idletime = m_target_idletime;
 493                 c->timeout(m_target_idletime);
 494
 495                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 496                     c->set_APDU_yazlog(1);
 497                 else
 498                     c->set_APDU_yazlog(0);
 499
 500                 return c;
 501             }
 502         }
 503         else
 504         {
 505             yaz_log (LOG_LOG, "%sNEW %d %s",
 506                      m_session_str, parent->m_seqno, m_proxyTarget);
 507             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 508             c->m_next = parent->m_clientPool;
 509             if (c->m_next)
 510                 c->m_next->m_prev = &c->m_next;
 511             parent->m_clientPool = c;
 512             c->m_prev = &parent->m_clientPool;
 513         }
 514
 515         xfree (c->m_cookie);
 516         c->m_cookie = 0;
 517         if (cookie)
 518             c->m_cookie = xstrdup(cookie);
 519
 520         c->m_seqno = parent->m_seqno;
 521         c->m_init_flag = 0;
 522         c->m_last_resultCount = 0;
 523         c->m_last_ok = 0;
 524         c->m_cache.clear();
 525         c->m_sr_transform = 0;
 526         c->m_waiting = 0;
 527         c->m_resultSetStartPoint = 0;
 528         (parent->m_seqno)++;
 529         if (c->client(m_proxyTarget))
 530         {
 531             delete c;
 532             return 0;
 533         }
 534         c->m_target_idletime = m_target_idletime;
 535         c->timeout(30);
 536
 537         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 538             c->set_APDU_yazlog(1);
 539         else
 540             c->set_APDU_yazlog(0);
 541     }
 542     yaz_log (LOG_DEBUG, "get_client 3 %p %p", this, c);
 543     return c;
 544 }
 545
 546 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 547 {
 548     int i;
 549     for (i = 0; i<num; i++)
 550     {
 551         oident *ent;
 552         Z_DefaultDiagFormat *r;
 553         Z_DiagRec *p = pp[i];
 554         if (p->which != Z_DiagRec_defaultFormat)
 555         {
 556             yaz_log(LOG_LOG, "%sError no diagnostics", m_session_str);
 557             return;
 558         }
 559         else
 560             r = p->u.defaultFormat;
 561         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 562             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 563             yaz_log(LOG_LOG, "%sError unknown diagnostic set", m_session_str);
 564         switch (r->which)
 565         {
 566         case Z_DefaultDiagFormat_v2Addinfo:
 567             yaz_log(LOG_LOG, "%sError %d %s:%s",
 568                     m_session_str,
 569                     *r->condition, diagbib1_str(*r->condition),
 570                     r->u.v2Addinfo);
 571             break;
 572         case Z_DefaultDiagFormat_v3Addinfo:
 573             yaz_log(LOG_LOG, "%sError %d %s:%s",
 574                     m_session_str,
 575                     *r->condition, diagbib1_str(*r->condition),
 576                     r->u.v3Addinfo);
 577             break;
 578         }
 579     }
 580 }
 581
 582 void Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p)
 583 {
 584     if (!m_stylesheet)
 585         return;
 586     xmlDocPtr xslt_doc = xmlParseFile(m_stylesheet);
 587     xsltStylesheetPtr xsp;
 588
 589     xsp = xsltParseStylesheetDoc(xslt_doc);
 590
 591     int i;
 592     for (i = 0; i < p->num_records; i++)
 593     {
 594         Z_NamePlusRecord *npr = p->records[i];
 595         if (npr->which == Z_NamePlusRecord_databaseRecord)
 596         {
 597             Z_External *r = npr->u.databaseRecord;
 598             if (r->which == Z_External_octet)
 599             {
 600                 xmlDocPtr res, doc = xmlParseMemory(
 601                     (char*) r->u.octet_aligned->buf,
 602                     r->u.octet_aligned->len);
 603
 604                 res = xsltApplyStylesheet(xsp, doc, 0);
 605
 606                 xmlChar *out_buf;
 607                 int out_len;
 608                 xmlDocDumpMemory (res, &out_buf, &out_len);
 609                 p->records[i]->u.databaseRecord =
 610                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 611                                  (char*) out_buf, out_len);
 612                 xmlFreeDoc(doc);
 613                 xmlFreeDoc(res);
 614             }
 615         }
 616     }
 617     xsltFreeStylesheet(xsp);
 618 }
 619
 620 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p)
 621 {
 622     int i;
 623
 624     yaz_marc_t mt = yaz_marc_create();
 625     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 626     for (i = 0; i < p->num_records; i++)
 627     {
 628         Z_NamePlusRecord *npr = p->records[i];
 629         if (npr->which == Z_NamePlusRecord_databaseRecord)
 630         {
 631             Z_External *r = npr->u.databaseRecord;
 632             if (r->which == Z_External_octet)
 633             {
 634                 int rlen;
 635                 char *result;
 636                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 637                                         r->u.octet_aligned->len,
 638                                         &result, &rlen))
 639                 {
 640                     yaz_iconv_t cd = yaz_iconv_open("UTF-8", "MARC-8");
 641                     WRBUF wrbuf = wrbuf_alloc();
 642
 643                     char outbuf[120];
 644                     size_t inbytesleft = rlen;
 645                     const char *inp = result;
 646                     while (cd && inbytesleft)
 647                     {
 648                         size_t outbytesleft = sizeof(outbuf);
 649                         char *outp = outbuf;
 650                         size_t r;
 651
 652                         r = yaz_iconv (cd, (char**) &inp,
 653                                        &inbytesleft,
 654                                        &outp, &outbytesleft);
 655                         if (r == (size_t) (-1))
 656                         {
 657                             int e = yaz_iconv_error(cd);
 658                             if (e != YAZ_ICONV_E2BIG)
 659                             {
 660                                 yaz_log(LOG_WARN, "conversion failure");
 661                                 break;
 662                             }
 663                         }
 664                         wrbuf_write(wrbuf, outbuf, outp - outbuf);
 665                     }
 666                     if (cd)
 667                         yaz_iconv_close(cd);
 668
 669                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 670                                                          VAL_TEXT_XML,
 671                                                          wrbuf_buf(wrbuf),
 672                                                          wrbuf_len(wrbuf));
 673                     wrbuf_free(wrbuf, 1);
 674                 }
 675             }
 676         }
 677     }
 678     yaz_marc_destroy(mt);
 679 }
 680
 681 void Yaz_Proxy::logtime()
 682 {
 683     if (m_time_tv.tv_sec)
 684     {
 685         struct timeval tv;
 686         gettimeofday(&tv, 0);
 687         long diff = (tv.tv_sec - m_time_tv.tv_sec)*1000000 +
 688             (tv.tv_usec - m_time_tv.tv_usec);
 689         if (diff >= 0)
 690             yaz_log(LOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 691                     diff/1000000, (diff/1000)%1000);
 692     }
 693     m_time_tv.tv_sec = 0;
 694     m_time_tv.tv_usec = 0;
 695 }
 696
 697 int Yaz_Proxy::send_http_response(int code)
 698 {
 699     ODR o = odr_encode();
 700     const char *ctype = "text/xml";
 701     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 702     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 703     if (m_http_version)
 704         hres->version = odr_strdup(o, m_http_version);
 705     m_http_keepalive = 0;
 706     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 707     {
 708         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 709                  gdu_name(gdu));
 710     }
 711     int len;
 712     int r = send_GDU(gdu, &len);
 713     logtime();
 714     return r;
 715 }
 716
 717 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 718 {
 719     ODR o = odr_encode();
 720     const char *ctype = "text/xml";
 721     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 722     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 723     if (m_http_version)
 724         hres->version = odr_strdup(o, m_http_version);
 725     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 726     if (m_http_keepalive)
 727         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 728
 729     static Z_SOAP_Handler soap_handlers[2] = {
 730 #if HAVE_XSLT
 731         {"http://www.loc.gov/zing/srw/", 0,
 732          (Z_SOAP_fun) yaz_srw_codec},
 733 #endif
 734         {0, 0, 0}
 735     };
 736
 737     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 738     soap_package->which = Z_SOAP_generic;
 739     soap_package->u.generic =
 740         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 741     soap_package->u.generic->no = 0;
 742     soap_package->u.generic->ns = soap_handlers[0].ns;
 743     soap_package->u.generic->p = (void *) srw_pdu;
 744     soap_package->ns = m_soap_ns;
 745     int ret = z_soap_codec_enc(o, &soap_package,
 746                                &hres->content_buf, &hres->content_len,
 747                                soap_handlers, 0);
 748     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 749     {
 750         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 751                  gdu_name(gdu));
 752     }
 753     int len;
 754     int r = send_GDU(gdu, &len);
 755     logtime();
 756     return r;
 757 }
 758
 759 int Yaz_Proxy::send_to_srw_client_error(int srw_error)
 760 {
 761     ODR o = odr_encode();
 762     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 763     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 764
 765     srw_res->num_diagnostics = 1;
 766     srw_res->diagnostics = (Z_SRW_diagnostic *)
 767         odr_malloc(o, sizeof(*srw_res->diagnostics));
 768     srw_res->diagnostics[0].code =  odr_intdup(o, srw_error);
 769     srw_res->diagnostics[0].details = 0;
 770     return send_srw_response(srw_pdu);
 771 }
 772
 773 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 774                              Z_DefaultDiagFormat *ddf)
 775 {
 776     int bib1_code = *ddf->condition;
 777     if (bib1_code == 109)
 778         return 404;
 779     srw_res->num_diagnostics = 1;
 780     srw_res->diagnostics = (Z_SRW_diagnostic *)
 781         odr_malloc(o, sizeof(*srw_res->diagnostics));
 782     srw_res->diagnostics[0].code =
 783         odr_intdup(o, yaz_diag_bib1_to_srw(*ddf->condition));
 784     srw_res->diagnostics[0].details = ddf->u.v2Addinfo;
 785     return 0;
 786 }
 787
 788 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
 789 {
 790     ODR o = odr_encode();
 791     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 792     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 793
 794     srw_res->numberOfRecords = odr_intdup (o, hits);
 795     if (records && records->which == Z_Records_DBOSD)
 796     {
 797         srw_res->num_records =
 798             records->u.databaseOrSurDiagnostics->num_records;
 799         int i;
 800         srw_res->records = (Z_SRW_record *)
 801             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
 802         for (i = 0; i < srw_res->num_records; i++)
 803         {
 804             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
 805             if (npr->which != Z_NamePlusRecord_databaseRecord)
 806             {
 807                 srw_res->records[i].recordSchema = "diagnostic";
 808                 srw_res->records[i].recordPacking = m_s2z_packing;
 809                 srw_res->records[i].recordData_buf = "67";
 810                 srw_res->records[i].recordData_len = 2;
 811                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 812                 continue;
 813             }
 814             Z_External *r = npr->u.databaseRecord;
 815             oident *ent = oid_getentbyoid(r->direct_reference);
 816             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
 817             {
 818                 srw_res->records[i].recordSchema = "http://www.loc.gov/marcxml/";
 819                 srw_res->records[i].recordPacking = m_s2z_packing;
 820                 srw_res->records[i].recordData_buf = (char*)
 821                     r->u.octet_aligned->buf;
 822                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
 823                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 824             }
 825             else
 826             {
 827                 srw_res->records[i].recordSchema = "diagnostic";
 828                 srw_res->records[i].recordPacking = m_s2z_packing;
 829                 srw_res->records[i].recordData_buf = "67";
 830                 srw_res->records[i].recordData_len = 2;
 831                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 832             }
 833         }
 834     }
 835     if (records && records->which == Z_Records_NSD)
 836     {
 837         int http_code;
 838         http_code = z_to_srw_diag(odr_encode(), srw_res,
 839                                    records->u.nonSurrogateDiagnostic);
 840         if (http_code)
 841             return send_http_response(http_code);
 842     }
 843     return send_srw_response(srw_pdu);
 844
 845 }
 846
 847 int Yaz_Proxy::send_srw_explain()
 848 {
 849     Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
 850     Z_SRW_explainResponse *er = res->u.explain_response;
 851
 852     Yaz_ProxyConfig *cfg = check_reconfigure();
 853     if (cfg)
 854     {
 855         int len;
 856         assert (m_proxyTarget);
 857         char *b = cfg->get_explain(odr_encode(), 0 /* target */,
 858                                    0 /* db */, &len);
 859         if (b)
 860         {
 861             er->record.recordData_buf = b;
 862             er->record.recordData_len = len;
 863             er->record.recordPacking = m_s2z_packing;
 864         }
 865     }
 866     return send_srw_response(res);
 867 }
 868
 869 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu, int *len)
 870 {
 871     if (m_http_version)
 872     {
 873         if (apdu->which == Z_APDU_initResponse)
 874         {
 875             Z_InitResponse *res = apdu->u.initResponse;
 876             if (*res->result == 0)
 877             {
 878                 send_to_srw_client_error(3);
 879             }
 880             else if (!m_s2z_search_apdu)
 881             {
 882                 send_srw_explain();
 883             }
 884             else
 885             {
 886                 handle_incoming_Z_PDU(m_s2z_search_apdu);
 887             }
 888         }
 889         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
 890         {
 891             m_s2z_search_apdu = 0;
 892             Z_SearchResponse *res = apdu->u.searchResponse;
 893             m_s2z_hit_count = *res->resultCount;
 894             if (res->records && res->records->which == Z_Records_NSD)
 895             {
 896                 send_to_srw_client_ok(0, res->records, 1);
 897             }
 898             else if (m_s2z_present_apdu)
 899             {
 900                 handle_incoming_Z_PDU(m_s2z_present_apdu);
 901             }
 902             else
 903             {
 904                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
 905             }
 906         }
 907         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
 908         {
 909             int start =
 910                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
 911
 912             m_s2z_present_apdu = 0;
 913             Z_PresentResponse *res = apdu->u.presentResponse;
 914             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
 915         }
 916     }
 917     else
 918     {
 919         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 920             yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 921                      apdu_name(apdu));
 922         int r = send_Z_PDU(apdu, len);
 923         logtime();
 924         return r;
 925     }
 926     return 0;
 927 }
 928
 929 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 930 {
 931     int len = 0;
 932     int kill_session = 0;
 933     if (apdu->which == Z_APDU_searchResponse)
 934     {
 935         Z_SearchResponse *sr = apdu->u.searchResponse;
 936         Z_Records *p = sr->records;
 937         if (p && p->which == Z_Records_NSD)
 938         {
 939             Z_DiagRec dr, *dr_p = &dr;
 940             dr.which = Z_DiagRec_defaultFormat;
 941             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 942
 943             *sr->searchStatus = 0;
 944             display_diagrecs(&dr_p, 1);
 945         }
 946         else
 947         {
 948             if (p && p->which == Z_Records_DBOSD)
 949             {
 950                 if (m_marcxml_flag)
 951                     convert_to_marcxml(p->u.databaseOrSurDiagnostics);
 952                 convert_xsl(p->u.databaseOrSurDiagnostics);
 953             }
 954             if (sr->resultCount)
 955             {
 956                 yaz_log(LOG_LOG, "%s%d hits", m_session_str,
 957                         *sr->resultCount);
 958                 if (*sr->resultCount < 0)
 959                 {
 960                     m_invalid_session = 1;
 961                     kill_session = 1;
 962
 963                     *sr->searchStatus = 0;
 964                     sr->records =
 965                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
 966                     *sr->resultCount = 0;
 967                 }
 968             }
 969         }
 970     }
 971     else if (apdu->which == Z_APDU_presentResponse)
 972     {
 973         Z_PresentResponse *sr = apdu->u.presentResponse;
 974         Z_Records *p = sr->records;
 975         if (p && p->which == Z_Records_NSD)
 976         {
 977             Z_DiagRec dr, *dr_p = &dr;
 978             dr.which = Z_DiagRec_defaultFormat;
 979             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 980             if (*sr->presentStatus == Z_PresentStatus_success)
 981                 *sr->presentStatus = Z_PresentStatus_failure;
 982             display_diagrecs(&dr_p, 1);
 983         }
 984         if (p && p->which == Z_Records_DBOSD)
 985         {
 986             if (m_marcxml_flag)
 987                 convert_to_marcxml(p->u.databaseOrSurDiagnostics);
 988             convert_xsl(p->u.databaseOrSurDiagnostics);
 989         }
 990     }
 991     int r = send_PDU_convert(apdu, &len);
 992     if (r)
 993         return r;
 994     m_bytes_sent += len;
 995     m_bw_stat.add_bytes(len);
 996     if (kill_session)
 997     {
 998         delete m_client;
 999         m_client = 0;
1000         m_parent->pre_init();
1001     }
1002     if (m_http_version)
1003     {
1004         if (!m_http_keepalive)
1005         {
1006 #if 1
1007             timeout(1);
1008 #else
1009             shutdown();
1010             return -1;
1011 #endif
1012         }
1013     }
1014     return r;
1015 }
1016
1017 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1018 {
1019     int len = 0;
1020     int r = send_Z_PDU(apdu, &len);
1021     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1022         yaz_log (LOG_LOG, "%sSending %s to %s %d bytes",
1023                  get_session_str(),
1024                  apdu_name(apdu), get_hostname(), len);
1025     m_bytes_sent += len;
1026     return r;
1027 }
1028
1029 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1030 {
1031     if (apdu->which == Z_APDU_presentRequest)
1032     {
1033         Z_PresentRequest *pr = apdu->u.presentRequest;
1034         int toget = *pr->numberOfRecordsRequested;
1035         int start = *pr->resultSetStartPoint;
1036
1037         yaz_log(LOG_LOG, "%sPresent %s %d+%d", m_session_str,
1038                 pr->resultSetId, start, toget);
1039
1040         if (*m_parent->m_optimize == '0')
1041             return apdu;
1042
1043         if (!m_client->m_last_resultSetId)
1044         {
1045             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1046             new_apdu->u.presentResponse->records =
1047                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1048                                                pr->resultSetId);
1049             send_to_client(new_apdu);
1050             return 0;
1051         }
1052         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1053         {
1054             if (start+toget-1 > m_client->m_last_resultCount)
1055             {
1056                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1057                 new_apdu->u.presentResponse->records =
1058                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1059                 send_to_client(new_apdu);
1060                 return 0;
1061             }
1062             Z_NamePlusRecordList *npr;
1063             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1064                                           pr->preferredRecordSyntax,
1065                                           pr->recordComposition))
1066             {
1067                 yaz_log (LOG_LOG, "%sReturned cached records for present request",
1068                          m_session_str);
1069                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1070                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1071
1072                 new_apdu->u.presentResponse->numberOfRecordsReturned
1073                     = odr_intdup(odr_encode(), toget);
1074
1075                 new_apdu->u.presentResponse->records = (Z_Records*)
1076                     odr_malloc(odr_encode(), sizeof(Z_Records));
1077                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1078                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1079                 new_apdu->u.presentResponse->nextResultSetPosition =
1080                     odr_intdup(odr_encode(), start+toget);
1081
1082                 send_to_client(new_apdu);
1083                 return 0;
1084             }
1085         }
1086     }
1087
1088     if (apdu->which != Z_APDU_searchRequest)
1089         return apdu;
1090     Z_SearchRequest *sr = apdu->u.searchRequest;
1091     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1092     Yaz_Z_Databases this_databases;
1093
1094     this_databases.set(sr->num_databaseNames, (const char **)
1095                        sr->databaseNames);
1096
1097     this_query->set_Z_Query(sr->query);
1098
1099     char query_str[120];
1100     this_query->print(query_str, sizeof(query_str)-1);
1101     yaz_log(LOG_LOG, "%sSearch %s", m_session_str, query_str);
1102
1103     if (*m_parent->m_optimize != '0' &&
1104         m_client->m_last_ok && m_client->m_last_query &&
1105         m_client->m_last_query->match(this_query) &&
1106         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1107         m_client->m_last_databases.match(this_databases))
1108     {
1109         delete this_query;
1110         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1111             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1112         {
1113             Z_NamePlusRecordList *npr;
1114             int toget = *sr->mediumSetPresentNumber;
1115             Z_RecordComposition *comp = 0;
1116
1117             if (toget > m_client->m_last_resultCount)
1118                 toget = m_client->m_last_resultCount;
1119
1120             if (sr->mediumSetElementSetNames)
1121             {
1122                 comp = (Z_RecordComposition *)
1123                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1124                 comp->which = Z_RecordComp_simple;
1125                 comp->u.simple = sr->mediumSetElementSetNames;
1126             }
1127
1128             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1129                                           sr->preferredRecordSyntax, comp))
1130             {
1131                 yaz_log (LOG_LOG, "%sReturned cached records for medium set",
1132                          m_session_str);
1133                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1134                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1135                 new_apdu->u.searchResponse->resultCount =
1136                     &m_client->m_last_resultCount;
1137
1138                 new_apdu->u.searchResponse->numberOfRecordsReturned
1139                     = odr_intdup(odr_encode(), toget);
1140
1141                 new_apdu->u.searchResponse->presentStatus =
1142                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1143                 new_apdu->u.searchResponse->records = (Z_Records*)
1144                     odr_malloc(odr_encode(), sizeof(Z_Records));
1145                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1146                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1147                 new_apdu->u.searchResponse->nextResultSetPosition =
1148                     odr_intdup(odr_encode(), toget+1);
1149                 send_to_client(new_apdu);
1150                 return 0;
1151             }
1152             else
1153             {
1154                 // medium Set
1155                 // send present request (medium size)
1156                 yaz_log (LOG_LOG, "%sOptimizing search for medium set",
1157                          m_session_str);
1158
1159                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1160                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1161                 pr->referenceId = sr->referenceId;
1162                 pr->resultSetId = sr->resultSetName;
1163                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1164                 *pr->numberOfRecordsRequested = toget;
1165                 pr->recordComposition = comp;
1166                 m_client->m_sr_transform = 1;
1167                 return new_apdu;
1168             }
1169         }
1170         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1171             m_client->m_last_resultCount <= 0)
1172         {
1173             // large set. Return pseudo-search response immediately
1174             yaz_log (LOG_LOG, "%sOptimizing search for large set",
1175                      m_session_str);
1176             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1177             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1178             new_apdu->u.searchResponse->resultCount =
1179                 &m_client->m_last_resultCount;
1180             send_to_client(new_apdu);
1181             return 0;
1182         }
1183         else
1184         {
1185             Z_NamePlusRecordList *npr;
1186             int toget = m_client->m_last_resultCount;
1187             Z_RecordComposition *comp = 0;
1188             // small set
1189             // send a present request (small set)
1190
1191             if (sr->smallSetElementSetNames)
1192             {
1193                 comp = (Z_RecordComposition *)
1194                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1195                 comp->which = Z_RecordComp_simple;
1196                 comp->u.simple = sr->smallSetElementSetNames;
1197             }
1198
1199             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1200                                           sr->preferredRecordSyntax, comp))
1201             {
1202                 yaz_log (LOG_LOG, "%sReturned cached records for small set",
1203                          m_session_str);
1204                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1205                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1206                 new_apdu->u.searchResponse->resultCount =
1207                     &m_client->m_last_resultCount;
1208
1209                 new_apdu->u.searchResponse->numberOfRecordsReturned
1210                     = odr_intdup(odr_encode(), toget);
1211
1212                 new_apdu->u.searchResponse->presentStatus =
1213                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1214                 new_apdu->u.searchResponse->records = (Z_Records*)
1215                     odr_malloc(odr_encode(), sizeof(Z_Records));
1216                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1217                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1218                 new_apdu->u.searchResponse->nextResultSetPosition =
1219                     odr_intdup(odr_encode(), toget+1);
1220                 send_to_client(new_apdu);
1221                 return 0;
1222             }
1223             else
1224             {
1225                 yaz_log (LOG_LOG, "%sOptimizing search for small set",
1226                          m_session_str);
1227                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1228                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1229                 pr->referenceId = sr->referenceId;
1230                 pr->resultSetId = sr->resultSetName;
1231                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1232                 *pr->numberOfRecordsRequested = toget;
1233                 pr->recordComposition = comp;
1234                 m_client->m_sr_transform = 1;
1235                 return new_apdu;
1236             }
1237         }
1238     }
1239     else  // query doesn't match
1240     {
1241         delete m_client->m_last_query;
1242         m_client->m_last_query = this_query;
1243         m_client->m_last_ok = 0;
1244         m_client->m_cache.clear();
1245         m_client->m_resultSetStartPoint = 0;
1246
1247         xfree (m_client->m_last_resultSetId);
1248         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1249
1250         m_client->m_last_databases.set(sr->num_databaseNames,
1251                                        (const char **) sr->databaseNames);
1252     }
1253     return apdu;
1254 }
1255
1256
1257 void Yaz_Proxy::inc_request_no()
1258 {
1259     char *cp = strchr(m_session_str, ' ');
1260     m_request_no++;
1261     if (cp)
1262         sprintf(cp+1, "%d ", m_request_no);
1263 }
1264
1265 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1266 {
1267     inc_request_no();
1268
1269     m_bytes_recv += len;
1270
1271     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
1272         yaz_log (LOG_DEBUG, "%sReceiving %s from client %d bytes",
1273                  m_session_str, gdu_name(apdu), len);
1274
1275     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1276         shutdown();
1277
1278     m_bw_stat.add_bytes(len);
1279     m_pdu_stat.add_bytes(1);
1280
1281     gettimeofday(&m_time_tv, 0);
1282
1283     int bw_total = m_bw_stat.get_total();
1284     int pdu_total = m_pdu_stat.get_total();
1285
1286     int reduce = 0;
1287     if (m_bw_max)
1288     {
1289         if (bw_total > m_bw_max)
1290         {
1291             reduce = (bw_total/m_bw_max);
1292         }
1293     }
1294     if (m_pdu_max)
1295     {
1296         if (pdu_total > m_pdu_max)
1297         {
1298             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1299             reduce = (reduce > nreduce) ? reduce : nreduce;
1300         }
1301     }
1302     if (reduce)
1303     {
1304         yaz_log(LOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1305                 m_session_str, reduce, bw_total, pdu_total,
1306                 m_bw_max, m_pdu_max);
1307
1308         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1309         timeout(reduce);       // call us reduce seconds later
1310     }
1311     else if (apdu->which == Z_GDU_Z3950)
1312         handle_incoming_Z_PDU(apdu->u.z3950);
1313     else if (apdu->which == Z_GDU_HTTP_Request)
1314         handle_incoming_HTTP(apdu->u.HTTP_Request);
1315 }
1316
1317 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1318 {
1319     if (m_max_record_retrieve)
1320     {
1321         if (apdu->which == Z_APDU_presentRequest)
1322         {
1323             Z_PresentRequest *pr = apdu->u.presentRequest;
1324             if (pr->numberOfRecordsRequested &&
1325                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1326                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1327         }
1328     }
1329 }
1330
1331 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1332                                                      int error,
1333                                                      const char *addinfo)
1334 {
1335     Z_Records *rec = (Z_Records *)
1336         odr_malloc (odr, sizeof(*rec));
1337     int *err = (int *)
1338         odr_malloc (odr, sizeof(*err));
1339     Z_DiagRec *drec = (Z_DiagRec *)
1340         odr_malloc (odr, sizeof(*drec));
1341     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1342         odr_malloc (odr, sizeof(*dr));
1343     *err = error;
1344     rec->which = Z_Records_NSD;
1345     rec->u.nonSurrogateDiagnostic = dr;
1346     dr->diagnosticSetId =
1347         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1348     dr->condition = err;
1349     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1350     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1351     return rec;
1352 }
1353
1354 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1355 {
1356     if (apdu->which == Z_APDU_searchRequest &&
1357         apdu->u.searchRequest->query &&
1358         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1359         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1360     {
1361         Z_RPNQuery *rpnquery = 0;
1362         Z_SearchRequest *sr = apdu->u.searchRequest;
1363
1364         yaz_log(LOG_LOG, "%sCQL: %s", m_session_str,
1365                 sr->query->u.type_104->u.cql);
1366
1367         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1368                                           &rpnquery, odr_encode());
1369         if (r == -3)
1370             yaz_log(LOG_LOG, "%sNo CQL to RPN table", m_session_str);
1371         else if (r)
1372             yaz_log(LOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1373         else
1374         {
1375             sr->query->which = Z_Query_type_1;
1376             sr->query->u.type_1 = rpnquery;
1377         }
1378         return apdu;
1379     }
1380     return apdu;
1381 }
1382
1383 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1384 {
1385     if (apdu->which == Z_APDU_searchRequest)
1386     {
1387         Z_SearchRequest *sr = apdu->u.searchRequest;
1388         int err = 0;
1389         char *addinfo = 0;
1390
1391         Yaz_ProxyConfig *cfg = check_reconfigure();
1392         if (cfg)
1393             err = cfg->check_query(odr_encode(), m_default_target,
1394                                    sr->query, &addinfo);
1395         if (err)
1396         {
1397             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1398
1399             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1400             new_apdu->u.searchResponse->records =
1401                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1402             *new_apdu->u.searchResponse->searchStatus = 0;
1403
1404             send_to_client(new_apdu);
1405
1406             return 0;
1407         }
1408     }
1409     return apdu;
1410 }
1411
1412 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1413 {
1414     m_marcxml_flag = 0;
1415     if (apdu->which == Z_APDU_searchRequest)
1416     {
1417         Z_SearchRequest *sr = apdu->u.searchRequest;
1418         int err = 0;
1419         char *addinfo = 0;
1420         Yaz_ProxyConfig *cfg = check_reconfigure();
1421
1422         Z_RecordComposition rc_temp, *rc = 0;
1423         if (sr->smallSetElementSetNames)
1424         {
1425             rc_temp.which = Z_RecordComp_simple;
1426             rc_temp.u.simple = sr->smallSetElementSetNames;
1427             rc = &rc_temp;
1428         }
1429
1430         if (cfg)
1431             err = cfg->check_syntax(odr_encode(),
1432                                     m_default_target,
1433                                     sr->preferredRecordSyntax, rc,
1434                                     &addinfo, &m_stylesheet);
1435         if (err == -1)
1436         {
1437             sr->preferredRecordSyntax =
1438                 yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN, VAL_USMARC);
1439             m_marcxml_flag = 1;
1440         }
1441         else if (err)
1442         {
1443             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1444
1445             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1446             new_apdu->u.searchResponse->records =
1447                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1448             *new_apdu->u.searchResponse->searchStatus = 0;
1449
1450             send_to_client(new_apdu);
1451
1452             return 0;
1453         }
1454     }
1455     else if (apdu->which == Z_APDU_presentRequest)
1456     {
1457         Z_PresentRequest *pr = apdu->u.presentRequest;
1458         int err = 0;
1459         char *addinfo = 0;
1460         Yaz_ProxyConfig *cfg = check_reconfigure();
1461
1462         if (cfg)
1463             err = cfg->check_syntax(odr_encode(), m_default_target,
1464                                     pr->preferredRecordSyntax,
1465                                     pr->recordComposition,
1466                                     &addinfo, &m_stylesheet);
1467         if (err == -1)
1468         {
1469             pr->preferredRecordSyntax =
1470                 yaz_oidval_to_z3950oid(odr_decode(), CLASS_RECSYN, VAL_USMARC);
1471             m_marcxml_flag = 1;
1472         }
1473         else if (err)
1474         {
1475             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1476
1477             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1478             new_apdu->u.presentResponse->records =
1479                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1480             *new_apdu->u.presentResponse->presentStatus =
1481                 Z_PresentStatus_failure;
1482
1483             send_to_client(new_apdu);
1484
1485             return 0;
1486         }
1487     }
1488     return apdu;
1489 }
1490
1491 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
1492 {
1493     if (!schema)
1494         return 0;
1495     Z_ElementSetNames *esn = (Z_ElementSetNames *)
1496         odr_malloc(o, sizeof(Z_ElementSetNames));
1497     esn->which = Z_ElementSetNames_generic;
1498     esn->u.generic = odr_strdup(o, schema);
1499     return esn;
1500 }
1501
1502 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1503 {
1504
1505     if (m_s2z_odr_init)
1506     {
1507         odr_destroy(m_s2z_odr_init);
1508         m_s2z_odr_init = 0;
1509     }
1510     if (m_s2z_odr_search)
1511     {
1512         odr_destroy(m_s2z_odr_search);
1513         m_s2z_odr_search = 0;
1514     }
1515
1516     m_http_keepalive = 0;
1517     m_http_version = 0;
1518     if (!strcmp(hreq->version, "1.0"))
1519     {
1520         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1521         if (v && !strcmp(v, "Keep-Alive"))
1522             m_http_keepalive = 1;
1523         else
1524             m_http_keepalive = 0;
1525         m_http_version = "1.0";
1526     }
1527     else
1528     {
1529         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1530         if (v && !strcmp(v, "close"))
1531             m_http_keepalive = 0;
1532         else
1533             m_http_keepalive = 1;
1534         m_http_version = "1.1";
1535     }
1536
1537     Z_SRW_PDU *srw_pdu = 0;
1538     Z_SOAP *soap_package = 0;
1539     char *charset = 0;
1540     if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1541                        &charset) == 0
1542         || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1543                           &charset) == 0)
1544     {
1545         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
1546         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
1547         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
1548         m_s2z_init_apdu = 0;
1549         m_s2z_search_apdu = 0;
1550         m_s2z_present_apdu = 0;
1551         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
1552         {
1553             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
1554
1555             // set packing for response records ..
1556             if (srw_req->recordPacking &&
1557                 !strcmp(srw_req->recordPacking, "xml"))
1558                 m_s2z_packing = Z_SRW_recordPacking_XML;
1559             else
1560                 m_s2z_packing = Z_SRW_recordPacking_string;
1561
1562             // prepare search PDU
1563             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
1564                                           Z_APDU_searchRequest);
1565             Z_SearchRequest *z_searchRequest =
1566                 m_s2z_search_apdu->u.searchRequest;
1567
1568             z_searchRequest->num_databaseNames = 1;
1569             z_searchRequest->databaseNames = (char**)
1570                 odr_malloc(m_s2z_odr_search, sizeof(char *));
1571             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
1572                                                            srw_req->database);
1573
1574             // query transformation
1575             Z_Query *query = (Z_Query *)
1576                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
1577             z_searchRequest->query = query;
1578
1579             if (srw_req->query_type == Z_SRW_query_type_cql)
1580             {
1581                 Z_External *ext = (Z_External *)
1582                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
1583                 ext->direct_reference =
1584                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
1585                 ext->indirect_reference = 0;
1586                 ext->descriptor = 0;
1587                 ext->which = Z_External_CQL;
1588                 ext->u.cql = srw_req->query.cql;
1589
1590                 query->which = Z_Query_type_104;
1591                 query->u.type_104 =  ext;
1592             }
1593             else if (srw_req->query_type == Z_SRW_query_type_pqf)
1594             {
1595                 Z_RPNQuery *RPNquery;
1596                 YAZ_PQF_Parser pqf_parser;
1597
1598                 pqf_parser = yaz_pqf_create ();
1599
1600                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
1601                                           srw_req->query.pqf);
1602                 if (!RPNquery)
1603                 {
1604                     const char *pqf_msg;
1605                     size_t off;
1606                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
1607                     yaz_log(LOG_LOG, "%*s^\n", off+4, "");
1608                     yaz_log(LOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
1609
1610                     send_to_srw_client_error(10);
1611                     return;
1612                 }
1613                 query->which = Z_Query_type_1;
1614                 query->u.type_1 =  RPNquery;
1615
1616                 yaz_pqf_destroy (pqf_parser);
1617             }
1618             else
1619             {
1620                 send_to_srw_client_error(11);
1621                 return;
1622             }
1623
1624             // present
1625             m_s2z_present_apdu = 0;
1626             int max = 0;
1627             if (srw_req->maximumRecords)
1628                 max = *srw_req->maximumRecords;
1629             int start = 1;
1630             if (srw_req->startRecord)
1631                 start = *srw_req->startRecord;
1632             if (max > 0)
1633             {
1634                 if (start <= 1)  // Z39.50 piggyback
1635                 {
1636                     *z_searchRequest->smallSetUpperBound = max;
1637                     *z_searchRequest->mediumSetPresentNumber = max;
1638                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
1639
1640                     z_searchRequest->preferredRecordSyntax =
1641                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1642                                                VAL_TEXT_XML);
1643                     if (srw_req->recordSchema)
1644                     {
1645                         z_searchRequest->smallSetElementSetNames =
1646                             z_searchRequest->mediumSetElementSetNames =
1647                             mk_esn_from_schema(m_s2z_odr_search,
1648                                                srw_req->recordSchema);
1649                     }
1650                 }
1651                 else   // Z39.50 present
1652                 {
1653                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
1654                                                    Z_APDU_presentRequest);
1655                     Z_PresentRequest *z_presentRequest =
1656                         m_s2z_present_apdu->u.presentRequest;
1657                     *z_presentRequest->resultSetStartPoint = start;
1658                     *z_presentRequest->numberOfRecordsRequested = max;
1659                     z_presentRequest->preferredRecordSyntax =
1660                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1661                                                VAL_TEXT_XML);
1662                     z_presentRequest->recordComposition =
1663                         (Z_RecordComposition *)
1664                         odr_malloc(m_s2z_odr_search,
1665                                    sizeof(Z_RecordComposition));
1666                     if (srw_req->recordSchema)
1667                     {
1668                         z_presentRequest->recordComposition->which =
1669                             Z_RecordComp_simple;
1670                         z_presentRequest->recordComposition->u.simple =
1671                             mk_esn_from_schema(m_s2z_odr_search,
1672                                                srw_req->recordSchema);
1673                     }
1674                 }
1675             }
1676             if (!m_client)
1677             {
1678                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1679                                             Z_APDU_initRequest);
1680
1681                 // prevent m_initRequest_apdu memory from being grabbed
1682                 // in Yaz_Proxy::handle_incoming_Z_PDU
1683                 m_initRequest_apdu = m_s2z_init_apdu;
1684                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1685                 return;
1686             }
1687             else
1688             {
1689                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1690                 return;
1691             }
1692         }
1693         else if (srw_pdu->which == Z_SRW_explain_request)
1694         {
1695             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
1696
1697             if (srw_req->recordPacking &&
1698                 !strcmp(srw_req->recordPacking, "xml"))
1699                 m_s2z_packing = Z_SRW_recordPacking_XML;
1700             else
1701                 m_s2z_packing = Z_SRW_recordPacking_string;
1702
1703             if (!m_client)
1704             {
1705                 yaz_log(LOG_LOG, "handle_incoming: initRequest");
1706                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1707                                             Z_APDU_initRequest);
1708
1709                 // prevent m_initRequest_apdu memory from being grabbed
1710                 // in Yaz_Proxy::handle_incoming_Z_PDU
1711                 m_initRequest_apdu = m_s2z_init_apdu;
1712                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1713             }
1714             else
1715                 send_srw_explain();
1716             return;
1717         }
1718     }
1719     int len = 0;
1720     Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
1721     send_GDU(p, &len);
1722     timeout(1);
1723 }
1724
1725 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
1726 {
1727     if (!m_client && m_invalid_session)
1728     {
1729         m_apdu_invalid_session = apdu;
1730         m_mem_invalid_session = odr_extract_mem(odr_decode());
1731         apdu = m_initRequest_apdu;
1732     }
1733
1734     // Determine our client.
1735     Z_OtherInformation **oi;
1736     get_otherInfoAPDU(apdu, &oi);
1737     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
1738     if (!m_client)
1739     {
1740         delete this;
1741         return;
1742     }
1743     m_client->m_server = this;
1744
1745     if (apdu->which == Z_APDU_initRequest)
1746     {
1747         if (apdu->u.initRequest->implementationId)
1748             yaz_log(LOG_LOG, "%simplementationId: %s",
1749                     m_session_str, apdu->u.initRequest->implementationId);
1750         if (apdu->u.initRequest->implementationName)
1751             yaz_log(LOG_LOG, "%simplementationName: %s",
1752                     m_session_str, apdu->u.initRequest->implementationName);
1753         if (apdu->u.initRequest->implementationVersion)
1754             yaz_log(LOG_LOG, "%simplementationVersion: %s",
1755                     m_session_str, apdu->u.initRequest->implementationVersion);
1756         if (m_initRequest_apdu == 0)
1757         {
1758             if (m_initRequest_mem)
1759                 nmem_destroy(m_initRequest_mem);
1760             m_initRequest_apdu = apdu;
1761             m_initRequest_mem = odr_extract_mem(odr_decode());
1762         }
1763         if (m_client->m_init_flag)
1764         {
1765             if (handle_init_response_for_invalid_session(apdu))
1766                 return;
1767             Z_APDU *apdu2 = m_client->m_initResponse;
1768             apdu2->u.initResponse->otherInfo = 0;
1769             if (m_client->m_cookie && *m_client->m_cookie)
1770                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
1771                                            m_client->m_cookie);
1772             apdu2->u.initResponse->referenceId =
1773                 apdu->u.initRequest->referenceId;
1774             send_to_client(apdu2);
1775             return;
1776         }
1777         m_client->m_init_flag = 1;
1778     }
1779     handle_max_record_retrieve(apdu);
1780
1781     if (apdu)
1782         apdu = handle_syntax_validation(apdu);
1783
1784     if (apdu)
1785         apdu = handle_query_transformation(apdu);
1786
1787     if (apdu)
1788         apdu = handle_query_validation(apdu);
1789
1790     if (apdu)
1791         apdu = result_set_optimize(apdu);
1792     if (!apdu)
1793     {
1794         m_client->timeout(m_target_idletime);  // mark it active even
1795         // though we didn't use it
1796         return;
1797     }
1798
1799     // delete other info part from PDU before sending to target
1800     get_otherInfoAPDU(apdu, &oi);
1801     if (oi)
1802         *oi = 0;
1803
1804     if (apdu->which == Z_APDU_presentRequest &&
1805         m_client->m_resultSetStartPoint == 0)
1806     {
1807         Z_PresentRequest *pr = apdu->u.presentRequest;
1808         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
1809         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
1810     } else {
1811         m_client->m_resultSetStartPoint = 0;
1812     }
1813     if (m_client->send_to_target(apdu) < 0)
1814     {
1815         delete m_client;
1816         m_client = 0;
1817         delete this;
1818     }
1819     else
1820         m_client->m_waiting = 1;
1821 }
1822
1823 void Yaz_Proxy::connectNotify()
1824 {
1825 }
1826
1827 void Yaz_Proxy::shutdown()
1828 {
1829     m_invalid_session = 0;
1830     // only keep if keep_alive flag is set...
1831     if (m_client &&
1832         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
1833         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
1834         m_client->m_waiting == 0)
1835     {
1836         yaz_log(LOG_LOG, "%sShutdown (client to proxy) keepalive %s",
1837                  m_session_str,
1838                  m_client->get_hostname());
1839         yaz_log(LOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1840                 m_session_str, m_client->m_pdu_recv,
1841                 m_client->m_bytes_sent + m_client->m_bytes_recv,
1842                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
1843         assert (m_client->m_waiting != 2);
1844         // Tell client (if any) that no server connection is there..
1845         m_client->m_server = 0;
1846         m_invalid_session = 0;
1847     }
1848     else if (m_client)
1849     {
1850         yaz_log (LOG_LOG, "%sShutdown (client to proxy) close %s",
1851                  m_session_str,
1852                  m_client->get_hostname());
1853         assert (m_client->m_waiting != 2);
1854         delete m_client;
1855     }
1856     else if (!m_parent)
1857     {
1858         yaz_log (LOG_LOG, "%sshutdown (client to proxy) bad state",
1859                  m_session_str);
1860         assert (m_parent);
1861     }
1862     else
1863     {
1864         yaz_log (LOG_LOG, "%sShutdown (client to proxy)",
1865                  m_session_str);
1866     }
1867     if (m_parent)
1868         m_parent->pre_init();
1869     delete this;
1870 }
1871
1872 const char *Yaz_ProxyClient::get_session_str()
1873 {
1874     if (!m_server)
1875         return "0 ";
1876     return m_server->get_session_str();
1877 }
1878
1879 void Yaz_ProxyClient::shutdown()
1880 {
1881     yaz_log (LOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
1882              get_hostname());
1883     delete m_server;
1884     delete this;
1885 }
1886
1887 void Yaz_Proxy::failNotify()
1888 {
1889     inc_request_no();
1890     yaz_log (LOG_LOG, "%sConnection closed by client",
1891              get_session_str());
1892     shutdown();
1893 }
1894
1895 void Yaz_ProxyClient::failNotify()
1896 {
1897     if (m_server)
1898         m_server->inc_request_no();
1899     yaz_log (LOG_LOG, "%sConnection closed by target %s",
1900              get_session_str(), get_hostname());
1901     shutdown();
1902 }
1903
1904 void Yaz_ProxyClient::connectNotify()
1905 {
1906     const char *s = get_session_str();
1907     const char *h = get_hostname();
1908     yaz_log (LOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
1909              m_target_idletime);
1910     timeout(m_target_idletime);
1911     if (!m_server)
1912         pre_init_client();
1913 }
1914
1915 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
1916                                                   *the_PDU_Observable, int fd)
1917 {
1918     return new Yaz_ProxyClient(the_PDU_Observable, 0);
1919 }
1920
1921 Yaz_ProxyClient::~Yaz_ProxyClient()
1922 {
1923     if (m_prev)
1924         *m_prev = m_next;
1925     if (m_next)
1926         m_next->m_prev = m_prev;
1927     m_waiting = 2;     // for debugging purposes only.
1928     odr_destroy(m_init_odr);
1929     delete m_last_query;
1930     xfree (m_last_resultSetId);
1931     xfree (m_cookie);
1932 }
1933
1934 void Yaz_ProxyClient::pre_init_client()
1935 {
1936     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
1937     Z_InitRequest *req = apdu->u.initRequest;
1938
1939     ODR_MASK_SET(req->options, Z_Options_search);
1940     ODR_MASK_SET(req->options, Z_Options_present);
1941     ODR_MASK_SET(req->options, Z_Options_namedResultSets);
1942     ODR_MASK_SET(req->options, Z_Options_triggerResourceCtrl);
1943     ODR_MASK_SET(req->options, Z_Options_scan);
1944     ODR_MASK_SET(req->options, Z_Options_sort);
1945     ODR_MASK_SET(req->options, Z_Options_extendedServices);
1946     ODR_MASK_SET(req->options, Z_Options_delSet);
1947
1948     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
1949     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
1950     ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3);
1951
1952     if (send_to_target(apdu) < 0)
1953     {
1954         delete this;
1955     }
1956     else
1957     {
1958         m_waiting = 1;
1959         m_init_flag = 1;
1960     }
1961 }
1962
1963 void Yaz_Proxy::pre_init()
1964 {
1965     int i;
1966     const char *name = 0;
1967     const char *zurl_in_use[MAX_ZURL_PLEX];
1968     int limit_bw, limit_pdu, limit_req;
1969     int target_idletime, client_idletime;
1970     int max_clients;
1971     int keepalive_limit_bw, keepalive_limit_pdu;
1972     int pre_init;
1973     const char *cql2rpn = 0;
1974     const char *zeerex = 0;
1975
1976     Yaz_ProxyConfig *cfg = check_reconfigure();
1977
1978     zurl_in_use[0] = 0;
1979
1980     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
1981         set_APDU_yazlog(1);
1982     else
1983         set_APDU_yazlog(0);
1984
1985     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
1986                                           &limit_bw, &limit_pdu, &limit_req,
1987                                           &target_idletime, &client_idletime,
1988                                           &max_clients,
1989                                           &keepalive_limit_bw,
1990                                           &keepalive_limit_pdu,
1991                                           &pre_init,
1992                                           &cql2rpn) ; i++)
1993     {
1994         if (pre_init)
1995         {
1996             int j;
1997             for (j = 0; zurl_in_use[j]; j++)
1998             {
1999                 Yaz_ProxyClient *c;
2000                 int spare = 0;
2001                 int in_use = 0;
2002                 int other = 0;
2003                 for (c = m_clientPool; c; c = c->m_next)
2004                 {
2005                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
2006                     {
2007                         if (c->m_cookie == 0)
2008                         {
2009                             if (c->m_server == 0)
2010                                 spare++;
2011                             else
2012                                 in_use++;
2013                         }
2014                         else
2015                             other++;
2016                     }
2017                 }
2018                 yaz_log(LOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2019                         "preinit=%d",m_session_str,
2020                         name, zurl_in_use[j], in_use, other, spare, pre_init);
2021                 if (spare < pre_init)
2022                 {
2023                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2024                     c->m_next = m_clientPool;
2025                     if (c->m_next)
2026                         c->m_next->m_prev = &c->m_next;
2027                     m_clientPool = c;
2028                     c->m_prev = &m_clientPool;
2029
2030                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
2031                         c->set_APDU_yazlog(1);
2032                     else
2033                         c->set_APDU_yazlog(0);
2034
2035                     if (c->client(zurl_in_use[j]))
2036                     {
2037                         timeout(60);
2038                         delete c;
2039                         return;
2040                     }
2041                     c->timeout(30);
2042                     c->m_waiting = 1;
2043                     c->m_target_idletime = target_idletime;
2044                     c->m_seqno = m_seqno++;
2045                 }
2046             }
2047         }
2048     }
2049 }
2050
2051 void Yaz_Proxy::timeoutNotify()
2052 {
2053     if (m_parent)
2054     {
2055         if (m_bw_hold_PDU)
2056         {
2057             timeout(m_client_idletime);
2058             Z_GDU *apdu = m_bw_hold_PDU;
2059             m_bw_hold_PDU = 0;
2060
2061             if (apdu->which == Z_GDU_Z3950)
2062                 handle_incoming_Z_PDU(apdu->u.z3950);
2063             else if (apdu->which == Z_GDU_HTTP_Request)
2064                 handle_incoming_HTTP(apdu->u.HTTP_Request);
2065         }
2066         else
2067         {
2068             inc_request_no();
2069
2070             yaz_log (LOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2071             shutdown();
2072         }
2073     }
2074     else
2075     {
2076         timeout(600);
2077         pre_init();
2078     }
2079 }
2080
2081 void Yaz_ProxyClient::timeoutNotify()
2082 {
2083     if (m_server)
2084         m_server->inc_request_no();
2085
2086     yaz_log (LOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2087              get_hostname());
2088     m_waiting = 1;
2089     m_root->pre_init();
2090     shutdown();
2091 }
2092
2093 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2094                                  Yaz_Proxy *parent) :
2095     Yaz_Z_Assoc (the_PDU_Observable)
2096 {
2097     m_cookie = 0;
2098     m_next = 0;
2099     m_prev = 0;
2100     m_init_flag = 0;
2101     m_last_query = 0;
2102     m_last_resultSetId = 0;
2103     m_last_resultCount = 0;
2104     m_last_ok = 0;
2105     m_sr_transform = 0;
2106     m_waiting = 0;
2107     m_init_odr = odr_createmem (ODR_DECODE);
2108     m_initResponse = 0;
2109     m_resultSetStartPoint = 0;
2110     m_bytes_sent = m_bytes_recv = 0;
2111     m_pdu_recv = 0;
2112     m_server = 0;
2113     m_seqno = 0;
2114     m_target_idletime = 600;
2115     m_root = parent;
2116 }
2117
2118 const char *Yaz_Proxy::option(const char *name, const char *value)
2119 {
2120     if (!strcmp (name, "optimize")) {
2121         if (value) {
2122             xfree (m_optimize);
2123             m_optimize = xstrdup (value);
2124         }
2125         return m_optimize;
2126     }
2127     return 0;
2128 }
2129
2130 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2131 {
2132
2133 }
2134
2135 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2136 {
2137     if (apdu->which == Z_GDU_Z3950)
2138         recv_Z_PDU(apdu->u.z3950, len);
2139     else if (apdu->which == Z_GDU_HTTP_Response)
2140         recv_HTTP_response(apdu->u.HTTP_Response, len);
2141     else
2142         shutdown();
2143 }
2144
2145 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2146 {
2147     if (!m_invalid_session)
2148         return 0;
2149     m_invalid_session = 0;
2150     handle_incoming_Z_PDU(m_apdu_invalid_session);
2151     assert (m_mem_invalid_session);
2152     nmem_destroy(m_mem_invalid_session);
2153     m_mem_invalid_session = 0;
2154     return 1;
2155 }
2156
2157 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2158 {
2159     m_bytes_recv += len;
2160     m_pdu_recv++;
2161     m_waiting = 0;
2162     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2163         yaz_log (LOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2164                  apdu_name(apdu), get_hostname(), len);
2165     if (apdu->which == Z_APDU_initResponse)
2166     {
2167         if (!m_server)  // if this is a pre init session , check for more
2168             m_root->pre_init();
2169         NMEM nmem = odr_extract_mem (odr_decode());
2170         odr_reset (m_init_odr);
2171         nmem_transfer (m_init_odr->mem, nmem);
2172         m_initResponse = apdu;
2173
2174         Z_InitResponse *ir = apdu->u.initResponse;
2175         char *im0 = ir->implementationName;
2176
2177         char *im1 = (char*)
2178             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2179         *im1 = '\0';
2180         if (im0)
2181         {
2182             strcat(im1, im0);
2183             strcat(im1, " ");
2184         }
2185         strcat(im1, "(YAZ Proxy)");
2186         ir->implementationName = im1;
2187
2188         nmem_destroy (nmem);
2189
2190         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2191             return;
2192     }
2193     if (apdu->which == Z_APDU_searchResponse)
2194     {
2195         Z_SearchResponse *sr = apdu->u.searchResponse;
2196         m_last_resultCount = *sr->resultCount;
2197         int status = *sr->searchStatus;
2198         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2199         {
2200             m_last_ok = 1;
2201
2202             if (sr->records && sr->records->which == Z_Records_DBOSD)
2203             {
2204                 m_cache.add(odr_decode(),
2205                             sr->records->u.databaseOrSurDiagnostics, 1,
2206                             *sr->resultCount);
2207             }
2208         }
2209     }
2210     if (apdu->which == Z_APDU_presentResponse)
2211     {
2212         Z_PresentResponse *pr = apdu->u.presentResponse;
2213         if (m_sr_transform)
2214         {
2215             m_sr_transform = 0;
2216             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2217             Z_SearchResponse *sr = new_apdu->u.searchResponse;
2218             sr->referenceId = pr->referenceId;
2219             *sr->resultCount = m_last_resultCount;
2220             sr->records = pr->records;
2221             sr->nextResultSetPosition = pr->nextResultSetPosition;
2222             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2223             apdu = new_apdu;
2224         }
2225         if (pr->records &&
2226             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2227         {
2228             m_cache.add(odr_decode(),
2229                         pr->records->u.databaseOrSurDiagnostics,
2230                         m_resultSetStartPoint, -1);
2231             m_resultSetStartPoint = 0;
2232         }
2233     }
2234     if (m_cookie)
2235         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2236     if (m_server)
2237     {
2238         m_server->send_to_client(apdu);
2239     }
2240     if (apdu->which == Z_APDU_close)
2241     {
2242         shutdown();
2243     }
2244 }
2245
2246 int Yaz_Proxy::server(const char *addr)
2247 {
2248     int r = Yaz_Z_Assoc::server(addr);
2249     if (!r)
2250     {
2251         yaz_log(LOG_LOG, "%sStarted listener on %s", m_session_str, addr);
2252         timeout(1);
2253     }
2254     return r;
2255 }
2256