src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.12 2004-11-30 21:10:45 adam Exp $
   2    Copyright (c) 1998-2004, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #else
  24 #include <unistd.h>
  25 #endif
  26
  27 #if HAVE_GETTIMEOFDAY
  28 #include <sys/time.h>
  29 #endif
  30
  31 #include <assert.h>
  32 #include <time.h>
  33 #include <sys/types.h>
  34 #include <fcntl.h>
  35
  36 #include <yaz/srw.h>
  37 #include <yaz/marcdisp.h>
  38 #include <yaz/yaz-iconv.h>
  39 #include <yaz/ylog.h>
  40 #include <yaz/diagbib1.h>
  41 #include <yazproxy/proxy.h>
  42 #include <yaz/pquery.h>
  43 #include <yaz/otherinfo.h>
  44
  45 #if HAVE_XSLT
  46 #include <libxml/parser.h>
  47 #include <libxml/tree.h>
  48 #include <libxslt/xsltutils.h>
  49 #include <libxslt/transform.h>
  50 #endif
  51
  52 static const char *apdu_name(Z_APDU *apdu)
  53 {
  54     switch (apdu->which)
  55     {
  56     case Z_APDU_initRequest:
  57         return "initRequest";
  58     case Z_APDU_initResponse:
  59         return "initResponse";
  60     case Z_APDU_searchRequest:
  61         return "searchRequest";
  62     case Z_APDU_searchResponse:
  63         return "searchResponse";
  64     case Z_APDU_presentRequest:
  65         return "presentRequest";
  66     case Z_APDU_presentResponse:
  67         return "presentResponse";
  68     case Z_APDU_deleteResultSetRequest:
  69         return "deleteResultSetRequest";
  70     case Z_APDU_deleteResultSetResponse:
  71         return "deleteResultSetResponse";
  72     case Z_APDU_scanRequest:
  73         return "scanRequest";
  74     case Z_APDU_scanResponse:
  75         return "scanResponse";
  76     case Z_APDU_sortRequest:
  77         return "sortRequest";
  78     case Z_APDU_sortResponse:
  79         return "sortResponse";
  80     case Z_APDU_extendedServicesRequest:
  81         return "extendedServicesRequest";
  82     case Z_APDU_extendedServicesResponse:
  83         return "extendedServicesResponse";
  84     case Z_APDU_close:
  85         return "close";
  86     }
  87     return "other";
  88 }
  89
  90 static const char *gdu_name(Z_GDU *gdu)
  91 {
  92     switch(gdu->which)
  93     {
  94     case Z_GDU_Z3950:
  95         return apdu_name(gdu->u.z3950);
  96     case Z_GDU_HTTP_Request:
  97         return "HTTP Request";
  98     case Z_GDU_HTTP_Response:
  99         return "HTTP Response";
 100     }
 101     return "Unknown request/response";
 102 }
 103
 104 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
 105                      Yaz_Proxy *parent) :
 106     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 107 {
 108     m_PDU_Observable = the_PDU_Observable;
 109     m_client = 0;
 110     m_parent = parent;
 111     m_clientPool = 0;
 112     m_seqno = 1;
 113     m_keepalive_limit_bw = 500000;
 114     m_keepalive_limit_pdu = 1000;
 115     m_proxyTarget = 0;
 116     m_default_target = 0;
 117     m_proxy_authentication = 0;
 118     m_max_clients = 150;
 119     m_log_mask = 0;
 120     m_seed = time(0);
 121     m_client_idletime = 600;
 122     m_target_idletime = 600;
 123     m_optimize = xstrdup ("1");
 124     strcpy(m_session_str, "0 ");
 125     m_session_no=0;
 126     m_bytes_sent = 0;
 127     m_bytes_recv = 0;
 128     m_bw_hold_PDU = 0;
 129     m_bw_max = 0;
 130     m_pdu_max = 0;
 131     m_max_record_retrieve = 0;
 132     m_reconfig_flag = 0;
 133     m_config_fname = 0;
 134     m_request_no = 0;
 135     m_invalid_session = 0;
 136     m_referenceId = 0;
 137     m_referenceId_mem = nmem_create();
 138     m_config = 0;
 139     m_marcxml_flag = 0;
 140     m_stylesheet_xsp = 0;
 141     m_stylesheet_nprl = 0;
 142     m_s2z_stylesheet = 0;
 143     m_s2z_database = 0;
 144     m_schema = 0;
 145     m_backend_type = 0;
 146     m_backend_charset = 0;
 147     m_frontend_type = 0;
 148     m_initRequest_apdu = 0;
 149     m_initRequest_mem = 0;
 150     m_initRequest_preferredMessageSize = 0;
 151     m_initRequest_maximumRecordSize = 0;
 152     m_initRequest_options = 0;
 153     m_initRequest_version = 0;
 154     m_apdu_invalid_session = 0;
 155     m_mem_invalid_session = 0;
 156     m_s2z_odr_init = 0;
 157     m_s2z_odr_search = 0;
 158     m_s2z_init_apdu = 0;
 159     m_s2z_search_apdu = 0;
 160     m_s2z_present_apdu = 0;
 161     m_http_keepalive = 0;
 162     m_http_version = 0;
 163     m_soap_ns = 0;
 164     m_s2z_packing = Z_SRW_recordPacking_string;
 165 #if HAVE_GETTIMEOFDAY
 166     m_time_tv = xmalloc(sizeof(struct timeval));
 167     struct timeval *tv = (struct timeval *) m_time_tv;
 168     tv->tv_sec = 0;
 169     tv->tv_usec = 0;
 170 #else
 171     m_time_tv = 0;
 172 #endif
 173     if (!m_parent)
 174         low_socket_open();
 175 }
 176
 177 Yaz_Proxy::~Yaz_Proxy()
 178 {
 179     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 180             m_bytes_sent, m_bytes_recv);
 181     nmem_destroy(m_initRequest_mem);
 182     nmem_destroy(m_mem_invalid_session);
 183     nmem_destroy(m_referenceId_mem);
 184
 185     xfree(m_proxyTarget);
 186     xfree(m_default_target);
 187     xfree(m_proxy_authentication);
 188     xfree(m_optimize);
 189
 190 #if HAVE_XSLT
 191     if (m_stylesheet_xsp)
 192         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 193 #endif
 194     xfree (m_time_tv);
 195
 196     xfree (m_schema);
 197     xfree (m_backend_type);
 198     xfree (m_backend_charset);
 199     if (m_s2z_odr_init)
 200         odr_destroy(m_s2z_odr_init);
 201     if (m_s2z_odr_search)
 202         odr_destroy(m_s2z_odr_search);
 203     if (!m_parent)
 204         low_socket_close();
 205     delete m_config;
 206 }
 207
 208 int Yaz_Proxy::set_config(const char *config)
 209 {
 210     delete m_config;
 211     m_config = new Yaz_ProxyConfig();
 212     xfree(m_config_fname);
 213     m_config_fname = xstrdup(config);
 214     int r = m_config->read_xml(config);
 215     if (!r)
 216         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 217     return r;
 218 }
 219
 220 void Yaz_Proxy::set_default_target(const char *target)
 221 {
 222     xfree (m_default_target);
 223     m_default_target = 0;
 224     if (target)
 225         m_default_target = (char *) xstrdup (target);
 226 }
 227
 228 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 229 {
 230     xfree (m_proxy_authentication);
 231     m_proxy_authentication = 0;
 232     if (auth)
 233         m_proxy_authentication = (char *) xstrdup (auth);
 234 }
 235
 236 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 237 {
 238     if (m_parent)
 239         return m_parent->check_reconfigure();
 240
 241     Yaz_ProxyConfig *cfg = m_config;
 242     if (m_reconfig_flag)
 243     {
 244         yaz_log(YLOG_LOG, "reconfigure");
 245         yaz_log_reopen();
 246         if (m_config_fname && cfg)
 247         {
 248             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 249             int r = cfg->read_xml(m_config_fname);
 250             if (r)
 251                 yaz_log(YLOG_WARN, "reconfigure failed");
 252             else
 253             {
 254                 m_log_mask = 0;
 255                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 256             }
 257         }
 258         else
 259             yaz_log(YLOG_LOG, "reconfigure");
 260         m_reconfig_flag = 0;
 261     }
 262     return cfg;
 263 }
 264
 265 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 266                                             *the_PDU_Observable, int fd)
 267 {
 268     check_reconfigure();
 269     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 270     new_proxy->m_config = 0;
 271     new_proxy->m_config_fname = 0;
 272     new_proxy->timeout(m_client_idletime);
 273     new_proxy->m_target_idletime = m_target_idletime;
 274     new_proxy->set_default_target(m_default_target);
 275     new_proxy->m_max_clients = m_max_clients;
 276     new_proxy->m_log_mask = m_log_mask;
 277     new_proxy->set_APDU_log(get_APDU_log());
 278     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 279         new_proxy->set_APDU_yazlog(1);
 280     else
 281         new_proxy->set_APDU_yazlog(0);
 282     new_proxy->set_proxy_authentication(m_proxy_authentication);
 283     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 284     m_session_no++;
 285     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 286              the_PDU_Observable->getpeername());
 287     return new_proxy;
 288 }
 289
 290 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 291 {
 292     int oid[OID_SIZE];
 293     Z_OtherInformationUnit *oi;
 294     struct oident ent;
 295     ent.proto = PROTO_Z3950;
 296     ent.oclass = CLASS_USERINFO;
 297     ent.value = (oid_value) VAL_COOKIE;
 298     assert (oid_ent_to_oid (&ent, oid));
 299
 300     if (oid_ent_to_oid (&ent, oid) &&
 301         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 302         oi->which == Z_OtherInfo_characterInfo)
 303         return oi->information.characterInfo;
 304     return 0;
 305 }
 306
 307 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 308 {
 309     int oid[OID_SIZE];
 310     Z_OtherInformationUnit *oi;
 311     struct oident ent;
 312     ent.proto = PROTO_Z3950;
 313     ent.oclass = CLASS_USERINFO;
 314     ent.value = (oid_value) VAL_PROXY;
 315     if (oid_ent_to_oid (&ent, oid) &&
 316         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 317         oi->which == Z_OtherInfo_characterInfo)
 318         return oi->information.characterInfo;
 319     return 0;
 320 }
 321
 322 const char *Yaz_Proxy::load_balance(const char **url)
 323 {
 324     int zurl_in_use[MAX_ZURL_PLEX];
 325     int zurl_in_spare[MAX_ZURL_PLEX];
 326     Yaz_ProxyClient *c;
 327     int i;
 328
 329     for (i = 0; i<MAX_ZURL_PLEX; i++)
 330     {
 331         zurl_in_use[i] = 0;
 332         zurl_in_spare[i] = 0;
 333     }
 334     for (c = m_parent->m_clientPool; c; c = c->m_next)
 335     {
 336         for (i = 0; url[i]; i++)
 337             if (!strcmp(url[i], c->get_hostname()))
 338             {
 339                 zurl_in_use[i]++;
 340                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 341                     zurl_in_spare[i]++;
 342             }
 343     }
 344     int min_use = 100000;
 345     int spare_for_min = 0;
 346     int max_spare = 0;
 347     const char *ret_min = 0;
 348     const char *ret_spare = 0;
 349     for (i = 0; url[i]; i++)
 350     {
 351         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 352                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 353         if (min_use > zurl_in_use[i])
 354         {
 355             ret_min = url[i];
 356             min_use = zurl_in_use[i];
 357             spare_for_min = zurl_in_spare[i];
 358         }
 359         if (max_spare < zurl_in_spare[i])
 360         {
 361             ret_spare = url[i];
 362             max_spare = zurl_in_spare[i];
 363         }
 364     }
 365     // use the one with minimum connections if spare is > 3
 366     if (spare_for_min > 3)
 367         return ret_min;
 368     // use one with most spares (if any)
 369     if (max_spare > 0)
 370         return ret_spare;
 371     return ret_min;
 372 }
 373
 374 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 375                                        const char *proxy_host)
 376 {
 377     assert (m_parent);
 378     Yaz_Proxy *parent = m_parent;
 379     Yaz_ProxyClient *c = m_client;
 380
 381     if (!m_proxyTarget)
 382     {
 383         const char *url[MAX_ZURL_PLEX];
 384         Yaz_ProxyConfig *cfg = check_reconfigure();
 385         if (proxy_host)
 386         {
 387 #if 0
 388 /* only to be enabled for debugging... */
 389             if (!strcmp(proxy_host, "stop"))
 390                 exit(0);
 391 #endif
 392             xfree(m_default_target);
 393             m_default_target = xstrdup(proxy_host);
 394         }
 395         proxy_host = m_default_target;
 396         int client_idletime = -1;
 397         const char *cql2rpn_fname = 0;
 398         const char *authentication = 0;
 399         url[0] = m_default_target;
 400         url[1] = 0;
 401         if (cfg)
 402         {
 403             int pre_init = 0;
 404             cfg->get_target_info(proxy_host, url, &m_bw_max,
 405                                  &m_pdu_max, &m_max_record_retrieve,
 406                                  &m_target_idletime, &client_idletime,
 407                                  &parent->m_max_clients,
 408                                  &m_keepalive_limit_bw,
 409                                  &m_keepalive_limit_pdu,
 410                                  &pre_init,
 411                                  &cql2rpn_fname,
 412                                  &authentication);
 413         }
 414         if (client_idletime != -1)
 415         {
 416             m_client_idletime = client_idletime;
 417             timeout(m_client_idletime);
 418         }
 419         if (cql2rpn_fname)
 420             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 421         if (authentication)
 422             set_proxy_authentication(authentication);
 423         if (!url[0])
 424         {
 425             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 426             return 0;
 427         }
 428         // we don't handle multiplexing for cookie session, so we just
 429         // pick the first one in this case (anonymous users will be able
 430         // to use any backend)
 431         if (cookie && *cookie)
 432             m_proxyTarget = (char*) xstrdup(url[0]);
 433         else
 434             m_proxyTarget = (char*) xstrdup(load_balance(url));
 435     }
 436     if (cookie && *cookie)
 437     {   // search in sessions with a cookie
 438         for (c = parent->m_clientPool; c; c = c->m_next)
 439         {
 440             assert (c->m_prev);
 441             assert (*c->m_prev == c);
 442             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 443                 !strcmp(m_proxyTarget, c->get_hostname()))
 444             {
 445                 // Found it in cache
 446                 // The following handles "cancel"
 447                 // If connection is busy (waiting for PDU) and
 448                 // we have an initRequest we can safely do re-open
 449                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 450                 {
 451                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 452                              c->get_hostname());
 453                     c->close();
 454                     c->m_init_flag = 0;
 455
 456                     c->m_last_ok = 0;
 457                     c->m_cache.clear();
 458                     c->m_last_resultCount = 0;
 459                     c->m_sr_transform = 0;
 460                     c->m_waiting = 0;
 461                     c->m_resultSetStartPoint = 0;
 462                     c->m_target_idletime = m_target_idletime;
 463                     if (c->client(m_proxyTarget))
 464                     {
 465                         delete c;
 466                         return 0;
 467                     }
 468                     c->timeout(30);
 469                 }
 470                 c->m_seqno = parent->m_seqno;
 471                 if (c->m_server && c->m_server != this)
 472                     c->m_server->m_client = 0;
 473                 c->m_server = this;
 474                 (parent->m_seqno)++;
 475                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 476                 return c;
 477             }
 478         }
 479     }
 480     else if (!c && apdu->which == Z_APDU_initRequest
 481              && apdu->u.initRequest->idAuthentication == 0)
 482     {
 483         // anonymous sessions without cookie.
 484         // if authentication is set it is NOT anonymous se we can't share them.
 485         for (c = parent->m_clientPool; c; c = c->m_next)
 486         {
 487             assert(c->m_prev);
 488             assert(*c->m_prev == c);
 489             if (c->m_server == 0 && c->m_cookie == 0 &&
 490                 c->m_waiting == 0 &&
 491                 !strcmp(m_proxyTarget, c->get_hostname()))
 492             {
 493                 // found it in cache
 494                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 495                          m_session_str, parent->m_seqno, c->get_hostname());
 496
 497                 c->m_seqno = parent->m_seqno;
 498                 assert(c->m_server == 0);
 499                 c->m_server = this;
 500
 501                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 502                     c->set_APDU_yazlog(1);
 503                 else
 504                     c->set_APDU_yazlog(0);
 505
 506                 (parent->m_seqno)++;
 507
 508                 parent->pre_init();
 509
 510                 return c;
 511             }
 512         }
 513     }
 514     if (!m_client)
 515     {
 516         if (apdu->which != Z_APDU_initRequest)
 517         {
 518             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 519             return 0;
 520         }
 521         Z_InitRequest *initRequest = apdu->u.initRequest;
 522
 523         if (!initRequest->idAuthentication)
 524         {
 525             if (m_proxy_authentication)
 526             {
 527                 initRequest->idAuthentication =
 528                     (Z_IdAuthentication *)
 529                     odr_malloc (odr_encode(),
 530                                 sizeof(*initRequest->idAuthentication));
 531                 initRequest->idAuthentication->which =
 532                     Z_IdAuthentication_open;
 533                 initRequest->idAuthentication->u.open =
 534                     odr_strdup (odr_encode(), m_proxy_authentication);
 535             }
 536         }
 537         else
 538         {
 539             // the client use authentication. We set the keepalive PDU
 540             // to 0 so we don't cache it in releaseClient
 541             m_keepalive_limit_pdu = 0;
 542         }
 543         // go through list of clients - and find the lowest/oldest one.
 544         Yaz_ProxyClient *c_min = 0;
 545         int min_seq = -1;
 546         int no_of_clients = 0;
 547         if (parent->m_clientPool)
 548             yaz_log (YLOG_DEBUG, "Existing sessions");
 549         for (c = parent->m_clientPool; c; c = c->m_next)
 550         {
 551             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 552                                c->m_waiting, c->get_hostname(),
 553                                c->m_cookie ? c->m_cookie : "");
 554             no_of_clients++;
 555             if (min_seq < 0 || c->m_seqno < min_seq)
 556             {
 557                 min_seq = c->m_seqno;
 558                 c_min = c;
 559             }
 560         }
 561         if (no_of_clients >= parent->m_max_clients)
 562         {
 563             c = c_min;
 564             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 565             {
 566                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 567                          m_session_str, parent->m_max_clients, c->m_seqno);
 568                 if (c->m_server && c->m_server != this)
 569                     delete c->m_server;
 570                 c->m_server = 0;
 571             }
 572             else
 573             {
 574                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 575                          m_session_str, parent->m_max_clients,
 576                          c->m_seqno, parent->m_seqno, c->get_hostname());
 577                 xfree (c->m_cookie);
 578                 c->m_cookie = 0;
 579                 if (cookie)
 580                     c->m_cookie = xstrdup(cookie);
 581                 c->m_seqno = parent->m_seqno;
 582                 if (c->m_server && c->m_server != this)
 583                 {
 584                     c->m_server->m_client = 0;
 585                     delete c->m_server;
 586                 }
 587                 (parent->m_seqno)++;
 588                 c->m_target_idletime = m_target_idletime;
 589                 c->timeout(m_target_idletime);
 590
 591                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 592                     c->set_APDU_yazlog(1);
 593                 else
 594                     c->set_APDU_yazlog(0);
 595
 596                 return c;
 597             }
 598         }
 599         else
 600         {
 601             yaz_log (YLOG_LOG, "%sNEW %d %s",
 602                      m_session_str, parent->m_seqno, m_proxyTarget);
 603             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 604             c->m_next = parent->m_clientPool;
 605             if (c->m_next)
 606                 c->m_next->m_prev = &c->m_next;
 607             parent->m_clientPool = c;
 608             c->m_prev = &parent->m_clientPool;
 609         }
 610
 611         xfree (c->m_cookie);
 612         c->m_cookie = 0;
 613         if (cookie)
 614             c->m_cookie = xstrdup(cookie);
 615
 616         c->m_seqno = parent->m_seqno;
 617         c->m_init_flag = 0;
 618         c->m_last_resultCount = 0;
 619         c->m_last_ok = 0;
 620         c->m_cache.clear();
 621         c->m_sr_transform = 0;
 622         c->m_waiting = 0;
 623         c->m_resultSetStartPoint = 0;
 624         (parent->m_seqno)++;
 625         if (c->client(m_proxyTarget))
 626         {
 627             delete c;
 628             return 0;
 629         }
 630         c->m_target_idletime = m_target_idletime;
 631         c->timeout(30);
 632
 633         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 634             c->set_APDU_yazlog(1);
 635         else
 636             c->set_APDU_yazlog(0);
 637     }
 638     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 639     return c;
 640 }
 641
 642 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 643 {
 644     int i;
 645     for (i = 0; i<num; i++)
 646     {
 647         oident *ent;
 648         Z_DefaultDiagFormat *r;
 649         Z_DiagRec *p = pp[i];
 650         if (p->which != Z_DiagRec_defaultFormat)
 651         {
 652             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 653             return;
 654         }
 655         else
 656             r = p->u.defaultFormat;
 657         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 658             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 659             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 660         switch (r->which)
 661         {
 662         case Z_DefaultDiagFormat_v2Addinfo:
 663             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 664                     m_session_str,
 665                     *r->condition, diagbib1_str(*r->condition),
 666                     r->u.v2Addinfo);
 667             break;
 668         case Z_DefaultDiagFormat_v3Addinfo:
 669             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 670                     m_session_str,
 671                     *r->condition, diagbib1_str(*r->condition),
 672                     r->u.v3Addinfo);
 673             break;
 674         }
 675     }
 676 }
 677
 678 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 679 {
 680     if (!m_stylesheet_xsp || p->num_records <= 0)
 681         return 0;  /* no XSLT to be done ... */
 682
 683     m_stylesheet_offset = 0;
 684     m_stylesheet_nprl = p;
 685     m_stylesheet_apdu = apdu;
 686     timeout(0);
 687     return 1;
 688 }
 689
 690 void Yaz_Proxy::convert_xsl_delay()
 691 {
 692     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 693 #if HAVE_XSLT
 694     if (npr->which == Z_NamePlusRecord_databaseRecord)
 695     {
 696         Z_External *r = npr->u.databaseRecord;
 697         if (r->which == Z_External_octet)
 698         {
 699 #if 0
 700             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 701 #endif
 702             xmlDocPtr res, doc = xmlParseMemory(
 703                 (char*) r->u.octet_aligned->buf,
 704                 r->u.octet_aligned->len);
 705
 706
 707             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 708                     m_session_str, m_stylesheet_offset);
 709             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 710                                       doc, 0);
 711
 712             if (res)
 713             {
 714                 xmlChar *out_buf;
 715                 int out_len;
 716                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 717
 718                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 719                     u.databaseRecord =
 720                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 721                                  (char*) out_buf, out_len);
 722                 xmlFree(out_buf);
 723                 xmlFreeDoc(res);
 724             }
 725
 726             xmlFreeDoc(doc);
 727         }
 728     }
 729 #endif
 730     m_stylesheet_offset++;
 731     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 732     {
 733         m_stylesheet_nprl = 0;
 734 #if HAVE_XSLT
 735         if (m_stylesheet_xsp)
 736             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 737 #endif
 738         m_stylesheet_xsp = 0;
 739         timeout(m_client_idletime);
 740         send_PDU_convert(m_stylesheet_apdu);
 741     }
 742     else
 743         timeout(0);
 744 }
 745
 746 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 747 {
 748     if (m_frontend_type != VAL_NONE)
 749     {
 750         int i;
 751         for (i = 0; i < p->num_records; i++)
 752         {
 753             Z_NamePlusRecord *npr = p->records[i];
 754             if (npr->which == Z_NamePlusRecord_databaseRecord)
 755             {
 756                 Z_External *r = npr->u.databaseRecord;
 757                 if (r->which == Z_External_octet)
 758                 {
 759                     npr->u.databaseRecord =
 760                         z_ext_record(odr_encode(),
 761                                      m_frontend_type,
 762                                      (char*) r->u.octet_aligned->buf,
 763                                      r->u.octet_aligned->len);
 764                 }
 765             }
 766         }
 767     }
 768 }
 769
 770 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
 771                                    const char *backend_charset)
 772 {
 773     int i;
 774
 775     if (!backend_charset)
 776         backend_charset = "MARC-8";
 777     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
 778     yaz_marc_t mt = yaz_marc_create();
 779     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 780     yaz_marc_iconv(mt, cd);
 781     for (i = 0; i < p->num_records; i++)
 782     {
 783         Z_NamePlusRecord *npr = p->records[i];
 784         if (npr->which == Z_NamePlusRecord_databaseRecord)
 785         {
 786             Z_External *r = npr->u.databaseRecord;
 787             if (r->which == Z_External_octet)
 788             {
 789                 int rlen;
 790                 char *result;
 791                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 792                                         r->u.octet_aligned->len,
 793                                         &result, &rlen))
 794                 {
 795                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 796                                                          VAL_TEXT_XML,
 797                                                          result, rlen);
 798                 }
 799             }
 800         }
 801     }
 802     if (cd)
 803         yaz_iconv_close(cd);
 804     yaz_marc_destroy(mt);
 805 }
 806
 807 void Yaz_Proxy::logtime()
 808 {
 809 #if HAVE_GETTIMEOFDAY
 810     struct timeval *tv = (struct timeval*) m_time_tv;
 811     if (tv->tv_sec)
 812     {
 813         struct timeval tv1;
 814         gettimeofday(&tv1, 0);
 815         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
 816             (tv1.tv_usec - tv->tv_usec);
 817         if (diff >= 0)
 818             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 819                     diff/1000000, (diff/1000)%1000);
 820     }
 821     tv->tv_sec = 0;
 822     tv->tv_usec = 0;
 823 #endif
 824 }
 825
 826 int Yaz_Proxy::send_http_response(int code)
 827 {
 828     ODR o = odr_encode();
 829     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 830     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 831     if (m_http_version)
 832         hres->version = odr_strdup(o, m_http_version);
 833     if (m_http_keepalive)
 834         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 835     else
 836         timeout(0);
 837
 838     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 839     {
 840         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
 841                  gdu_name(gdu));
 842     }
 843     int len;
 844     int r = send_GDU(gdu, &len);
 845     m_bytes_sent += len;
 846     m_bw_stat.add_bytes(len);
 847     logtime();
 848     return r;
 849 }
 850
 851 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 852 {
 853     ODR o = odr_encode();
 854     const char *ctype = "text/xml";
 855     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 856     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 857     if (m_http_version)
 858         hres->version = odr_strdup(o, m_http_version);
 859     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 860     if (m_http_keepalive)
 861         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 862     else
 863         timeout(0);
 864
 865     static Z_SOAP_Handler soap_handlers[2] = {
 866 #if HAVE_XSLT
 867         {"http://www.loc.gov/zing/srw/", 0,
 868          (Z_SOAP_fun) yaz_srw_codec},
 869 #endif
 870         {0, 0, 0}
 871     };
 872
 873     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 874     soap_package->which = Z_SOAP_generic;
 875     soap_package->u.generic =
 876         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 877     soap_package->u.generic->no = 0;
 878     soap_package->u.generic->ns = soap_handlers[0].ns;
 879     soap_package->u.generic->p = (void *) srw_pdu;
 880     soap_package->ns = m_soap_ns;
 881     z_soap_codec_enc_xsl(o, &soap_package,
 882                          &hres->content_buf, &hres->content_len,
 883                          soap_handlers, 0, m_s2z_stylesheet);
 884     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 885     {
 886         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
 887                  gdu_name(gdu));
 888     }
 889     int len;
 890     int r = send_GDU(gdu, &len);
 891     m_bytes_sent += len;
 892     m_bw_stat.add_bytes(len);
 893     logtime();
 894     return r;
 895 }
 896
 897 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
 898 {
 899     ODR o = odr_encode();
 900     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 901     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 902
 903     srw_res->num_diagnostics = 1;
 904     srw_res->diagnostics = (Z_SRW_diagnostic *)
 905         odr_malloc(o, sizeof(*srw_res->diagnostics));
 906     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
 907     return send_srw_response(srw_pdu);
 908 }
 909
 910 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 911                              Z_DefaultDiagFormat *ddf)
 912 {
 913     int bib1_code = *ddf->condition;
 914     if (bib1_code == 109)
 915         return 404;
 916     srw_res->num_diagnostics = 1;
 917     srw_res->diagnostics = (Z_SRW_diagnostic *)
 918         odr_malloc(o, sizeof(*srw_res->diagnostics));
 919     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
 920                           yaz_diag_bib1_to_srw(*ddf->condition),
 921                           ddf->u.v2Addinfo);
 922     return 0;
 923 }
 924
 925 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
 926 {
 927     ODR o = odr_encode();
 928     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 929     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 930
 931     srw_res->numberOfRecords = odr_intdup (o, hits);
 932     if (records && records->which == Z_Records_DBOSD)
 933     {
 934         srw_res->num_records =
 935             records->u.databaseOrSurDiagnostics->num_records;
 936         int i;
 937         srw_res->records = (Z_SRW_record *)
 938             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
 939         for (i = 0; i < srw_res->num_records; i++)
 940         {
 941             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
 942             if (npr->which != Z_NamePlusRecord_databaseRecord)
 943             {
 944                 srw_res->records[i].recordSchema = "diagnostic";
 945                 srw_res->records[i].recordPacking = m_s2z_packing;
 946                 srw_res->records[i].recordData_buf = "67";
 947                 srw_res->records[i].recordData_len = 2;
 948                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 949                 continue;
 950             }
 951             Z_External *r = npr->u.databaseRecord;
 952             oident *ent = oid_getentbyoid(r->direct_reference);
 953             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
 954             {
 955                 srw_res->records[i].recordSchema = m_schema;
 956                 srw_res->records[i].recordPacking = m_s2z_packing;
 957                 srw_res->records[i].recordData_buf = (char*)
 958                     r->u.octet_aligned->buf;
 959                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
 960                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 961             }
 962             else
 963             {
 964                 srw_res->records[i].recordSchema = "diagnostic";
 965                 srw_res->records[i].recordPacking = m_s2z_packing;
 966                 srw_res->records[i].recordData_buf = "67";
 967                 srw_res->records[i].recordData_len = 2;
 968                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 969             }
 970         }
 971     }
 972     if (records && records->which == Z_Records_NSD)
 973     {
 974         int http_code;
 975         http_code = z_to_srw_diag(odr_encode(), srw_res,
 976                                    records->u.nonSurrogateDiagnostic);
 977         if (http_code)
 978             return send_http_response(http_code);
 979     }
 980     return send_srw_response(srw_pdu);
 981
 982 }
 983
 984 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
 985                                         int num_diagnostics)
 986 {
 987     Yaz_ProxyConfig *cfg = check_reconfigure();
 988     if (cfg)
 989     {
 990         int len;
 991         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
 992                                        m_s2z_database, &len);
 993         if (b)
 994         {
 995             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
 996             Z_SRW_explainResponse *er = res->u.explain_response;
 997
 998             er->record.recordData_buf = b;
 999             er->record.recordData_len = len;
1000             er->record.recordPacking = m_s2z_packing;
1001             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1002
1003             er->diagnostics = diagnostics;
1004             er->num_diagnostics = num_diagnostics;
1005             return send_srw_response(res);
1006         }
1007     }
1008     return send_http_response(404);
1009 }
1010
1011 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1012 {
1013     if (m_http_version)
1014     {
1015         if (apdu->which == Z_APDU_initResponse)
1016         {
1017             Z_InitResponse *res = apdu->u.initResponse;
1018             if (*res->result == 0)
1019             {
1020                 send_to_srw_client_error(3, 0);
1021             }
1022             else if (!m_s2z_search_apdu)
1023             {
1024                 send_srw_explain_response(0, 0);
1025             }
1026             else
1027             {
1028                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1029             }
1030         }
1031         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1032         {
1033             m_s2z_search_apdu = 0;
1034             Z_SearchResponse *res = apdu->u.searchResponse;
1035             m_s2z_hit_count = *res->resultCount;
1036             if (res->records && res->records->which == Z_Records_NSD)
1037             {
1038                 send_to_srw_client_ok(0, res->records, 1);
1039             }
1040             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1041             {
1042                 // adjust
1043                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1044
1045                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1046                 {
1047                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1048                         > m_s2z_hit_count)
1049                         *pr->numberOfRecordsRequested =
1050                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1051                 }
1052                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1053             }
1054             else
1055             {
1056                 m_s2z_present_apdu = 0;
1057                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1058             }
1059         }
1060         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1061         {
1062             int start =
1063                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1064
1065             m_s2z_present_apdu = 0;
1066             Z_PresentResponse *res = apdu->u.presentResponse;
1067             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1068         }
1069     }
1070     else
1071     {
1072         int len = 0;
1073         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1074             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1075                      apdu_name(apdu));
1076         int r = send_Z_PDU(apdu, &len);
1077         m_bytes_sent += len;
1078         m_bw_stat.add_bytes(len);
1079         logtime();
1080         return r;
1081     }
1082     return 0;
1083 }
1084
1085 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1086 {
1087     int kill_session = 0;
1088     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1089
1090     if (new_id)
1091         *new_id = m_referenceId;
1092
1093     if (apdu->which == Z_APDU_searchResponse)
1094     {
1095         Z_SearchResponse *sr = apdu->u.searchResponse;
1096         Z_Records *p = sr->records;
1097         if (p && p->which == Z_Records_NSD)
1098         {
1099             Z_DiagRec dr, *dr_p = &dr;
1100             dr.which = Z_DiagRec_defaultFormat;
1101             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1102
1103             *sr->searchStatus = 0;
1104             display_diagrecs(&dr_p, 1);
1105         }
1106         else
1107         {
1108             if (p && p->which == Z_Records_DBOSD)
1109             {
1110                 if (m_backend_type)
1111                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1112                 if (m_marcxml_flag)
1113                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1114                                        m_backend_charset);
1115                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1116                     return 0;
1117
1118             }
1119             if (sr->resultCount)
1120             {
1121                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1122                         *sr->resultCount);
1123                 if (*sr->resultCount < 0)
1124                 {
1125                     m_invalid_session = 1;
1126                     kill_session = 1;
1127
1128                     *sr->searchStatus = 0;
1129                     sr->records =
1130                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1131                     *sr->resultCount = 0;
1132                 }
1133             }
1134         }
1135     }
1136     else if (apdu->which == Z_APDU_presentResponse)
1137     {
1138         Z_PresentResponse *sr = apdu->u.presentResponse;
1139         Z_Records *p = sr->records;
1140         if (p && p->which == Z_Records_NSD)
1141         {
1142             Z_DiagRec dr, *dr_p = &dr;
1143             dr.which = Z_DiagRec_defaultFormat;
1144             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1145             if (*sr->presentStatus == Z_PresentStatus_success)
1146                 *sr->presentStatus = Z_PresentStatus_failure;
1147             display_diagrecs(&dr_p, 1);
1148         }
1149         if (p && p->which == Z_Records_DBOSD)
1150         {
1151             if (m_backend_type)
1152                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1153             if (m_marcxml_flag)
1154                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1155                                    m_backend_charset);
1156             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1157                 return 0;
1158         }
1159     }
1160     else if (apdu->which == Z_APDU_initResponse)
1161     {
1162         if (m_initRequest_options)
1163         {
1164             Z_Options *nopt =
1165                 (Odr_bitmask *)odr_malloc(odr_encode(),
1166                                           sizeof(Odr_bitmask));
1167             ODR_MASK_ZERO(nopt);
1168
1169             int i;
1170             for (i = 0; i<24; i++)
1171                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1172                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1173                     ODR_MASK_SET(nopt, i);
1174             apdu->u.initResponse->options = nopt;
1175         }
1176         if (m_initRequest_version)
1177         {
1178             Z_ProtocolVersion *nopt =
1179                 (Odr_bitmask *)odr_malloc(odr_encode(),
1180                                           sizeof(Odr_bitmask));
1181             ODR_MASK_ZERO(nopt);
1182
1183             int i;
1184             for (i = 0; i<8; i++)
1185                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1186                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1187                     ODR_MASK_SET(nopt, i);
1188             apdu->u.initResponse->protocolVersion = nopt;
1189         }
1190         apdu->u.initResponse->preferredMessageSize =
1191             odr_intdup(odr_encode(),
1192                        m_client->m_initResponse_preferredMessageSize >
1193                        m_initRequest_preferredMessageSize ?
1194                        m_initRequest_preferredMessageSize :
1195                        m_client->m_initResponse_preferredMessageSize);
1196         apdu->u.initResponse->maximumRecordSize =
1197             odr_intdup(odr_encode(),
1198                        m_client->m_initResponse_maximumRecordSize >
1199                        m_initRequest_maximumRecordSize ?
1200                        m_initRequest_maximumRecordSize :
1201                        m_client->m_initResponse_maximumRecordSize);
1202     }
1203     int r = send_PDU_convert(apdu);
1204     if (r)
1205         return r;
1206     if (kill_session)
1207     {
1208         delete m_client;
1209         m_client = 0;
1210         m_parent->pre_init();
1211     }
1212     return r;
1213 }
1214
1215 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1216 {
1217     int len = 0;
1218     const char *apdu_name_tmp = apdu_name(apdu);
1219     int r = send_Z_PDU(apdu, &len);
1220     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1221         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1222                  get_session_str(),
1223                  apdu_name_tmp, get_hostname(), len);
1224     m_bytes_sent += len;
1225     return r;
1226 }
1227
1228 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1229 {
1230     if (apdu->which == Z_APDU_presentRequest)
1231     {
1232         Z_PresentRequest *pr = apdu->u.presentRequest;
1233         int toget = *pr->numberOfRecordsRequested;
1234         int start = *pr->resultSetStartPoint;
1235
1236         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1237                 pr->resultSetId, start, toget);
1238
1239         if (*m_parent->m_optimize == '0')
1240             return apdu;
1241
1242         if (!m_client->m_last_resultSetId)
1243         {
1244             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1245             new_apdu->u.presentResponse->records =
1246                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1247                                                pr->resultSetId);
1248             send_to_client(new_apdu);
1249             return 0;
1250         }
1251         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1252         {
1253             if (start+toget-1 > m_client->m_last_resultCount)
1254             {
1255                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1256                 new_apdu->u.presentResponse->records =
1257                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1258                 send_to_client(new_apdu);
1259                 return 0;
1260             }
1261             Z_NamePlusRecordList *npr;
1262 #if 0
1263             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1264                     m_session_str, start, toget, yaz_z3950oid_to_str(
1265                         pr->preferredRecordSyntax, &oclass));
1266 #endif
1267             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1268                                           pr->preferredRecordSyntax,
1269                                           pr->recordComposition))
1270             {
1271                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1272                          m_session_str);
1273                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1274                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1275
1276                 new_apdu->u.presentResponse->numberOfRecordsReturned
1277                     = odr_intdup(odr_encode(), toget);
1278
1279                 new_apdu->u.presentResponse->records = (Z_Records*)
1280                     odr_malloc(odr_encode(), sizeof(Z_Records));
1281                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1282                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1283                 new_apdu->u.presentResponse->nextResultSetPosition =
1284                     odr_intdup(odr_encode(), start+toget);
1285
1286                 send_to_client(new_apdu);
1287                 return 0;
1288             }
1289         }
1290     }
1291
1292     if (apdu->which != Z_APDU_searchRequest)
1293         return apdu;
1294     Z_SearchRequest *sr = apdu->u.searchRequest;
1295     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1296     Yaz_Z_Databases this_databases;
1297
1298     this_databases.set(sr->num_databaseNames, (const char **)
1299                        sr->databaseNames);
1300
1301     this_query->set_Z_Query(sr->query);
1302
1303     char query_str[120];
1304     this_query->print(query_str, sizeof(query_str)-1);
1305     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1306
1307     if (*m_parent->m_optimize != '0' &&
1308         m_client->m_last_ok && m_client->m_last_query &&
1309         m_client->m_last_query->match(this_query) &&
1310         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1311         m_client->m_last_databases.match(this_databases))
1312     {
1313         delete this_query;
1314         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1315             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1316         {
1317             Z_NamePlusRecordList *npr;
1318             int toget = *sr->mediumSetPresentNumber;
1319             Z_RecordComposition *comp = 0;
1320
1321             if (toget > m_client->m_last_resultCount)
1322                 toget = m_client->m_last_resultCount;
1323
1324             if (sr->mediumSetElementSetNames)
1325             {
1326                 comp = (Z_RecordComposition *)
1327                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1328                 comp->which = Z_RecordComp_simple;
1329                 comp->u.simple = sr->mediumSetElementSetNames;
1330             }
1331
1332             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1333                                           sr->preferredRecordSyntax, comp))
1334             {
1335                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1336                          m_session_str);
1337                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1338                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1339                 new_apdu->u.searchResponse->resultCount =
1340                     &m_client->m_last_resultCount;
1341
1342                 new_apdu->u.searchResponse->numberOfRecordsReturned
1343                     = odr_intdup(odr_encode(), toget);
1344
1345                 new_apdu->u.searchResponse->presentStatus =
1346                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1347                 new_apdu->u.searchResponse->records = (Z_Records*)
1348                     odr_malloc(odr_encode(), sizeof(Z_Records));
1349                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1350                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1351                 new_apdu->u.searchResponse->nextResultSetPosition =
1352                     odr_intdup(odr_encode(), toget+1);
1353                 send_to_client(new_apdu);
1354                 return 0;
1355             }
1356             else
1357             {
1358                 // medium Set
1359                 // send present request (medium size)
1360                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1361                          m_session_str);
1362
1363                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1364                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1365                 pr->referenceId = sr->referenceId;
1366                 pr->resultSetId = sr->resultSetName;
1367                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1368                 *pr->numberOfRecordsRequested = toget;
1369                 pr->recordComposition = comp;
1370                 m_client->m_sr_transform = 1;
1371                 return new_apdu;
1372             }
1373         }
1374         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1375             m_client->m_last_resultCount <= 0)
1376         {
1377             // large set. Return pseudo-search response immediately
1378             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1379                      m_session_str);
1380             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1381             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1382             new_apdu->u.searchResponse->resultCount =
1383                 &m_client->m_last_resultCount;
1384             send_to_client(new_apdu);
1385             return 0;
1386         }
1387         else
1388         {
1389             Z_NamePlusRecordList *npr;
1390             int toget = m_client->m_last_resultCount;
1391             Z_RecordComposition *comp = 0;
1392             // small set
1393             // send a present request (small set)
1394
1395             if (sr->smallSetElementSetNames)
1396             {
1397                 comp = (Z_RecordComposition *)
1398                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1399                 comp->which = Z_RecordComp_simple;
1400                 comp->u.simple = sr->smallSetElementSetNames;
1401             }
1402
1403             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1404                                           sr->preferredRecordSyntax, comp))
1405             {
1406                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1407                          m_session_str);
1408                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1409                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1410                 new_apdu->u.searchResponse->resultCount =
1411                     &m_client->m_last_resultCount;
1412
1413                 new_apdu->u.searchResponse->numberOfRecordsReturned
1414                     = odr_intdup(odr_encode(), toget);
1415
1416                 new_apdu->u.searchResponse->presentStatus =
1417                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1418                 new_apdu->u.searchResponse->records = (Z_Records*)
1419                     odr_malloc(odr_encode(), sizeof(Z_Records));
1420                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1421                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1422                 new_apdu->u.searchResponse->nextResultSetPosition =
1423                     odr_intdup(odr_encode(), toget+1);
1424                 send_to_client(new_apdu);
1425                 return 0;
1426             }
1427             else
1428             {
1429                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1430                          m_session_str);
1431                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1432                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1433                 pr->referenceId = sr->referenceId;
1434                 pr->resultSetId = sr->resultSetName;
1435                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1436                 *pr->numberOfRecordsRequested = toget;
1437                 pr->recordComposition = comp;
1438                 m_client->m_sr_transform = 1;
1439                 return new_apdu;
1440             }
1441         }
1442     }
1443     else  // query doesn't match
1444     {
1445         delete m_client->m_last_query;
1446         m_client->m_last_query = this_query;
1447         m_client->m_last_ok = 0;
1448         m_client->m_cache.clear();
1449         m_client->m_resultSetStartPoint = 0;
1450
1451         xfree (m_client->m_last_resultSetId);
1452         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1453
1454         m_client->m_last_databases.set(sr->num_databaseNames,
1455                                        (const char **) sr->databaseNames);
1456     }
1457     return apdu;
1458 }
1459
1460
1461 void Yaz_Proxy::inc_request_no()
1462 {
1463     char *cp = strchr(m_session_str, ' ');
1464     m_request_no++;
1465     if (cp)
1466         sprintf(cp+1, "%d ", m_request_no);
1467 }
1468
1469 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1470 {
1471     inc_request_no();
1472
1473     m_bytes_recv += len;
1474
1475     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1476         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1477                  m_session_str, gdu_name(apdu), len);
1478
1479     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1480         shutdown();
1481
1482     m_bw_stat.add_bytes(len);
1483     m_pdu_stat.add_bytes(1);
1484
1485 #if HAVE_GETTIMEOFDAY
1486     gettimeofday((struct timeval *) m_time_tv, 0);
1487 #endif
1488
1489     int bw_total = m_bw_stat.get_total();
1490     int pdu_total = m_pdu_stat.get_total();
1491
1492     int reduce = 0;
1493     if (m_bw_max)
1494     {
1495         if (bw_total > m_bw_max)
1496         {
1497             reduce = (bw_total/m_bw_max);
1498         }
1499     }
1500     if (m_pdu_max)
1501     {
1502         if (pdu_total > m_pdu_max)
1503         {
1504             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1505             reduce = (reduce > nreduce) ? reduce : nreduce;
1506         }
1507     }
1508     if (reduce)
1509     {
1510         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1511                 m_session_str, reduce, bw_total, pdu_total,
1512                 m_bw_max, m_pdu_max);
1513
1514         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1515         timeout(reduce);       // call us reduce seconds later
1516     }
1517     else if (apdu->which == Z_GDU_Z3950)
1518         handle_incoming_Z_PDU(apdu->u.z3950);
1519     else if (apdu->which == Z_GDU_HTTP_Request)
1520         handle_incoming_HTTP(apdu->u.HTTP_Request);
1521 }
1522
1523 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1524 {
1525     if (m_max_record_retrieve)
1526     {
1527         if (apdu->which == Z_APDU_presentRequest)
1528         {
1529             Z_PresentRequest *pr = apdu->u.presentRequest;
1530             if (pr->numberOfRecordsRequested &&
1531                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1532                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1533         }
1534     }
1535 }
1536
1537 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1538                                                      int error,
1539                                                      const char *addinfo)
1540 {
1541     Z_Records *rec = (Z_Records *)
1542         odr_malloc (odr, sizeof(*rec));
1543     int *err = (int *)
1544         odr_malloc (odr, sizeof(*err));
1545     Z_DiagRec *drec = (Z_DiagRec *)
1546         odr_malloc (odr, sizeof(*drec));
1547     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1548         odr_malloc (odr, sizeof(*dr));
1549     *err = error;
1550     rec->which = Z_Records_NSD;
1551     rec->u.nonSurrogateDiagnostic = dr;
1552     dr->diagnosticSetId =
1553         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1554     dr->condition = err;
1555     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1556     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1557     return rec;
1558 }
1559
1560 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1561 {
1562     if (apdu->which == Z_APDU_searchRequest &&
1563         apdu->u.searchRequest->query &&
1564         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1565         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1566     {
1567         Z_RPNQuery *rpnquery = 0;
1568         Z_SearchRequest *sr = apdu->u.searchRequest;
1569         char *addinfo = 0;
1570
1571         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
1572                 sr->query->u.type_104->u.cql);
1573
1574         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1575                                           &rpnquery, odr_encode(),
1576                                           &addinfo);
1577         if (r == -3)
1578             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
1579         else if (r)
1580         {
1581             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1582             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1583
1584             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1585             new_apdu->u.searchResponse->records =
1586                 create_nonSurrogateDiagnostics(odr_encode(),
1587                                                yaz_diag_srw_to_bib1(r),
1588                                                addinfo);
1589             *new_apdu->u.searchResponse->searchStatus = 0;
1590
1591             send_to_client(new_apdu);
1592
1593             return 0;
1594         }
1595         else
1596         {
1597             sr->query->which = Z_Query_type_1;
1598             sr->query->u.type_1 = rpnquery;
1599         }
1600         return apdu;
1601     }
1602     return apdu;
1603 }
1604
1605 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1606 {
1607     if (apdu->which == Z_APDU_searchRequest)
1608     {
1609         Z_SearchRequest *sr = apdu->u.searchRequest;
1610         int err = 0;
1611         char *addinfo = 0;
1612
1613         Yaz_ProxyConfig *cfg = check_reconfigure();
1614         if (cfg)
1615             err = cfg->check_query(odr_encode(), m_default_target,
1616                                    sr->query, &addinfo);
1617         if (err)
1618         {
1619             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1620
1621             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1622             new_apdu->u.searchResponse->records =
1623                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1624             *new_apdu->u.searchResponse->searchStatus = 0;
1625
1626             send_to_client(new_apdu);
1627
1628             return 0;
1629         }
1630     }
1631     return apdu;
1632 }
1633
1634 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1635 {
1636     m_marcxml_flag = 0;
1637     if (apdu->which == Z_APDU_searchRequest)
1638     {
1639         Z_SearchRequest *sr = apdu->u.searchRequest;
1640         int err = 0;
1641         char *addinfo = 0;
1642         Yaz_ProxyConfig *cfg = check_reconfigure();
1643
1644         Z_RecordComposition rc_temp, *rc = 0;
1645         if (sr->smallSetElementSetNames)
1646         {
1647             rc_temp.which = Z_RecordComp_simple;
1648             rc_temp.u.simple = sr->smallSetElementSetNames;
1649             rc = &rc_temp;
1650         }
1651
1652         if (sr->preferredRecordSyntax)
1653         {
1654             struct oident *ent;
1655             ent = oid_getentbyoid(sr->preferredRecordSyntax);
1656             m_frontend_type = ent->value;
1657         }
1658         else
1659             m_frontend_type = VAL_NONE;
1660
1661         char *stylesheet_name = 0;
1662         if (cfg)
1663             err = cfg->check_syntax(odr_encode(),
1664                                     m_default_target,
1665                                     sr->preferredRecordSyntax, rc,
1666                                     &addinfo, &stylesheet_name, &m_schema,
1667                                     &m_backend_type, &m_backend_charset);
1668         if (stylesheet_name)
1669         {
1670             m_parent->low_socket_close();
1671
1672 #if HAVE_XSLT
1673             if (m_stylesheet_xsp)
1674                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
1675             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1676                                                        stylesheet_name);
1677 #endif
1678             m_stylesheet_offset = 0;
1679             xfree(stylesheet_name);
1680
1681             m_parent->low_socket_open();
1682         }
1683         if (err == -1)
1684         {
1685             sr->smallSetElementSetNames = 0;
1686             sr->mediumSetElementSetNames = 0;
1687             if (m_backend_type)
1688             {
1689
1690                 sr->preferredRecordSyntax =
1691                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
1692                                         m_backend_type);
1693             }
1694             else
1695                 sr->preferredRecordSyntax =
1696                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
1697                                            VAL_USMARC);
1698             m_marcxml_flag = 1;
1699         }
1700         else if (err)
1701         {
1702             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1703
1704             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1705             new_apdu->u.searchResponse->records =
1706                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1707             *new_apdu->u.searchResponse->searchStatus = 0;
1708
1709             send_to_client(new_apdu);
1710
1711             return 0;
1712         }
1713         else if (m_backend_type)
1714         {
1715             sr->preferredRecordSyntax =
1716                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
1717         }
1718     }
1719     else if (apdu->which == Z_APDU_presentRequest)
1720     {
1721         Z_PresentRequest *pr = apdu->u.presentRequest;
1722         int err = 0;
1723         char *addinfo = 0;
1724         Yaz_ProxyConfig *cfg = check_reconfigure();
1725
1726         if (pr->preferredRecordSyntax)
1727         {
1728             struct oident *ent;
1729             ent = oid_getentbyoid(pr->preferredRecordSyntax);
1730             m_frontend_type = ent->value;
1731         }
1732         else
1733             m_frontend_type = VAL_NONE;
1734
1735         char *stylesheet_name = 0;
1736         if (cfg)
1737             err = cfg->check_syntax(odr_encode(), m_default_target,
1738                                     pr->preferredRecordSyntax,
1739                                     pr->recordComposition,
1740                                     &addinfo, &stylesheet_name, &m_schema,
1741                                     &m_backend_type, &m_backend_charset);
1742         if (stylesheet_name)
1743         {
1744             m_parent->low_socket_close();
1745
1746 #if HAVE_XSLT
1747             if (m_stylesheet_xsp)
1748                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
1749             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1750                                                        stylesheet_name);
1751 #endif
1752             m_stylesheet_offset = 0;
1753             xfree(stylesheet_name);
1754
1755             m_parent->low_socket_open();
1756         }
1757         if (err == -1)
1758         {
1759             pr->recordComposition = 0;
1760             if (m_backend_type)
1761             {
1762
1763                 pr->preferredRecordSyntax =
1764                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
1765                                         m_backend_type);
1766             }
1767             else
1768                 pr->preferredRecordSyntax =
1769                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
1770                                            VAL_USMARC);
1771             m_marcxml_flag = 1;
1772         }
1773         else if (err)
1774         {
1775             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1776
1777             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1778             new_apdu->u.presentResponse->records =
1779                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1780             *new_apdu->u.presentResponse->presentStatus =
1781                 Z_PresentStatus_failure;
1782
1783             send_to_client(new_apdu);
1784
1785             return 0;
1786         }
1787         else if (m_backend_type)
1788         {
1789             pr->preferredRecordSyntax =
1790                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
1791         }
1792     }
1793     return apdu;
1794 }
1795
1796 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
1797 {
1798     if (!schema)
1799         return 0;
1800     Z_ElementSetNames *esn = (Z_ElementSetNames *)
1801         odr_malloc(o, sizeof(Z_ElementSetNames));
1802     esn->which = Z_ElementSetNames_generic;
1803     esn->u.generic = odr_strdup(o, schema);
1804     return esn;
1805 }
1806
1807 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
1808 {
1809     const char *t = 0;
1810     Yaz_ProxyConfig *cfg = check_reconfigure();
1811     if (cfg)
1812         t = cfg->get_explain_name(db, backend_db);
1813
1814     if (m_client && m_default_target && t && strcmp(m_default_target, t))
1815     {
1816         releaseClient();
1817     }
1818
1819     if (t)
1820     {
1821         xfree(m_default_target);
1822         m_default_target = xstrdup(t);
1823     }
1824 }
1825
1826 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1827 {
1828     if (m_s2z_odr_init)
1829     {
1830         odr_destroy(m_s2z_odr_init);
1831         m_s2z_odr_init = 0;
1832     }
1833     if (m_s2z_odr_search)
1834     {
1835         odr_destroy(m_s2z_odr_search);
1836         m_s2z_odr_search = 0;
1837     }
1838
1839     m_http_keepalive = 0;
1840     m_http_version = 0;
1841     if (!strcmp(hreq->version, "1.0"))
1842     {
1843         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1844         if (v && !strcmp(v, "Keep-Alive"))
1845             m_http_keepalive = 1;
1846         else
1847             m_http_keepalive = 0;
1848         m_http_version = "1.0";
1849     }
1850     else
1851     {
1852         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1853         if (v && !strcmp(v, "close"))
1854             m_http_keepalive = 0;
1855         else
1856             m_http_keepalive = 1;
1857         m_http_version = "1.1";
1858     }
1859
1860     Z_SRW_PDU *srw_pdu = 0;
1861     Z_SOAP *soap_package = 0;
1862     char *charset = 0;
1863     Z_SRW_diagnostic *diagnostic = 0;
1864     int num_diagnostic = 0;
1865     if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1866                        &charset) == 0
1867         || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1868                           &charset, &diagnostic, &num_diagnostic) == 0)
1869     {
1870         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
1871         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
1872         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
1873         m_s2z_init_apdu = 0;
1874         m_s2z_search_apdu = 0;
1875         m_s2z_present_apdu = 0;
1876
1877         m_s2z_stylesheet = 0;
1878
1879         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
1880         {
1881             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
1882
1883             const char *backend_db = srw_req->database;
1884             srw_get_client(srw_req->database, &backend_db);
1885
1886             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1887             // recordXPath unsupported.
1888             if (srw_req->recordXPath)
1889             {
1890                 yaz_add_srw_diagnostic(odr_decode(),
1891                                        &diagnostic, &num_diagnostic,
1892                                        72, 0);
1893             }
1894             // sort unsupported
1895             if (srw_req->sort_type != Z_SRW_sort_type_none)
1896             {
1897                 yaz_add_srw_diagnostic(odr_decode(),
1898                                        &diagnostic, &num_diagnostic,
1899                                        80, 0);
1900             }
1901             // save stylesheet
1902             if (srw_req->stylesheet)
1903                 m_s2z_stylesheet =
1904                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1905
1906             // set packing for response records ..
1907             if (srw_req->recordPacking &&
1908                 !strcmp(srw_req->recordPacking, "xml"))
1909                 m_s2z_packing = Z_SRW_recordPacking_XML;
1910             else
1911                 m_s2z_packing = Z_SRW_recordPacking_string;
1912
1913             if (num_diagnostic)
1914             {
1915                 Z_SRW_PDU *srw_pdu =
1916                     yaz_srw_get(odr_encode(),
1917                                 Z_SRW_searchRetrieve_response);
1918                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1919
1920                 srw_res->diagnostics = diagnostic;
1921                 srw_res->num_diagnostics = num_diagnostic;
1922                 send_srw_response(srw_pdu);
1923                 return;
1924             }
1925
1926             // prepare search PDU
1927             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
1928                                           Z_APDU_searchRequest);
1929             Z_SearchRequest *z_searchRequest =
1930                 m_s2z_search_apdu->u.searchRequest;
1931
1932             z_searchRequest->num_databaseNames = 1;
1933             z_searchRequest->databaseNames = (char**)
1934                 odr_malloc(m_s2z_odr_search, sizeof(char *));
1935             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
1936                                                            backend_db);
1937
1938             // query transformation
1939             Z_Query *query = (Z_Query *)
1940                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
1941             z_searchRequest->query = query;
1942
1943             if (srw_req->query_type == Z_SRW_query_type_cql)
1944             {
1945                 Z_External *ext = (Z_External *)
1946                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
1947                 ext->direct_reference =
1948                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
1949                 ext->indirect_reference = 0;
1950                 ext->descriptor = 0;
1951                 ext->which = Z_External_CQL;
1952                 ext->u.cql = srw_req->query.cql;
1953
1954                 query->which = Z_Query_type_104;
1955                 query->u.type_104 =  ext;
1956             }
1957             else if (srw_req->query_type == Z_SRW_query_type_pqf)
1958             {
1959                 Z_RPNQuery *RPNquery;
1960                 YAZ_PQF_Parser pqf_parser;
1961
1962                 pqf_parser = yaz_pqf_create ();
1963
1964                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
1965                                           srw_req->query.pqf);
1966                 if (!RPNquery)
1967                 {
1968                     const char *pqf_msg;
1969                     size_t off;
1970                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
1971                     yaz_log(YLOG_LOG, "%*s^\n", off+4, "");
1972                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
1973
1974                     send_to_srw_client_error(10, 0);
1975                     return;
1976                 }
1977                 query->which = Z_Query_type_1;
1978                 query->u.type_1 =  RPNquery;
1979
1980                 yaz_pqf_destroy (pqf_parser);
1981             }
1982             else
1983             {
1984                 send_to_srw_client_error(7, "query");
1985                 return;
1986             }
1987
1988             // present
1989             m_s2z_present_apdu = 0;
1990             int max = 0;
1991             if (srw_req->maximumRecords)
1992                 max = *srw_req->maximumRecords;
1993             int start = 1;
1994             if (srw_req->startRecord)
1995                 start = *srw_req->startRecord;
1996             if (max > 0)
1997             {
1998                 // Some backend, such as Voyager doesn't honor piggyback
1999                 // So we use present always (0 &&).
2000                 if (0 && start <= 1)  // Z39.50 piggyback
2001                 {
2002                     *z_searchRequest->smallSetUpperBound = max;
2003                     *z_searchRequest->mediumSetPresentNumber = max;
2004                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2005
2006                     z_searchRequest->preferredRecordSyntax =
2007                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2008                                                VAL_TEXT_XML);
2009                     if (srw_req->recordSchema)
2010                     {
2011                         z_searchRequest->smallSetElementSetNames =
2012                             z_searchRequest->mediumSetElementSetNames =
2013                             mk_esn_from_schema(m_s2z_odr_search,
2014                                                srw_req->recordSchema);
2015                     }
2016                 }
2017                 else   // Z39.50 present
2018                 {
2019                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2020                                                    Z_APDU_presentRequest);
2021                     Z_PresentRequest *z_presentRequest =
2022                         m_s2z_present_apdu->u.presentRequest;
2023                     *z_presentRequest->resultSetStartPoint = start;
2024                     *z_presentRequest->numberOfRecordsRequested = max;
2025                     z_presentRequest->preferredRecordSyntax =
2026                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2027                                                VAL_TEXT_XML);
2028                     if (srw_req->recordSchema)
2029                     {
2030                         z_presentRequest->recordComposition =
2031                             (Z_RecordComposition *)
2032                             odr_malloc(m_s2z_odr_search,
2033                                        sizeof(Z_RecordComposition));
2034                         z_presentRequest->recordComposition->which =
2035                             Z_RecordComp_simple;
2036                         z_presentRequest->recordComposition->u.simple =
2037                             mk_esn_from_schema(m_s2z_odr_search,
2038                                                srw_req->recordSchema);
2039                     }
2040                 }
2041             }
2042             if (!m_client)
2043             {
2044                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2045                                             Z_APDU_initRequest);
2046
2047                 // prevent m_initRequest_apdu memory from being grabbed
2048                 // in Yaz_Proxy::handle_incoming_Z_PDU
2049                 m_initRequest_apdu = m_s2z_init_apdu;
2050                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2051                 return;
2052             }
2053             else
2054             {
2055                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2056                 return;
2057             }
2058         }
2059         else if (srw_pdu->which == Z_SRW_explain_request)
2060         {
2061             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2062
2063             const char *backend_db = srw_req->database;
2064             srw_get_client(srw_req->database, &backend_db);
2065
2066             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2067
2068             // save stylesheet
2069             if (srw_req->stylesheet)
2070                 m_s2z_stylesheet =
2071                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2072
2073             if (srw_req->recordPacking &&
2074                 !strcmp(srw_req->recordPacking, "xml"))
2075                 m_s2z_packing = Z_SRW_recordPacking_XML;
2076             else
2077                 m_s2z_packing = Z_SRW_recordPacking_string;
2078
2079             if (num_diagnostic)
2080             {
2081                 send_srw_explain_response(diagnostic, num_diagnostic);
2082                 return;
2083             }
2084
2085             if (!m_client)
2086             {
2087                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2088                                             Z_APDU_initRequest);
2089
2090                 // prevent m_initRequest_apdu memory from being grabbed
2091                 // in Yaz_Proxy::handle_incoming_Z_PDU
2092                 m_initRequest_apdu = m_s2z_init_apdu;
2093                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2094             }
2095             else
2096                 send_srw_explain_response(0, 0);
2097             return;
2098         }
2099         else if (srw_pdu->which == Z_SRW_scan_request)
2100         {
2101             m_s2z_database = odr_strdup(m_s2z_odr_init,
2102                                         srw_pdu->u.scan_request->database);
2103
2104             yaz_add_srw_diagnostic(odr_decode(),
2105                                    &diagnostic, &num_diagnostic,
2106                                    4, "scan");
2107             Z_SRW_PDU *srw_pdu =
2108                 yaz_srw_get(odr_encode(),
2109                             Z_SRW_scan_response);
2110             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2111
2112             srw_res->diagnostics = diagnostic;
2113             srw_res->num_diagnostics = num_diagnostic;
2114             send_srw_response(srw_pdu);
2115             return;
2116         }
2117         else
2118         {
2119             m_s2z_database = 0;
2120
2121             send_to_srw_client_error(4, 0);
2122         }
2123     }
2124     int len = 0;
2125     Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
2126     timeout(0);
2127     send_GDU(p, &len);
2128 }
2129
2130 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2131 {
2132     Z_ReferenceId **refid = get_referenceIdP(apdu);
2133     nmem_reset(m_referenceId_mem);
2134     if (refid && *refid)
2135     {
2136         m_referenceId = (Z_ReferenceId *)
2137             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2138         m_referenceId->len = m_referenceId->size = (*refid)->len;
2139         m_referenceId->buf = (unsigned char *)
2140             nmem_malloc(m_referenceId_mem, (*refid)->len);
2141         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2142     }
2143     else
2144         m_referenceId = 0;
2145
2146     if (!m_client && m_invalid_session)
2147     {
2148         m_apdu_invalid_session = apdu;
2149         m_mem_invalid_session = odr_extract_mem(odr_decode());
2150         apdu = m_initRequest_apdu;
2151     }
2152
2153     // Determine our client.
2154     Z_OtherInformation **oi;
2155     get_otherInfoAPDU(apdu, &oi);
2156     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2157     if (!m_client)
2158     {
2159         delete this;
2160         return;
2161     }
2162     m_client->m_server = this;
2163
2164     if (apdu->which == Z_APDU_initRequest)
2165     {
2166         if (apdu->u.initRequest->implementationId)
2167             yaz_log(YLOG_LOG, "%simplementationId: %s",
2168                     m_session_str, apdu->u.initRequest->implementationId);
2169         if (apdu->u.initRequest->implementationName)
2170             yaz_log(YLOG_LOG, "%simplementationName: %s",
2171                     m_session_str, apdu->u.initRequest->implementationName);
2172         if (apdu->u.initRequest->implementationVersion)
2173             yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2174                     m_session_str, apdu->u.initRequest->implementationVersion);
2175         if (m_initRequest_apdu == 0)
2176         {
2177             if (m_initRequest_mem)
2178                 nmem_destroy(m_initRequest_mem);
2179             m_initRequest_apdu = apdu;
2180             m_initRequest_mem = odr_extract_mem(odr_decode());
2181
2182             m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2183                 preferredMessageSize;
2184             *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2185             m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2186                 maximumRecordSize;
2187             *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2188
2189             // save init options for the response..
2190             m_initRequest_options = apdu->u.initRequest->options;
2191
2192             apdu->u.initRequest->options =
2193                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2194                                            sizeof(Odr_bitmask));
2195             ODR_MASK_ZERO(apdu->u.initRequest->options);
2196             int i;
2197             for (i = 0; i<= 24; i++)
2198                 ODR_MASK_SET(apdu->u.initRequest->options, i);
2199             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2200                            Z_Options_negotiationModel);
2201             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2202                            Z_Options_concurrentOperations);
2203
2204             // make new version
2205             m_initRequest_version = apdu->u.initRequest->protocolVersion;
2206             apdu->u.initRequest->protocolVersion =
2207                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2208                                            sizeof(Odr_bitmask));
2209             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2210
2211             for (i = 0; i<= 8; i++)
2212                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2213         }
2214         if (m_client->m_init_flag)
2215         {
2216             if (handle_init_response_for_invalid_session(apdu))
2217                 return;
2218             if (m_client->m_initResponse)
2219             {
2220                 Z_APDU *apdu2 = m_client->m_initResponse;
2221                 apdu2->u.initResponse->otherInfo = 0;
2222                 if (m_client->m_cookie && *m_client->m_cookie)
2223                     set_otherInformationString(apdu2, VAL_COOKIE, 1,
2224                                                m_client->m_cookie);
2225                 apdu2->u.initResponse->referenceId =
2226                     apdu->u.initRequest->referenceId;
2227                 apdu2->u.initResponse->options = m_client->m_initResponse_options;
2228                 apdu2->u.initResponse->protocolVersion =
2229                     m_client->m_initResponse_version;
2230
2231                 send_to_client(apdu2);
2232                 return;
2233             }
2234         }
2235         m_client->m_init_flag = 1;
2236     }
2237     handle_max_record_retrieve(apdu);
2238
2239     if (apdu)
2240         apdu = handle_syntax_validation(apdu);
2241
2242     if (apdu)
2243         apdu = handle_query_transformation(apdu);
2244
2245     if (apdu)
2246         apdu = handle_query_validation(apdu);
2247
2248     if (apdu)
2249         apdu = result_set_optimize(apdu);
2250     if (!apdu)
2251     {
2252         m_client->timeout(m_target_idletime);  // mark it active even
2253         // though we didn't use it
2254         return;
2255     }
2256
2257     // delete other info construct completely if 0 elements
2258     get_otherInfoAPDU(apdu, &oi);
2259     if (*oi && (*oi)->num_elements == 0)
2260         *oi = 0;
2261
2262     if (apdu->which == Z_APDU_presentRequest &&
2263         m_client->m_resultSetStartPoint == 0)
2264     {
2265         Z_PresentRequest *pr = apdu->u.presentRequest;
2266         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2267         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2268     } else {
2269         m_client->m_resultSetStartPoint = 0;
2270     }
2271     if (m_client->send_to_target(apdu) < 0)
2272     {
2273         delete m_client;
2274         m_client = 0;
2275         delete this;
2276     }
2277     else
2278         m_client->m_waiting = 1;
2279 }
2280
2281 void Yaz_Proxy::connectNotify()
2282 {
2283 }
2284
2285 void Yaz_Proxy::releaseClient()
2286 {
2287     xfree(m_proxyTarget);
2288     m_proxyTarget = 0;
2289     m_invalid_session = 0;
2290     // only keep if keep_alive flag is set...
2291     if (m_client &&
2292         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2293         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2294         m_client->m_waiting == 0)
2295     {
2296         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2297                  m_session_str,
2298                  m_client->get_hostname());
2299         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2300                 m_session_str, m_client->m_pdu_recv,
2301                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2302                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2303         assert (m_client->m_waiting != 2);
2304         // Tell client (if any) that no server connection is there..
2305         m_client->m_server = 0;
2306         m_client = 0;
2307     }
2308     else if (m_client)
2309     {
2310         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
2311                  m_session_str,
2312                  m_client->get_hostname());
2313         assert (m_client->m_waiting != 2);
2314         delete m_client;
2315         m_client = 0;
2316     }
2317     else if (!m_parent)
2318     {
2319         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
2320                  m_session_str);
2321         assert (m_parent);
2322     }
2323     else
2324     {
2325         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
2326                  m_session_str);
2327     }
2328     if (m_parent)
2329         m_parent->pre_init();
2330 }
2331
2332 void Yaz_Proxy::shutdown()
2333 {
2334     releaseClient();
2335     delete this;
2336 }
2337
2338 const char *Yaz_ProxyClient::get_session_str()
2339 {
2340     if (!m_server)
2341         return "0 ";
2342     return m_server->get_session_str();
2343 }
2344
2345 void Yaz_ProxyClient::shutdown()
2346 {
2347     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2348              get_hostname());
2349     delete m_server;
2350     delete this;
2351 }
2352
2353 void Yaz_Proxy::failNotify()
2354 {
2355     inc_request_no();
2356     yaz_log (YLOG_LOG, "%sConnection closed by client",
2357              get_session_str());
2358     shutdown();
2359 }
2360
2361 void Yaz_ProxyClient::failNotify()
2362 {
2363     if (m_server)
2364         m_server->inc_request_no();
2365     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
2366              get_session_str(), get_hostname());
2367     shutdown();
2368 }
2369
2370 void Yaz_ProxyClient::connectNotify()
2371 {
2372     const char *s = get_session_str();
2373     const char *h = get_hostname();
2374     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2375              m_target_idletime);
2376     timeout(m_target_idletime);
2377     if (!m_server)
2378         pre_init_client();
2379 }
2380
2381 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2382                                                   *the_PDU_Observable, int fd)
2383 {
2384     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2385 }
2386
2387 Yaz_ProxyClient::~Yaz_ProxyClient()
2388 {
2389     if (m_prev)
2390         *m_prev = m_next;
2391     if (m_next)
2392         m_next->m_prev = m_prev;
2393     m_waiting = 2;     // for debugging purposes only.
2394     odr_destroy(m_init_odr);
2395     delete m_last_query;
2396     xfree (m_last_resultSetId);
2397     xfree (m_cookie);
2398 }
2399
2400 void Yaz_ProxyClient::pre_init_client()
2401 {
2402     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2403     Z_InitRequest *req = apdu->u.initRequest;
2404
2405     int i;
2406     for (i = 0; i<= 24; i++)
2407         ODR_MASK_SET(req->options, i);
2408     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2409                    Z_Options_negotiationModel);
2410     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2411                    Z_Options_concurrentOperations);
2412     for (i = 0; i<= 10; i++)
2413         ODR_MASK_SET(req->protocolVersion, i);
2414
2415     if (send_to_target(apdu) < 0)
2416     {
2417         delete this;
2418     }
2419     else
2420     {
2421         m_waiting = 1;
2422         m_init_flag = 1;
2423     }
2424 }
2425
2426 void Yaz_Proxy::pre_init()
2427 {
2428     int i;
2429     const char *name = 0;
2430     const char *zurl_in_use[MAX_ZURL_PLEX];
2431     int limit_bw, limit_pdu, limit_req;
2432     int target_idletime, client_idletime;
2433     int max_clients;
2434     int keepalive_limit_bw, keepalive_limit_pdu;
2435     int pre_init;
2436     const char *cql2rpn = 0;
2437     const char *authentication = 0;
2438
2439     Yaz_ProxyConfig *cfg = check_reconfigure();
2440
2441     zurl_in_use[0] = 0;
2442
2443     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2444         set_APDU_yazlog(1);
2445     else
2446         set_APDU_yazlog(0);
2447
2448     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2449                                           &limit_bw, &limit_pdu, &limit_req,
2450                                           &target_idletime, &client_idletime,
2451                                           &max_clients,
2452                                           &keepalive_limit_bw,
2453                                           &keepalive_limit_pdu,
2454                                           &pre_init,
2455                                           &cql2rpn,
2456                                           &authentication) ; i++)
2457     {
2458         if (pre_init)
2459         {
2460             int j;
2461             for (j = 0; zurl_in_use[j]; j++)
2462             {
2463                 Yaz_ProxyClient *c;
2464                 int spare = 0;
2465                 int spare_waiting = 0;
2466                 int in_use = 0;
2467                 int other = 0;
2468                 for (c = m_clientPool; c; c = c->m_next)
2469                 {
2470                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
2471                     {
2472                         if (c->m_cookie == 0)
2473                         {
2474                             if (c->m_server == 0)
2475                                 if (c->m_waiting)
2476                                     spare_waiting++;
2477                                 else
2478                                     spare++;
2479                             else
2480                                 in_use++;
2481                         }
2482                         else
2483                             other++;
2484                     }
2485                 }
2486                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2487                         "sparew=%d preinit=%d",m_session_str,
2488                         name, zurl_in_use[j], in_use, other,
2489                         spare, spare_waiting, pre_init);
2490                 if (spare + spare_waiting < pre_init)
2491                 {
2492                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2493                     c->m_next = m_clientPool;
2494                     if (c->m_next)
2495                         c->m_next->m_prev = &c->m_next;
2496                     m_clientPool = c;
2497                     c->m_prev = &m_clientPool;
2498
2499                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
2500                         c->set_APDU_yazlog(1);
2501                     else
2502                         c->set_APDU_yazlog(0);
2503
2504                     if (c->client(zurl_in_use[j]))
2505                     {
2506                         timeout(60);
2507                         delete c;
2508                         return;
2509                     }
2510                     c->timeout(30);
2511                     c->m_waiting = 1;
2512                     c->m_target_idletime = target_idletime;
2513                     c->m_seqno = m_seqno++;
2514                 }
2515             }
2516         }
2517     }
2518 }
2519
2520 void Yaz_Proxy::timeoutNotify()
2521 {
2522     if (m_parent)
2523     {
2524         if (m_bw_hold_PDU)
2525         {
2526             timeout(m_client_idletime);
2527             Z_GDU *apdu = m_bw_hold_PDU;
2528             m_bw_hold_PDU = 0;
2529
2530             if (apdu->which == Z_GDU_Z3950)
2531                 handle_incoming_Z_PDU(apdu->u.z3950);
2532             else if (apdu->which == Z_GDU_HTTP_Request)
2533                 handle_incoming_HTTP(apdu->u.HTTP_Request);
2534         }
2535         else if (m_stylesheet_nprl)
2536             convert_xsl_delay();
2537         else
2538         {
2539             inc_request_no();
2540
2541             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2542             shutdown();
2543         }
2544     }
2545     else
2546     {
2547         timeout(600);
2548         pre_init();
2549     }
2550 }
2551
2552 void Yaz_Proxy::markInvalid()
2553 {
2554     m_client = 0;
2555     m_invalid_session = 1;
2556 }
2557
2558 void Yaz_ProxyClient::timeoutNotify()
2559 {
2560     if (m_server)
2561         m_server->inc_request_no();
2562
2563     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2564              get_hostname());
2565     m_waiting = 1;
2566     m_root->pre_init();
2567     if (m_server && m_init_flag)
2568     {
2569         // target timed out in a session that was properly initialized
2570         // server object stay alive but we mark it as invalid so it
2571         // gets initialized again
2572         m_server->markInvalid();
2573         m_server = 0;
2574     }
2575     shutdown();
2576 }
2577
2578 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2579                                  Yaz_Proxy *parent) :
2580     Yaz_Z_Assoc (the_PDU_Observable)
2581 {
2582     m_cookie = 0;
2583     m_next = 0;
2584     m_prev = 0;
2585     m_init_flag = 0;
2586     m_last_query = 0;
2587     m_last_resultSetId = 0;
2588     m_last_resultCount = 0;
2589     m_last_ok = 0;
2590     m_sr_transform = 0;
2591     m_waiting = 0;
2592     m_init_odr = odr_createmem (ODR_DECODE);
2593     m_initResponse = 0;
2594     m_initResponse_options = 0;
2595     m_initResponse_version = 0;
2596     m_initResponse_preferredMessageSize = 0;
2597     m_initResponse_maximumRecordSize = 0;
2598     m_resultSetStartPoint = 0;
2599     m_bytes_sent = m_bytes_recv = 0;
2600     m_pdu_recv = 0;
2601     m_server = 0;
2602     m_seqno = 0;
2603     m_target_idletime = 600;
2604     m_root = parent;
2605 }
2606
2607 const char *Yaz_Proxy::option(const char *name, const char *value)
2608 {
2609     if (!strcmp (name, "optimize")) {
2610         if (value) {
2611             xfree (m_optimize);
2612             m_optimize = xstrdup (value);
2613         }
2614         return m_optimize;
2615     }
2616     return 0;
2617 }
2618
2619 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2620 {
2621
2622 }
2623
2624 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2625 {
2626     if (apdu->which == Z_GDU_Z3950)
2627         recv_Z_PDU(apdu->u.z3950, len);
2628     else if (apdu->which == Z_GDU_HTTP_Response)
2629         recv_HTTP_response(apdu->u.HTTP_Response, len);
2630     else
2631         shutdown();
2632 }
2633
2634 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2635 {
2636     if (!m_invalid_session)
2637         return 0;
2638     m_invalid_session = 0;
2639     handle_incoming_Z_PDU(m_apdu_invalid_session);
2640     assert (m_mem_invalid_session);
2641     nmem_destroy(m_mem_invalid_session);
2642     m_mem_invalid_session = 0;
2643     return 1;
2644 }
2645
2646 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2647 {
2648     m_bytes_recv += len;
2649
2650     m_pdu_recv++;
2651     m_waiting = 0;
2652     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2653         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2654                  apdu_name(apdu), get_hostname(), len);
2655     if (apdu->which == Z_APDU_initResponse)
2656     {
2657         if (!m_server)  // if this is a pre init session , check for more
2658             m_root->pre_init();
2659         NMEM nmem = odr_extract_mem (odr_decode());
2660         odr_reset (m_init_odr);
2661         nmem_transfer (m_init_odr->mem, nmem);
2662         m_initResponse = apdu;
2663         m_initResponse_options = apdu->u.initResponse->options;
2664         m_initResponse_version = apdu->u.initResponse->protocolVersion;
2665         m_initResponse_preferredMessageSize =
2666             *apdu->u.initResponse->preferredMessageSize;
2667         m_initResponse_maximumRecordSize =
2668             *apdu->u.initResponse->maximumRecordSize;
2669
2670         Z_InitResponse *ir = apdu->u.initResponse;
2671         char *im0 = ir->implementationName;
2672
2673         char *im1 = (char*)
2674             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2675         *im1 = '\0';
2676         if (im0)
2677         {
2678             strcat(im1, im0);
2679             strcat(im1, " ");
2680         }
2681         strcat(im1, "(YAZ Proxy)");
2682         ir->implementationName = im1;
2683
2684         nmem_destroy (nmem);
2685
2686         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2687             return;
2688     }
2689     if (apdu->which == Z_APDU_searchResponse)
2690     {
2691         Z_SearchResponse *sr = apdu->u.searchResponse;
2692         m_last_resultCount = *sr->resultCount;
2693         int status = *sr->searchStatus;
2694         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2695         {
2696             m_last_ok = 1;
2697
2698             if (sr->records && sr->records->which == Z_Records_DBOSD)
2699             {
2700                 m_cache.add(odr_decode(),
2701                             sr->records->u.databaseOrSurDiagnostics, 1,
2702                             *sr->resultCount);
2703             }
2704         }
2705     }
2706     if (apdu->which == Z_APDU_presentResponse)
2707     {
2708         Z_PresentResponse *pr = apdu->u.presentResponse;
2709         if (m_sr_transform)
2710         {
2711             m_sr_transform = 0;
2712             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2713             Z_SearchResponse *sr = new_apdu->u.searchResponse;
2714             sr->referenceId = pr->referenceId;
2715             *sr->resultCount = m_last_resultCount;
2716             sr->records = pr->records;
2717             sr->nextResultSetPosition = pr->nextResultSetPosition;
2718             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2719             apdu = new_apdu;
2720         }
2721         if (pr->records &&
2722             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2723         {
2724             m_cache.add(odr_decode(),
2725                         pr->records->u.databaseOrSurDiagnostics,
2726                         m_resultSetStartPoint, -1);
2727             m_resultSetStartPoint = 0;
2728         }
2729     }
2730     if (m_cookie)
2731         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2732     if (m_server)
2733     {
2734         m_server->send_to_client(apdu);
2735     }
2736     if (apdu->which == Z_APDU_close)
2737     {
2738         shutdown();
2739     }
2740 }
2741
2742 void Yaz_Proxy::low_socket_close()
2743 {
2744 #if WIN32
2745 #else
2746     int i;
2747     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
2748         if  (m_lo_fd[i] >= 0)
2749             ::close(m_lo_fd[i]);
2750 #endif
2751 }
2752
2753 void Yaz_Proxy::low_socket_open()
2754 {
2755 #if WIN32
2756 #else
2757     int i;
2758     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
2759         m_lo_fd[i] = open("/dev/null", O_RDONLY);
2760 #endif
2761 }
2762
2763 int Yaz_Proxy::server(const char *addr)
2764 {
2765     int r = Yaz_Z_Assoc::server(addr);
2766     if (!r)
2767     {
2768         yaz_log(YLOG_LOG, "%sStarted proxy "
2769 #ifdef VERSION
2770             VERSION
2771 #endif
2772             " on %s", m_session_str, addr);
2773         timeout(1);
2774     }
2775     return r;
2776 }
2777