src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.8 2004-10-19 18:34:15 adam Exp $
   2    Copyright (c) 1998-2004, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #else
  24 #include <unistd.h>
  25 #endif
  26
  27 #if HAVE_GETTIMEOFDAY
  28 #include <sys/time.h>
  29 #endif
  30
  31 #include <assert.h>
  32 #include <time.h>
  33 #include <sys/types.h>
  34 #include <fcntl.h>
  35
  36 #include <yaz/srw.h>
  37 #include <yaz/marcdisp.h>
  38 #include <yaz/yaz-iconv.h>
  39 #include <yaz/log.h>
  40 #include <yaz/diagbib1.h>
  41 #include <yazproxy/proxy.h>
  42 #include <yaz/pquery.h>
  43 #include <yaz/otherinfo.h>
  44
  45 #if HAVE_XSLT
  46 #include <libxml/parser.h>
  47 #include <libxml/tree.h>
  48 #include <libxslt/xsltutils.h>
  49 #include <libxslt/transform.h>
  50 #endif
  51
  52 static const char *apdu_name(Z_APDU *apdu)
  53 {
  54     switch (apdu->which)
  55     {
  56     case Z_APDU_initRequest:
  57         return "initRequest";
  58     case Z_APDU_initResponse:
  59         return "initResponse";
  60     case Z_APDU_searchRequest:
  61         return "searchRequest";
  62     case Z_APDU_searchResponse:
  63         return "searchResponse";
  64     case Z_APDU_presentRequest:
  65         return "presentRequest";
  66     case Z_APDU_presentResponse:
  67         return "presentResponse";
  68     case Z_APDU_deleteResultSetRequest:
  69         return "deleteResultSetRequest";
  70     case Z_APDU_deleteResultSetResponse:
  71         return "deleteResultSetResponse";
  72     case Z_APDU_scanRequest:
  73         return "scanRequest";
  74     case Z_APDU_scanResponse:
  75         return "scanResponse";
  76     case Z_APDU_sortRequest:
  77         return "sortRequest";
  78     case Z_APDU_sortResponse:
  79         return "sortResponse";
  80     case Z_APDU_extendedServicesRequest:
  81         return "extendedServicesRequest";
  82     case Z_APDU_extendedServicesResponse:
  83         return "extendedServicesResponse";
  84     case Z_APDU_close:
  85         return "close";
  86     }
  87     return "other";
  88 }
  89
  90 static const char *gdu_name(Z_GDU *gdu)
  91 {
  92     switch(gdu->which)
  93     {
  94     case Z_GDU_Z3950:
  95         return apdu_name(gdu->u.z3950);
  96     case Z_GDU_HTTP_Request:
  97         return "HTTP Request";
  98     case Z_GDU_HTTP_Response:
  99         return "HTTP Response";
 100     }
 101     return "Unknown request/response";
 102 }
 103
 104 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
 105                      Yaz_Proxy *parent) :
 106     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 107 {
 108     m_PDU_Observable = the_PDU_Observable;
 109     m_client = 0;
 110     m_parent = parent;
 111     m_clientPool = 0;
 112     m_seqno = 1;
 113     m_keepalive_limit_bw = 500000;
 114     m_keepalive_limit_pdu = 1000;
 115     m_proxyTarget = 0;
 116     m_default_target = 0;
 117     m_proxy_authentication = 0;
 118     m_max_clients = 150;
 119     m_log_mask = 0;
 120     m_seed = time(0);
 121     m_client_idletime = 600;
 122     m_target_idletime = 600;
 123     m_optimize = xstrdup ("1");
 124     strcpy(m_session_str, "0 ");
 125     m_session_no=0;
 126     m_bytes_sent = 0;
 127     m_bytes_recv = 0;
 128     m_bw_hold_PDU = 0;
 129     m_bw_max = 0;
 130     m_pdu_max = 0;
 131     m_max_record_retrieve = 0;
 132     m_reconfig_flag = 0;
 133     m_config_fname = 0;
 134     m_request_no = 0;
 135     m_invalid_session = 0;
 136     m_referenceId = 0;
 137     m_referenceId_mem = nmem_create();
 138     m_config = 0;
 139     m_marcxml_flag = 0;
 140     m_stylesheet_xsp = 0;
 141     m_stylesheet_nprl = 0;
 142     m_s2z_stylesheet = 0;
 143     m_s2z_database = 0;
 144     m_schema = 0;
 145     m_backend_type = 0;
 146     m_backend_charset = 0;
 147     m_frontend_type = 0;
 148     m_initRequest_apdu = 0;
 149     m_initRequest_mem = 0;
 150     m_initRequest_preferredMessageSize = 0;
 151     m_initRequest_maximumRecordSize = 0;
 152     m_initRequest_options = 0;
 153     m_initRequest_version = 0;
 154     m_apdu_invalid_session = 0;
 155     m_mem_invalid_session = 0;
 156     m_s2z_odr_init = 0;
 157     m_s2z_odr_search = 0;
 158     m_s2z_init_apdu = 0;
 159     m_s2z_search_apdu = 0;
 160     m_s2z_present_apdu = 0;
 161     m_http_keepalive = 0;
 162     m_http_version = 0;
 163     m_soap_ns = 0;
 164     m_s2z_packing = Z_SRW_recordPacking_string;
 165 #if HAVE_GETTIMEOFDAY
 166     m_time_tv = xmalloc(sizeof(struct timeval));
 167     struct timeval *tv = (struct timeval *) m_time_tv;
 168     tv->tv_sec = 0;
 169     tv->tv_usec = 0;
 170 #else
 171     m_time_tv = 0;
 172 #endif
 173     if (!m_parent)
 174         low_socket_open();
 175 }
 176
 177 Yaz_Proxy::~Yaz_Proxy()
 178 {
 179     yaz_log(LOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 180             m_bytes_sent, m_bytes_recv);
 181     nmem_destroy(m_initRequest_mem);
 182     nmem_destroy(m_mem_invalid_session);
 183     nmem_destroy(m_referenceId_mem);
 184
 185     xfree(m_proxyTarget);
 186     xfree(m_default_target);
 187     xfree(m_proxy_authentication);
 188     xfree(m_optimize);
 189
 190 #if HAVE_XSLT
 191     if (m_stylesheet_xsp)
 192         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 193 #endif
 194     xfree (m_time_tv);
 195
 196     xfree (m_schema);
 197     xfree (m_backend_type);
 198     xfree (m_backend_charset);
 199     if (m_s2z_odr_init)
 200         odr_destroy(m_s2z_odr_init);
 201     if (m_s2z_odr_search)
 202         odr_destroy(m_s2z_odr_search);
 203     if (!m_parent)
 204         low_socket_close();
 205     delete m_config;
 206 }
 207
 208 int Yaz_Proxy::set_config(const char *config)
 209 {
 210     delete m_config;
 211     m_config = new Yaz_ProxyConfig();
 212     xfree(m_config_fname);
 213     m_config_fname = xstrdup(config);
 214     int r = m_config->read_xml(config);
 215     if (!r)
 216         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 217     return r;
 218 }
 219
 220 void Yaz_Proxy::set_default_target(const char *target)
 221 {
 222     xfree (m_default_target);
 223     m_default_target = 0;
 224     if (target)
 225         m_default_target = (char *) xstrdup (target);
 226 }
 227
 228 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 229 {
 230     xfree (m_proxy_authentication);
 231     m_proxy_authentication = 0;
 232     if (auth)
 233         m_proxy_authentication = (char *) xstrdup (auth);
 234 }
 235
 236 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 237 {
 238     if (m_parent)
 239         return m_parent->check_reconfigure();
 240
 241     Yaz_ProxyConfig *cfg = m_config;
 242     if (m_reconfig_flag)
 243     {
 244         yaz_log(LOG_LOG, "reconfigure");
 245         yaz_log_reopen();
 246         if (m_config_fname && cfg)
 247         {
 248             yaz_log(LOG_LOG, "reconfigure config %s", m_config_fname);
 249             int r = cfg->read_xml(m_config_fname);
 250             if (r)
 251                 yaz_log(LOG_WARN, "reconfigure failed");
 252             else
 253             {
 254                 m_log_mask = 0;
 255                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 256             }
 257         }
 258         else
 259             yaz_log(LOG_LOG, "reconfigure");
 260         m_reconfig_flag = 0;
 261     }
 262     return cfg;
 263 }
 264
 265 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 266                                             *the_PDU_Observable, int fd)
 267 {
 268     check_reconfigure();
 269     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 270     new_proxy->m_config = 0;
 271     new_proxy->m_config_fname = 0;
 272     new_proxy->timeout(m_client_idletime);
 273     new_proxy->m_target_idletime = m_target_idletime;
 274     new_proxy->set_default_target(m_default_target);
 275     new_proxy->m_max_clients = m_max_clients;
 276     new_proxy->m_log_mask = m_log_mask;
 277     new_proxy->set_APDU_log(get_APDU_log());
 278     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 279         new_proxy->set_APDU_yazlog(1);
 280     else
 281         new_proxy->set_APDU_yazlog(0);
 282     new_proxy->set_proxy_authentication(m_proxy_authentication);
 283     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 284     m_session_no++;
 285     yaz_log (LOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 286              the_PDU_Observable->getpeername());
 287     return new_proxy;
 288 }
 289
 290 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 291 {
 292     int oid[OID_SIZE];
 293     Z_OtherInformationUnit *oi;
 294     struct oident ent;
 295     ent.proto = PROTO_Z3950;
 296     ent.oclass = CLASS_USERINFO;
 297     ent.value = (oid_value) VAL_COOKIE;
 298     assert (oid_ent_to_oid (&ent, oid));
 299
 300     if (oid_ent_to_oid (&ent, oid) &&
 301         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 302         oi->which == Z_OtherInfo_characterInfo)
 303         return oi->information.characterInfo;
 304     return 0;
 305 }
 306
 307 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 308 {
 309     int oid[OID_SIZE];
 310     Z_OtherInformationUnit *oi;
 311     struct oident ent;
 312     ent.proto = PROTO_Z3950;
 313     ent.oclass = CLASS_USERINFO;
 314     ent.value = (oid_value) VAL_PROXY;
 315     if (oid_ent_to_oid (&ent, oid) &&
 316         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 317         oi->which == Z_OtherInfo_characterInfo)
 318         return oi->information.characterInfo;
 319     return 0;
 320 }
 321
 322 const char *Yaz_Proxy::load_balance(const char **url)
 323 {
 324     int zurl_in_use[MAX_ZURL_PLEX];
 325     int zurl_in_spare[MAX_ZURL_PLEX];
 326     Yaz_ProxyClient *c;
 327     int i;
 328
 329     for (i = 0; i<MAX_ZURL_PLEX; i++)
 330     {
 331         zurl_in_use[i] = 0;
 332         zurl_in_spare[i] = 0;
 333     }
 334     for (c = m_parent->m_clientPool; c; c = c->m_next)
 335     {
 336         for (i = 0; url[i]; i++)
 337             if (!strcmp(url[i], c->get_hostname()))
 338             {
 339                 zurl_in_use[i]++;
 340                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 341                     zurl_in_spare[i]++;
 342             }
 343     }
 344     int min_use = 100000;
 345     int spare_for_min = 0;
 346     int max_spare = 0;
 347     const char *ret_min = 0;
 348     const char *ret_spare = 0;
 349     for (i = 0; url[i]; i++)
 350     {
 351         yaz_log(LOG_DEBUG, "%szurl=%s use=%d spare=%d",
 352                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 353         if (min_use > zurl_in_use[i])
 354         {
 355             ret_min = url[i];
 356             min_use = zurl_in_use[i];
 357             spare_for_min = zurl_in_spare[i];
 358         }
 359         if (max_spare < zurl_in_spare[i])
 360         {
 361             ret_spare = url[i];
 362             max_spare = zurl_in_spare[i];
 363         }
 364     }
 365     // use the one with minimum connections if spare is > 3
 366     if (spare_for_min > 3)
 367         return ret_min;
 368     // use one with most spares (if any)
 369     if (max_spare > 0)
 370         return ret_spare;
 371     return ret_min;
 372 }
 373
 374 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 375                                        const char *proxy_host)
 376 {
 377     assert (m_parent);
 378     Yaz_Proxy *parent = m_parent;
 379     Yaz_ProxyClient *c = m_client;
 380
 381     if (!m_proxyTarget)
 382     {
 383         const char *url[MAX_ZURL_PLEX];
 384         Yaz_ProxyConfig *cfg = check_reconfigure();
 385         if (proxy_host)
 386         {
 387 #if 0
 388 /* only to be enabled for debugging... */
 389             if (!strcmp(proxy_host, "stop"))
 390                 exit(0);
 391 #endif
 392             xfree(m_default_target);
 393             m_default_target = xstrdup(proxy_host);
 394         }
 395         proxy_host = m_default_target;
 396         int client_idletime = -1;
 397         const char *cql2rpn_fname = 0;
 398         url[0] = m_default_target;
 399         url[1] = 0;
 400         if (cfg)
 401         {
 402             int pre_init = 0;
 403             cfg->get_target_info(proxy_host, url, &m_bw_max,
 404                                  &m_pdu_max, &m_max_record_retrieve,
 405                                  &m_target_idletime, &client_idletime,
 406                                  &parent->m_max_clients,
 407                                  &m_keepalive_limit_bw,
 408                                  &m_keepalive_limit_pdu,
 409                                  &pre_init,
 410                                  &cql2rpn_fname);
 411         }
 412         if (client_idletime != -1)
 413         {
 414             m_client_idletime = client_idletime;
 415             timeout(m_client_idletime);
 416         }
 417         if (cql2rpn_fname)
 418             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 419         if (!url[0])
 420         {
 421             yaz_log(LOG_LOG, "%sNo default target", m_session_str);
 422             return 0;
 423         }
 424         // we don't handle multiplexing for cookie session, so we just
 425         // pick the first one in this case (anonymous users will be able
 426         // to use any backend)
 427         if (cookie && *cookie)
 428             m_proxyTarget = (char*) xstrdup(url[0]);
 429         else
 430             m_proxyTarget = (char*) xstrdup(load_balance(url));
 431     }
 432     if (cookie && *cookie)
 433     {   // search in sessions with a cookie
 434         for (c = parent->m_clientPool; c; c = c->m_next)
 435         {
 436             assert (c->m_prev);
 437             assert (*c->m_prev == c);
 438             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 439                 !strcmp(m_proxyTarget, c->get_hostname()))
 440             {
 441                 // Found it in cache
 442                 // The following handles "cancel"
 443                 // If connection is busy (waiting for PDU) and
 444                 // we have an initRequest we can safely do re-open
 445                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 446                 {
 447                     yaz_log (LOG_LOG, "%s REOPEN target=%s", m_session_str,
 448                              c->get_hostname());
 449                     c->close();
 450                     c->m_init_flag = 0;
 451
 452                     c->m_last_ok = 0;
 453                     c->m_cache.clear();
 454                     c->m_last_resultCount = 0;
 455                     c->m_sr_transform = 0;
 456                     c->m_waiting = 0;
 457                     c->m_resultSetStartPoint = 0;
 458                     c->m_target_idletime = m_target_idletime;
 459                     if (c->client(m_proxyTarget))
 460                     {
 461                         delete c;
 462                         return 0;
 463                     }
 464                     c->timeout(30);
 465                 }
 466                 c->m_seqno = parent->m_seqno;
 467                 if (c->m_server && c->m_server != this)
 468                     c->m_server->m_client = 0;
 469                 c->m_server = this;
 470                 (parent->m_seqno)++;
 471                 yaz_log (LOG_DEBUG, "get_client 1 %p %p", this, c);
 472                 return c;
 473             }
 474         }
 475     }
 476     else if (!c)
 477     {
 478         // don't have a client session yet. Search in session w/o cookie
 479         for (c = parent->m_clientPool; c; c = c->m_next)
 480         {
 481             assert (c->m_prev);
 482             assert (*c->m_prev == c);
 483             if (c->m_server == 0 && c->m_cookie == 0 &&
 484                 c->m_waiting == 0 &&
 485                 !strcmp(m_proxyTarget, c->get_hostname()))
 486             {
 487                 // found it in cache
 488                 yaz_log (LOG_LOG, "%sREUSE %d %s",
 489                          m_session_str, parent->m_seqno, c->get_hostname());
 490
 491                 c->m_seqno = parent->m_seqno;
 492                 assert(c->m_server == 0);
 493                 c->m_server = this;
 494
 495                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 496                     c->set_APDU_yazlog(1);
 497                 else
 498                     c->set_APDU_yazlog(0);
 499
 500                 (parent->m_seqno)++;
 501
 502                 parent->pre_init();
 503
 504                 return c;
 505             }
 506         }
 507     }
 508     if (!m_client)
 509     {
 510         if (apdu->which != Z_APDU_initRequest)
 511         {
 512             yaz_log (LOG_LOG, "%sno init request as first PDU", m_session_str);
 513             return 0;
 514         }
 515         Z_InitRequest *initRequest = apdu->u.initRequest;
 516
 517         if (!initRequest->idAuthentication)
 518         {
 519             if (m_proxy_authentication)
 520             {
 521                 initRequest->idAuthentication =
 522                     (Z_IdAuthentication *)
 523                     odr_malloc (odr_encode(),
 524                                 sizeof(*initRequest->idAuthentication));
 525                 initRequest->idAuthentication->which =
 526                     Z_IdAuthentication_open;
 527                 initRequest->idAuthentication->u.open =
 528                     odr_strdup (odr_encode(), m_proxy_authentication);
 529             }
 530         }
 531         // go through list of clients - and find the lowest/oldest one.
 532         Yaz_ProxyClient *c_min = 0;
 533         int min_seq = -1;
 534         int no_of_clients = 0;
 535         if (parent->m_clientPool)
 536             yaz_log (LOG_DEBUG, "Existing sessions");
 537         for (c = parent->m_clientPool; c; c = c->m_next)
 538         {
 539             yaz_log (LOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 540                                c->m_waiting, c->get_hostname(),
 541                                c->m_cookie ? c->m_cookie : "");
 542             no_of_clients++;
 543             if (min_seq < 0 || c->m_seqno < min_seq)
 544             {
 545                 min_seq = c->m_seqno;
 546                 c_min = c;
 547             }
 548         }
 549         if (no_of_clients >= parent->m_max_clients)
 550         {
 551             c = c_min;
 552             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 553             {
 554                 yaz_log (LOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 555                          m_session_str, parent->m_max_clients, c->m_seqno);
 556                 if (c->m_server && c->m_server != this)
 557                     delete c->m_server;
 558                 c->m_server = 0;
 559             }
 560             else
 561             {
 562                 yaz_log (LOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 563                          m_session_str, parent->m_max_clients,
 564                          c->m_seqno, parent->m_seqno, c->get_hostname());
 565                 xfree (c->m_cookie);
 566                 c->m_cookie = 0;
 567                 if (cookie)
 568                     c->m_cookie = xstrdup(cookie);
 569                 c->m_seqno = parent->m_seqno;
 570                 if (c->m_server && c->m_server != this)
 571                 {
 572                     c->m_server->m_client = 0;
 573                     delete c->m_server;
 574                 }
 575                 (parent->m_seqno)++;
 576                 c->m_target_idletime = m_target_idletime;
 577                 c->timeout(m_target_idletime);
 578
 579                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 580                     c->set_APDU_yazlog(1);
 581                 else
 582                     c->set_APDU_yazlog(0);
 583
 584                 return c;
 585             }
 586         }
 587         else
 588         {
 589             yaz_log (LOG_LOG, "%sNEW %d %s",
 590                      m_session_str, parent->m_seqno, m_proxyTarget);
 591             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 592             c->m_next = parent->m_clientPool;
 593             if (c->m_next)
 594                 c->m_next->m_prev = &c->m_next;
 595             parent->m_clientPool = c;
 596             c->m_prev = &parent->m_clientPool;
 597         }
 598
 599         xfree (c->m_cookie);
 600         c->m_cookie = 0;
 601         if (cookie)
 602             c->m_cookie = xstrdup(cookie);
 603
 604         c->m_seqno = parent->m_seqno;
 605         c->m_init_flag = 0;
 606         c->m_last_resultCount = 0;
 607         c->m_last_ok = 0;
 608         c->m_cache.clear();
 609         c->m_sr_transform = 0;
 610         c->m_waiting = 0;
 611         c->m_resultSetStartPoint = 0;
 612         (parent->m_seqno)++;
 613         if (c->client(m_proxyTarget))
 614         {
 615             delete c;
 616             return 0;
 617         }
 618         c->m_target_idletime = m_target_idletime;
 619         c->timeout(30);
 620
 621         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 622             c->set_APDU_yazlog(1);
 623         else
 624             c->set_APDU_yazlog(0);
 625     }
 626     yaz_log (LOG_DEBUG, "get_client 3 %p %p", this, c);
 627     return c;
 628 }
 629
 630 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 631 {
 632     int i;
 633     for (i = 0; i<num; i++)
 634     {
 635         oident *ent;
 636         Z_DefaultDiagFormat *r;
 637         Z_DiagRec *p = pp[i];
 638         if (p->which != Z_DiagRec_defaultFormat)
 639         {
 640             yaz_log(LOG_LOG, "%sError no diagnostics", m_session_str);
 641             return;
 642         }
 643         else
 644             r = p->u.defaultFormat;
 645         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 646             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 647             yaz_log(LOG_LOG, "%sError unknown diagnostic set", m_session_str);
 648         switch (r->which)
 649         {
 650         case Z_DefaultDiagFormat_v2Addinfo:
 651             yaz_log(LOG_LOG, "%sError %d %s:%s",
 652                     m_session_str,
 653                     *r->condition, diagbib1_str(*r->condition),
 654                     r->u.v2Addinfo);
 655             break;
 656         case Z_DefaultDiagFormat_v3Addinfo:
 657             yaz_log(LOG_LOG, "%sError %d %s:%s",
 658                     m_session_str,
 659                     *r->condition, diagbib1_str(*r->condition),
 660                     r->u.v3Addinfo);
 661             break;
 662         }
 663     }
 664 }
 665
 666 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 667 {
 668     if (!m_stylesheet_xsp || p->num_records <= 0)
 669         return 0;  /* no XSLT to be done ... */
 670
 671     m_stylesheet_offset = 0;
 672     m_stylesheet_nprl = p;
 673     m_stylesheet_apdu = apdu;
 674     timeout(0);
 675     return 1;
 676 }
 677
 678 void Yaz_Proxy::convert_xsl_delay()
 679 {
 680     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 681 #if HAVE_XSLT
 682     if (npr->which == Z_NamePlusRecord_databaseRecord)
 683     {
 684         Z_External *r = npr->u.databaseRecord;
 685         if (r->which == Z_External_octet)
 686         {
 687 #if 0
 688             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 689 #endif
 690             xmlDocPtr res, doc = xmlParseMemory(
 691                 (char*) r->u.octet_aligned->buf,
 692                 r->u.octet_aligned->len);
 693
 694
 695             yaz_log(LOG_LOG, "%sXSLT convert %d",
 696                     m_session_str, m_stylesheet_offset);
 697             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 698                                       doc, 0);
 699
 700             if (res)
 701             {
 702                 xmlChar *out_buf;
 703                 int out_len;
 704                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 705
 706                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 707                     u.databaseRecord =
 708                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 709                                  (char*) out_buf, out_len);
 710                 xmlFree(out_buf);
 711                 xmlFreeDoc(res);
 712             }
 713
 714             xmlFreeDoc(doc);
 715         }
 716     }
 717 #endif
 718     m_stylesheet_offset++;
 719     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 720     {
 721         m_stylesheet_nprl = 0;
 722 #if HAVE_XSLT
 723         if (m_stylesheet_xsp)
 724             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 725 #endif
 726         m_stylesheet_xsp = 0;
 727         timeout(m_client_idletime);
 728         send_PDU_convert(m_stylesheet_apdu);
 729     }
 730     else
 731         timeout(0);
 732 }
 733
 734 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 735 {
 736     if (m_frontend_type != VAL_NONE)
 737     {
 738         int i;
 739         for (i = 0; i < p->num_records; i++)
 740         {
 741             Z_NamePlusRecord *npr = p->records[i];
 742             if (npr->which == Z_NamePlusRecord_databaseRecord)
 743             {
 744                 Z_External *r = npr->u.databaseRecord;
 745                 if (r->which == Z_External_octet)
 746                 {
 747                     npr->u.databaseRecord =
 748                         z_ext_record(odr_encode(),
 749                                      m_frontend_type,
 750                                      (char*) r->u.octet_aligned->buf,
 751                                      r->u.octet_aligned->len);
 752                 }
 753             }
 754         }
 755     }
 756 }
 757
 758 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
 759                                    const char *backend_charset)
 760 {
 761     int i;
 762
 763     if (!backend_charset)
 764         backend_charset = "MARC-8";
 765     yaz_iconv_t cd = yaz_iconv_open("UTF-8", "MARC-8");
 766     yaz_marc_t mt = yaz_marc_create();
 767     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 768     yaz_marc_iconv(mt, cd);
 769     for (i = 0; i < p->num_records; i++)
 770     {
 771         Z_NamePlusRecord *npr = p->records[i];
 772         if (npr->which == Z_NamePlusRecord_databaseRecord)
 773         {
 774             Z_External *r = npr->u.databaseRecord;
 775             if (r->which == Z_External_octet)
 776             {
 777                 int rlen;
 778                 char *result;
 779                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 780                                         r->u.octet_aligned->len,
 781                                         &result, &rlen))
 782                 {
 783                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 784                                                          VAL_TEXT_XML,
 785                                                          result, rlen);
 786                 }
 787             }
 788         }
 789     }
 790     if (cd)
 791         yaz_iconv_close(cd);
 792     yaz_marc_destroy(mt);
 793 }
 794
 795 void Yaz_Proxy::logtime()
 796 {
 797 #if HAVE_GETTIMEOFDAY
 798     struct timeval *tv = (struct timeval*) m_time_tv;
 799     if (tv->tv_sec)
 800     {
 801         struct timeval tv1;
 802         gettimeofday(&tv1, 0);
 803         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
 804             (tv1.tv_usec - tv->tv_usec);
 805         if (diff >= 0)
 806             yaz_log(LOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 807                     diff/1000000, (diff/1000)%1000);
 808     }
 809     tv->tv_sec = 0;
 810     tv->tv_usec = 0;
 811 #endif
 812 }
 813
 814 int Yaz_Proxy::send_http_response(int code)
 815 {
 816     ODR o = odr_encode();
 817     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 818     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 819     if (m_http_version)
 820         hres->version = odr_strdup(o, m_http_version);
 821     if (m_http_keepalive)
 822         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 823     else
 824         timeout(0);
 825
 826     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 827     {
 828         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 829                  gdu_name(gdu));
 830     }
 831     int len;
 832     int r = send_GDU(gdu, &len);
 833     m_bytes_sent += len;
 834     m_bw_stat.add_bytes(len);
 835     logtime();
 836     return r;
 837 }
 838
 839 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 840 {
 841     ODR o = odr_encode();
 842     const char *ctype = "text/xml";
 843     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 844     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 845     if (m_http_version)
 846         hres->version = odr_strdup(o, m_http_version);
 847     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 848     if (m_http_keepalive)
 849         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 850     else
 851         timeout(0);
 852
 853     static Z_SOAP_Handler soap_handlers[2] = {
 854 #if HAVE_XSLT
 855         {"http://www.loc.gov/zing/srw/", 0,
 856          (Z_SOAP_fun) yaz_srw_codec},
 857 #endif
 858         {0, 0, 0}
 859     };
 860
 861     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 862     soap_package->which = Z_SOAP_generic;
 863     soap_package->u.generic =
 864         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 865     soap_package->u.generic->no = 0;
 866     soap_package->u.generic->ns = soap_handlers[0].ns;
 867     soap_package->u.generic->p = (void *) srw_pdu;
 868     soap_package->ns = m_soap_ns;
 869     z_soap_codec_enc_xsl(o, &soap_package,
 870                          &hres->content_buf, &hres->content_len,
 871                          soap_handlers, 0, m_s2z_stylesheet);
 872     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 873     {
 874         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 875                  gdu_name(gdu));
 876     }
 877     int len;
 878     int r = send_GDU(gdu, &len);
 879     m_bytes_sent += len;
 880     m_bw_stat.add_bytes(len);
 881     logtime();
 882     return r;
 883 }
 884
 885 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
 886 {
 887     ODR o = odr_encode();
 888     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 889     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 890
 891     srw_res->num_diagnostics = 1;
 892     srw_res->diagnostics = (Z_SRW_diagnostic *)
 893         odr_malloc(o, sizeof(*srw_res->diagnostics));
 894     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
 895     return send_srw_response(srw_pdu);
 896 }
 897
 898 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 899                              Z_DefaultDiagFormat *ddf)
 900 {
 901     int bib1_code = *ddf->condition;
 902     if (bib1_code == 109)
 903         return 404;
 904     srw_res->num_diagnostics = 1;
 905     srw_res->diagnostics = (Z_SRW_diagnostic *)
 906         odr_malloc(o, sizeof(*srw_res->diagnostics));
 907     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
 908                           yaz_diag_bib1_to_srw(*ddf->condition),
 909                           ddf->u.v2Addinfo);
 910     return 0;
 911 }
 912
 913 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
 914 {
 915     ODR o = odr_encode();
 916     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 917     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 918
 919     srw_res->numberOfRecords = odr_intdup (o, hits);
 920     if (records && records->which == Z_Records_DBOSD)
 921     {
 922         srw_res->num_records =
 923             records->u.databaseOrSurDiagnostics->num_records;
 924         int i;
 925         srw_res->records = (Z_SRW_record *)
 926             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
 927         for (i = 0; i < srw_res->num_records; i++)
 928         {
 929             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
 930             if (npr->which != Z_NamePlusRecord_databaseRecord)
 931             {
 932                 srw_res->records[i].recordSchema = "diagnostic";
 933                 srw_res->records[i].recordPacking = m_s2z_packing;
 934                 srw_res->records[i].recordData_buf = "67";
 935                 srw_res->records[i].recordData_len = 2;
 936                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 937                 continue;
 938             }
 939             Z_External *r = npr->u.databaseRecord;
 940             oident *ent = oid_getentbyoid(r->direct_reference);
 941             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
 942             {
 943                 srw_res->records[i].recordSchema = m_schema;
 944                 srw_res->records[i].recordPacking = m_s2z_packing;
 945                 srw_res->records[i].recordData_buf = (char*)
 946                     r->u.octet_aligned->buf;
 947                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
 948                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 949             }
 950             else
 951             {
 952                 srw_res->records[i].recordSchema = "diagnostic";
 953                 srw_res->records[i].recordPacking = m_s2z_packing;
 954                 srw_res->records[i].recordData_buf = "67";
 955                 srw_res->records[i].recordData_len = 2;
 956                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 957             }
 958         }
 959     }
 960     if (records && records->which == Z_Records_NSD)
 961     {
 962         int http_code;
 963         http_code = z_to_srw_diag(odr_encode(), srw_res,
 964                                    records->u.nonSurrogateDiagnostic);
 965         if (http_code)
 966             return send_http_response(http_code);
 967     }
 968     return send_srw_response(srw_pdu);
 969
 970 }
 971
 972 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
 973                                         int num_diagnostics)
 974 {
 975     Yaz_ProxyConfig *cfg = check_reconfigure();
 976     if (cfg)
 977     {
 978         int len;
 979         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
 980                                        m_s2z_database, &len);
 981         if (b)
 982         {
 983             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
 984             Z_SRW_explainResponse *er = res->u.explain_response;
 985
 986             er->record.recordData_buf = b;
 987             er->record.recordData_len = len;
 988             er->record.recordPacking = m_s2z_packing;
 989             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
 990
 991             er->diagnostics = diagnostics;
 992             er->num_diagnostics = num_diagnostics;
 993             return send_srw_response(res);
 994         }
 995     }
 996     return send_http_response(404);
 997 }
 998
 999 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1000 {
1001     if (m_http_version)
1002     {
1003         if (apdu->which == Z_APDU_initResponse)
1004         {
1005             Z_InitResponse *res = apdu->u.initResponse;
1006             if (*res->result == 0)
1007             {
1008                 send_to_srw_client_error(3, 0);
1009             }
1010             else if (!m_s2z_search_apdu)
1011             {
1012                 send_srw_explain_response(0, 0);
1013             }
1014             else
1015             {
1016                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1017             }
1018         }
1019         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1020         {
1021             m_s2z_search_apdu = 0;
1022             Z_SearchResponse *res = apdu->u.searchResponse;
1023             m_s2z_hit_count = *res->resultCount;
1024             if (res->records && res->records->which == Z_Records_NSD)
1025             {
1026                 send_to_srw_client_ok(0, res->records, 1);
1027             }
1028             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1029             {
1030                 // adjust
1031                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1032
1033                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1034                 {
1035                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1036                         > m_s2z_hit_count)
1037                         *pr->numberOfRecordsRequested =
1038                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1039                 }
1040                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1041             }
1042             else
1043             {
1044                 m_s2z_present_apdu = 0;
1045                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1046             }
1047         }
1048         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1049         {
1050             int start =
1051                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1052
1053             m_s2z_present_apdu = 0;
1054             Z_PresentResponse *res = apdu->u.presentResponse;
1055             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1056         }
1057     }
1058     else
1059     {
1060         int len = 0;
1061         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1062             yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
1063                      apdu_name(apdu));
1064         int r = send_Z_PDU(apdu, &len);
1065         m_bytes_sent += len;
1066         m_bw_stat.add_bytes(len);
1067         logtime();
1068         return r;
1069     }
1070     return 0;
1071 }
1072
1073 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1074 {
1075     int kill_session = 0;
1076     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1077
1078     if (new_id)
1079         *new_id = m_referenceId;
1080
1081     if (apdu->which == Z_APDU_searchResponse)
1082     {
1083         Z_SearchResponse *sr = apdu->u.searchResponse;
1084         Z_Records *p = sr->records;
1085         if (p && p->which == Z_Records_NSD)
1086         {
1087             Z_DiagRec dr, *dr_p = &dr;
1088             dr.which = Z_DiagRec_defaultFormat;
1089             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1090
1091             *sr->searchStatus = 0;
1092             display_diagrecs(&dr_p, 1);
1093         }
1094         else
1095         {
1096             if (p && p->which == Z_Records_DBOSD)
1097             {
1098                 if (m_backend_type)
1099                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1100                 if (m_marcxml_flag)
1101                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1102                                        m_backend_charset);
1103                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1104                     return 0;
1105
1106             }
1107             if (sr->resultCount)
1108             {
1109                 yaz_log(LOG_LOG, "%s%d hits", m_session_str,
1110                         *sr->resultCount);
1111                 if (*sr->resultCount < 0)
1112                 {
1113                     m_invalid_session = 1;
1114                     kill_session = 1;
1115
1116                     *sr->searchStatus = 0;
1117                     sr->records =
1118                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1119                     *sr->resultCount = 0;
1120                 }
1121             }
1122         }
1123     }
1124     else if (apdu->which == Z_APDU_presentResponse)
1125     {
1126         Z_PresentResponse *sr = apdu->u.presentResponse;
1127         Z_Records *p = sr->records;
1128         if (p && p->which == Z_Records_NSD)
1129         {
1130             Z_DiagRec dr, *dr_p = &dr;
1131             dr.which = Z_DiagRec_defaultFormat;
1132             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1133             if (*sr->presentStatus == Z_PresentStatus_success)
1134                 *sr->presentStatus = Z_PresentStatus_failure;
1135             display_diagrecs(&dr_p, 1);
1136         }
1137         if (p && p->which == Z_Records_DBOSD)
1138         {
1139             if (m_backend_type)
1140                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1141             if (m_marcxml_flag)
1142                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1143                                    m_backend_charset);
1144             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1145                 return 0;
1146         }
1147     }
1148     else if (apdu->which == Z_APDU_initResponse)
1149     {
1150         if (m_initRequest_options)
1151         {
1152             Z_Options *nopt =
1153                 (Odr_bitmask *)odr_malloc(odr_encode(),
1154                                           sizeof(Odr_bitmask));
1155             ODR_MASK_ZERO(nopt);
1156
1157             int i;
1158             for (i = 0; i<24; i++)
1159                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1160                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1161                     ODR_MASK_SET(nopt, i);
1162             apdu->u.initResponse->options = nopt;
1163         }
1164         if (m_initRequest_version)
1165         {
1166             Z_ProtocolVersion *nopt =
1167                 (Odr_bitmask *)odr_malloc(odr_encode(),
1168                                           sizeof(Odr_bitmask));
1169             ODR_MASK_ZERO(nopt);
1170
1171             int i;
1172             for (i = 0; i<8; i++)
1173                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1174                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1175                     ODR_MASK_SET(nopt, i);
1176             apdu->u.initResponse->protocolVersion = nopt;
1177         }
1178         apdu->u.initResponse->preferredMessageSize =
1179             odr_intdup(odr_encode(),
1180                        m_client->m_initResponse_preferredMessageSize >
1181                        m_initRequest_preferredMessageSize ?
1182                        m_initRequest_preferredMessageSize :
1183                        m_client->m_initResponse_preferredMessageSize);
1184         apdu->u.initResponse->maximumRecordSize =
1185             odr_intdup(odr_encode(),
1186                        m_client->m_initResponse_maximumRecordSize >
1187                        m_initRequest_maximumRecordSize ?
1188                        m_initRequest_maximumRecordSize :
1189                        m_client->m_initResponse_maximumRecordSize);
1190     }
1191     int r = send_PDU_convert(apdu);
1192     if (r)
1193         return r;
1194     if (kill_session)
1195     {
1196         delete m_client;
1197         m_client = 0;
1198         m_parent->pre_init();
1199     }
1200     return r;
1201 }
1202
1203 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1204 {
1205     int len = 0;
1206     const char *apdu_name_tmp = apdu_name(apdu);
1207     int r = send_Z_PDU(apdu, &len);
1208     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1209         yaz_log (LOG_LOG, "%sSending %s to %s %d bytes",
1210                  get_session_str(),
1211                  apdu_name_tmp, get_hostname(), len);
1212     m_bytes_sent += len;
1213     return r;
1214 }
1215
1216 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1217 {
1218     if (apdu->which == Z_APDU_presentRequest)
1219     {
1220         Z_PresentRequest *pr = apdu->u.presentRequest;
1221         int toget = *pr->numberOfRecordsRequested;
1222         int start = *pr->resultSetStartPoint;
1223
1224         yaz_log(LOG_LOG, "%sPresent %s %d+%d", m_session_str,
1225                 pr->resultSetId, start, toget);
1226
1227         if (*m_parent->m_optimize == '0')
1228             return apdu;
1229
1230         if (!m_client->m_last_resultSetId)
1231         {
1232             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1233             new_apdu->u.presentResponse->records =
1234                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1235                                                pr->resultSetId);
1236             send_to_client(new_apdu);
1237             return 0;
1238         }
1239         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1240         {
1241             if (start+toget-1 > m_client->m_last_resultCount)
1242             {
1243                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1244                 new_apdu->u.presentResponse->records =
1245                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1246                 send_to_client(new_apdu);
1247                 return 0;
1248             }
1249             Z_NamePlusRecordList *npr;
1250 #if 0
1251             yaz_log(LOG_LOG, "%sCache lookup %d+%d syntax=%s",
1252                     m_session_str, start, toget, yaz_z3950oid_to_str(
1253                         pr->preferredRecordSyntax, &oclass));
1254 #endif
1255             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1256                                           pr->preferredRecordSyntax,
1257                                           pr->recordComposition))
1258             {
1259                 yaz_log (LOG_LOG, "%sReturned cached records for present request",
1260                          m_session_str);
1261                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1262                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1263
1264                 new_apdu->u.presentResponse->numberOfRecordsReturned
1265                     = odr_intdup(odr_encode(), toget);
1266
1267                 new_apdu->u.presentResponse->records = (Z_Records*)
1268                     odr_malloc(odr_encode(), sizeof(Z_Records));
1269                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1270                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1271                 new_apdu->u.presentResponse->nextResultSetPosition =
1272                     odr_intdup(odr_encode(), start+toget);
1273
1274                 send_to_client(new_apdu);
1275                 return 0;
1276             }
1277         }
1278     }
1279
1280     if (apdu->which != Z_APDU_searchRequest)
1281         return apdu;
1282     Z_SearchRequest *sr = apdu->u.searchRequest;
1283     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1284     Yaz_Z_Databases this_databases;
1285
1286     this_databases.set(sr->num_databaseNames, (const char **)
1287                        sr->databaseNames);
1288
1289     this_query->set_Z_Query(sr->query);
1290
1291     char query_str[120];
1292     this_query->print(query_str, sizeof(query_str)-1);
1293     yaz_log(LOG_LOG, "%sSearch %s", m_session_str, query_str);
1294
1295     if (*m_parent->m_optimize != '0' &&
1296         m_client->m_last_ok && m_client->m_last_query &&
1297         m_client->m_last_query->match(this_query) &&
1298         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1299         m_client->m_last_databases.match(this_databases))
1300     {
1301         delete this_query;
1302         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1303             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1304         {
1305             Z_NamePlusRecordList *npr;
1306             int toget = *sr->mediumSetPresentNumber;
1307             Z_RecordComposition *comp = 0;
1308
1309             if (toget > m_client->m_last_resultCount)
1310                 toget = m_client->m_last_resultCount;
1311
1312             if (sr->mediumSetElementSetNames)
1313             {
1314                 comp = (Z_RecordComposition *)
1315                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1316                 comp->which = Z_RecordComp_simple;
1317                 comp->u.simple = sr->mediumSetElementSetNames;
1318             }
1319
1320             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1321                                           sr->preferredRecordSyntax, comp))
1322             {
1323                 yaz_log (LOG_LOG, "%sReturned cached records for medium set",
1324                          m_session_str);
1325                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1326                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1327                 new_apdu->u.searchResponse->resultCount =
1328                     &m_client->m_last_resultCount;
1329
1330                 new_apdu->u.searchResponse->numberOfRecordsReturned
1331                     = odr_intdup(odr_encode(), toget);
1332
1333                 new_apdu->u.searchResponse->presentStatus =
1334                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1335                 new_apdu->u.searchResponse->records = (Z_Records*)
1336                     odr_malloc(odr_encode(), sizeof(Z_Records));
1337                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1338                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1339                 new_apdu->u.searchResponse->nextResultSetPosition =
1340                     odr_intdup(odr_encode(), toget+1);
1341                 send_to_client(new_apdu);
1342                 return 0;
1343             }
1344             else
1345             {
1346                 // medium Set
1347                 // send present request (medium size)
1348                 yaz_log (LOG_LOG, "%sOptimizing search for medium set",
1349                          m_session_str);
1350
1351                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1352                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1353                 pr->referenceId = sr->referenceId;
1354                 pr->resultSetId = sr->resultSetName;
1355                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1356                 *pr->numberOfRecordsRequested = toget;
1357                 pr->recordComposition = comp;
1358                 m_client->m_sr_transform = 1;
1359                 return new_apdu;
1360             }
1361         }
1362         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1363             m_client->m_last_resultCount <= 0)
1364         {
1365             // large set. Return pseudo-search response immediately
1366             yaz_log (LOG_LOG, "%sOptimizing search for large set",
1367                      m_session_str);
1368             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1369             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1370             new_apdu->u.searchResponse->resultCount =
1371                 &m_client->m_last_resultCount;
1372             send_to_client(new_apdu);
1373             return 0;
1374         }
1375         else
1376         {
1377             Z_NamePlusRecordList *npr;
1378             int toget = m_client->m_last_resultCount;
1379             Z_RecordComposition *comp = 0;
1380             // small set
1381             // send a present request (small set)
1382
1383             if (sr->smallSetElementSetNames)
1384             {
1385                 comp = (Z_RecordComposition *)
1386                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1387                 comp->which = Z_RecordComp_simple;
1388                 comp->u.simple = sr->smallSetElementSetNames;
1389             }
1390
1391             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1392                                           sr->preferredRecordSyntax, comp))
1393             {
1394                 yaz_log (LOG_LOG, "%sReturned cached records for small set",
1395                          m_session_str);
1396                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1397                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1398                 new_apdu->u.searchResponse->resultCount =
1399                     &m_client->m_last_resultCount;
1400
1401                 new_apdu->u.searchResponse->numberOfRecordsReturned
1402                     = odr_intdup(odr_encode(), toget);
1403
1404                 new_apdu->u.searchResponse->presentStatus =
1405                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1406                 new_apdu->u.searchResponse->records = (Z_Records*)
1407                     odr_malloc(odr_encode(), sizeof(Z_Records));
1408                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1409                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1410                 new_apdu->u.searchResponse->nextResultSetPosition =
1411                     odr_intdup(odr_encode(), toget+1);
1412                 send_to_client(new_apdu);
1413                 return 0;
1414             }
1415             else
1416             {
1417                 yaz_log (LOG_LOG, "%sOptimizing search for small set",
1418                          m_session_str);
1419                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1420                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1421                 pr->referenceId = sr->referenceId;
1422                 pr->resultSetId = sr->resultSetName;
1423                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1424                 *pr->numberOfRecordsRequested = toget;
1425                 pr->recordComposition = comp;
1426                 m_client->m_sr_transform = 1;
1427                 return new_apdu;
1428             }
1429         }
1430     }
1431     else  // query doesn't match
1432     {
1433         delete m_client->m_last_query;
1434         m_client->m_last_query = this_query;
1435         m_client->m_last_ok = 0;
1436         m_client->m_cache.clear();
1437         m_client->m_resultSetStartPoint = 0;
1438
1439         xfree (m_client->m_last_resultSetId);
1440         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1441
1442         m_client->m_last_databases.set(sr->num_databaseNames,
1443                                        (const char **) sr->databaseNames);
1444     }
1445     return apdu;
1446 }
1447
1448
1449 void Yaz_Proxy::inc_request_no()
1450 {
1451     char *cp = strchr(m_session_str, ' ');
1452     m_request_no++;
1453     if (cp)
1454         sprintf(cp+1, "%d ", m_request_no);
1455 }
1456
1457 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1458 {
1459     inc_request_no();
1460
1461     m_bytes_recv += len;
1462
1463     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1464         yaz_log (LOG_LOG, "%sReceiving %s from client %d bytes",
1465                  m_session_str, gdu_name(apdu), len);
1466
1467     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1468         shutdown();
1469
1470     m_bw_stat.add_bytes(len);
1471     m_pdu_stat.add_bytes(1);
1472
1473 #if HAVE_GETTIMEOFDAY
1474     gettimeofday((struct timeval *) m_time_tv, 0);
1475 #endif
1476
1477     int bw_total = m_bw_stat.get_total();
1478     int pdu_total = m_pdu_stat.get_total();
1479
1480     int reduce = 0;
1481     if (m_bw_max)
1482     {
1483         if (bw_total > m_bw_max)
1484         {
1485             reduce = (bw_total/m_bw_max);
1486         }
1487     }
1488     if (m_pdu_max)
1489     {
1490         if (pdu_total > m_pdu_max)
1491         {
1492             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1493             reduce = (reduce > nreduce) ? reduce : nreduce;
1494         }
1495     }
1496     if (reduce)
1497     {
1498         yaz_log(LOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1499                 m_session_str, reduce, bw_total, pdu_total,
1500                 m_bw_max, m_pdu_max);
1501
1502         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1503         timeout(reduce);       // call us reduce seconds later
1504     }
1505     else if (apdu->which == Z_GDU_Z3950)
1506         handle_incoming_Z_PDU(apdu->u.z3950);
1507     else if (apdu->which == Z_GDU_HTTP_Request)
1508         handle_incoming_HTTP(apdu->u.HTTP_Request);
1509 }
1510
1511 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1512 {
1513     if (m_max_record_retrieve)
1514     {
1515         if (apdu->which == Z_APDU_presentRequest)
1516         {
1517             Z_PresentRequest *pr = apdu->u.presentRequest;
1518             if (pr->numberOfRecordsRequested &&
1519                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1520                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1521         }
1522     }
1523 }
1524
1525 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1526                                                      int error,
1527                                                      const char *addinfo)
1528 {
1529     Z_Records *rec = (Z_Records *)
1530         odr_malloc (odr, sizeof(*rec));
1531     int *err = (int *)
1532         odr_malloc (odr, sizeof(*err));
1533     Z_DiagRec *drec = (Z_DiagRec *)
1534         odr_malloc (odr, sizeof(*drec));
1535     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1536         odr_malloc (odr, sizeof(*dr));
1537     *err = error;
1538     rec->which = Z_Records_NSD;
1539     rec->u.nonSurrogateDiagnostic = dr;
1540     dr->diagnosticSetId =
1541         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1542     dr->condition = err;
1543     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1544     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1545     return rec;
1546 }
1547
1548 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1549 {
1550     if (apdu->which == Z_APDU_searchRequest &&
1551         apdu->u.searchRequest->query &&
1552         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1553         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1554     {
1555         Z_RPNQuery *rpnquery = 0;
1556         Z_SearchRequest *sr = apdu->u.searchRequest;
1557         char *addinfo = 0;
1558
1559         yaz_log(LOG_LOG, "%sCQL: %s", m_session_str,
1560                 sr->query->u.type_104->u.cql);
1561
1562         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1563                                           &rpnquery, odr_encode(),
1564                                           &addinfo);
1565         if (r == -3)
1566             yaz_log(LOG_LOG, "%sNo CQL to RPN table", m_session_str);
1567         else if (r)
1568         {
1569             yaz_log(LOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1570             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1571
1572             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1573             new_apdu->u.searchResponse->records =
1574                 create_nonSurrogateDiagnostics(odr_encode(),
1575                                                yaz_diag_srw_to_bib1(r),
1576                                                addinfo);
1577             *new_apdu->u.searchResponse->searchStatus = 0;
1578
1579             send_to_client(new_apdu);
1580
1581             return 0;
1582         }
1583         else
1584         {
1585             sr->query->which = Z_Query_type_1;
1586             sr->query->u.type_1 = rpnquery;
1587         }
1588         return apdu;
1589     }
1590     return apdu;
1591 }
1592
1593 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1594 {
1595     if (apdu->which == Z_APDU_searchRequest)
1596     {
1597         Z_SearchRequest *sr = apdu->u.searchRequest;
1598         int err = 0;
1599         char *addinfo = 0;
1600
1601         Yaz_ProxyConfig *cfg = check_reconfigure();
1602         if (cfg)
1603             err = cfg->check_query(odr_encode(), m_default_target,
1604                                    sr->query, &addinfo);
1605         if (err)
1606         {
1607             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1608
1609             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1610             new_apdu->u.searchResponse->records =
1611                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1612             *new_apdu->u.searchResponse->searchStatus = 0;
1613
1614             send_to_client(new_apdu);
1615
1616             return 0;
1617         }
1618     }
1619     return apdu;
1620 }
1621
1622 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1623 {
1624     m_marcxml_flag = 0;
1625     if (apdu->which == Z_APDU_searchRequest)
1626     {
1627         Z_SearchRequest *sr = apdu->u.searchRequest;
1628         int err = 0;
1629         char *addinfo = 0;
1630         Yaz_ProxyConfig *cfg = check_reconfigure();
1631
1632         Z_RecordComposition rc_temp, *rc = 0;
1633         if (sr->smallSetElementSetNames)
1634         {
1635             rc_temp.which = Z_RecordComp_simple;
1636             rc_temp.u.simple = sr->smallSetElementSetNames;
1637             rc = &rc_temp;
1638         }
1639
1640         if (sr->preferredRecordSyntax)
1641         {
1642             struct oident *ent;
1643             ent = oid_getentbyoid(sr->preferredRecordSyntax);
1644             m_frontend_type = ent->value;
1645         }
1646         else
1647             m_frontend_type = VAL_NONE;
1648
1649         char *stylesheet_name = 0;
1650         if (cfg)
1651             err = cfg->check_syntax(odr_encode(),
1652                                     m_default_target,
1653                                     sr->preferredRecordSyntax, rc,
1654                                     &addinfo, &stylesheet_name, &m_schema,
1655                                     &m_backend_type, &m_backend_charset);
1656         if (stylesheet_name)
1657         {
1658             m_parent->low_socket_close();
1659
1660 #if HAVE_XSLT
1661             if (m_stylesheet_xsp)
1662                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
1663             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1664                                                        stylesheet_name);
1665 #endif
1666             m_stylesheet_offset = 0;
1667             xfree(stylesheet_name);
1668
1669             m_parent->low_socket_open();
1670         }
1671         if (err == -1)
1672         {
1673             sr->smallSetElementSetNames = 0;
1674             sr->mediumSetElementSetNames = 0;
1675             if (m_backend_type)
1676             {
1677
1678                 sr->preferredRecordSyntax =
1679                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
1680                                         m_backend_type);
1681             }
1682             else
1683                 sr->preferredRecordSyntax =
1684                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
1685                                            VAL_USMARC);
1686             m_marcxml_flag = 1;
1687         }
1688         else if (err)
1689         {
1690             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1691
1692             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1693             new_apdu->u.searchResponse->records =
1694                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1695             *new_apdu->u.searchResponse->searchStatus = 0;
1696
1697             send_to_client(new_apdu);
1698
1699             return 0;
1700         }
1701         else if (m_backend_type)
1702         {
1703             sr->preferredRecordSyntax =
1704                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
1705         }
1706     }
1707     else if (apdu->which == Z_APDU_presentRequest)
1708     {
1709         Z_PresentRequest *pr = apdu->u.presentRequest;
1710         int err = 0;
1711         char *addinfo = 0;
1712         Yaz_ProxyConfig *cfg = check_reconfigure();
1713
1714         if (pr->preferredRecordSyntax)
1715         {
1716             struct oident *ent;
1717             ent = oid_getentbyoid(pr->preferredRecordSyntax);
1718             m_frontend_type = ent->value;
1719         }
1720         else
1721             m_frontend_type = VAL_NONE;
1722
1723         char *stylesheet_name = 0;
1724         if (cfg)
1725             err = cfg->check_syntax(odr_encode(), m_default_target,
1726                                     pr->preferredRecordSyntax,
1727                                     pr->recordComposition,
1728                                     &addinfo, &stylesheet_name, &m_schema,
1729                                     &m_backend_type, &m_backend_charset);
1730         if (stylesheet_name)
1731         {
1732             m_parent->low_socket_close();
1733
1734 #if HAVE_XSLT
1735             if (m_stylesheet_xsp)
1736                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
1737             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1738                                                        stylesheet_name);
1739 #endif
1740             m_stylesheet_offset = 0;
1741             xfree(stylesheet_name);
1742
1743             m_parent->low_socket_open();
1744         }
1745         if (err == -1)
1746         {
1747             pr->recordComposition = 0;
1748             if (m_backend_type)
1749             {
1750
1751                 pr->preferredRecordSyntax =
1752                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
1753                                         m_backend_type);
1754             }
1755             else
1756                 pr->preferredRecordSyntax =
1757                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
1758                                            VAL_USMARC);
1759             m_marcxml_flag = 1;
1760         }
1761         else if (err)
1762         {
1763             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1764
1765             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1766             new_apdu->u.presentResponse->records =
1767                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1768             *new_apdu->u.presentResponse->presentStatus =
1769                 Z_PresentStatus_failure;
1770
1771             send_to_client(new_apdu);
1772
1773             return 0;
1774         }
1775         else if (m_backend_type)
1776         {
1777             pr->preferredRecordSyntax =
1778                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
1779         }
1780     }
1781     return apdu;
1782 }
1783
1784 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
1785 {
1786     if (!schema)
1787         return 0;
1788     Z_ElementSetNames *esn = (Z_ElementSetNames *)
1789         odr_malloc(o, sizeof(Z_ElementSetNames));
1790     esn->which = Z_ElementSetNames_generic;
1791     esn->u.generic = odr_strdup(o, schema);
1792     return esn;
1793 }
1794
1795 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
1796 {
1797     const char *t = 0;
1798     Yaz_ProxyConfig *cfg = check_reconfigure();
1799     if (cfg)
1800         t = cfg->get_explain_name(db, backend_db);
1801
1802     if (m_client && m_default_target && t && strcmp(m_default_target, t))
1803     {
1804         releaseClient();
1805     }
1806
1807     if (t)
1808     {
1809         xfree(m_default_target);
1810         m_default_target = xstrdup(t);
1811     }
1812 }
1813
1814 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1815 {
1816     if (m_s2z_odr_init)
1817     {
1818         odr_destroy(m_s2z_odr_init);
1819         m_s2z_odr_init = 0;
1820     }
1821     if (m_s2z_odr_search)
1822     {
1823         odr_destroy(m_s2z_odr_search);
1824         m_s2z_odr_search = 0;
1825     }
1826
1827     m_http_keepalive = 0;
1828     m_http_version = 0;
1829     if (!strcmp(hreq->version, "1.0"))
1830     {
1831         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1832         if (v && !strcmp(v, "Keep-Alive"))
1833             m_http_keepalive = 1;
1834         else
1835             m_http_keepalive = 0;
1836         m_http_version = "1.0";
1837     }
1838     else
1839     {
1840         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1841         if (v && !strcmp(v, "close"))
1842             m_http_keepalive = 0;
1843         else
1844             m_http_keepalive = 1;
1845         m_http_version = "1.1";
1846     }
1847
1848     Z_SRW_PDU *srw_pdu = 0;
1849     Z_SOAP *soap_package = 0;
1850     char *charset = 0;
1851     Z_SRW_diagnostic *diagnostic = 0;
1852     int num_diagnostic = 0;
1853     if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1854                        &charset) == 0
1855         || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1856                           &charset, &diagnostic, &num_diagnostic) == 0)
1857     {
1858         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
1859         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
1860         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
1861         m_s2z_init_apdu = 0;
1862         m_s2z_search_apdu = 0;
1863         m_s2z_present_apdu = 0;
1864
1865         m_s2z_stylesheet = 0;
1866
1867         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
1868         {
1869             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
1870
1871             const char *backend_db = srw_req->database;
1872             srw_get_client(srw_req->database, &backend_db);
1873
1874             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1875             // recordXPath unsupported.
1876             if (srw_req->recordXPath)
1877             {
1878                 yaz_add_srw_diagnostic(odr_decode(),
1879                                        &diagnostic, &num_diagnostic,
1880                                        72, 0);
1881             }
1882             // sort unsupported
1883             if (srw_req->sort_type != Z_SRW_sort_type_none)
1884             {
1885                 yaz_add_srw_diagnostic(odr_decode(),
1886                                        &diagnostic, &num_diagnostic,
1887                                        80, 0);
1888             }
1889             // save stylesheet
1890             if (srw_req->stylesheet)
1891                 m_s2z_stylesheet =
1892                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1893
1894             // set packing for response records ..
1895             if (srw_req->recordPacking &&
1896                 !strcmp(srw_req->recordPacking, "xml"))
1897                 m_s2z_packing = Z_SRW_recordPacking_XML;
1898             else
1899                 m_s2z_packing = Z_SRW_recordPacking_string;
1900
1901             if (num_diagnostic)
1902             {
1903                 Z_SRW_PDU *srw_pdu =
1904                     yaz_srw_get(odr_encode(),
1905                                 Z_SRW_searchRetrieve_response);
1906                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1907
1908                 srw_res->diagnostics = diagnostic;
1909                 srw_res->num_diagnostics = num_diagnostic;
1910                 send_srw_response(srw_pdu);
1911                 return;
1912             }
1913
1914             // prepare search PDU
1915             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
1916                                           Z_APDU_searchRequest);
1917             Z_SearchRequest *z_searchRequest =
1918                 m_s2z_search_apdu->u.searchRequest;
1919
1920             z_searchRequest->num_databaseNames = 1;
1921             z_searchRequest->databaseNames = (char**)
1922                 odr_malloc(m_s2z_odr_search, sizeof(char *));
1923             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
1924                                                            backend_db);
1925
1926             // query transformation
1927             Z_Query *query = (Z_Query *)
1928                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
1929             z_searchRequest->query = query;
1930
1931             if (srw_req->query_type == Z_SRW_query_type_cql)
1932             {
1933                 Z_External *ext = (Z_External *)
1934                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
1935                 ext->direct_reference =
1936                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
1937                 ext->indirect_reference = 0;
1938                 ext->descriptor = 0;
1939                 ext->which = Z_External_CQL;
1940                 ext->u.cql = srw_req->query.cql;
1941
1942                 query->which = Z_Query_type_104;
1943                 query->u.type_104 =  ext;
1944             }
1945             else if (srw_req->query_type == Z_SRW_query_type_pqf)
1946             {
1947                 Z_RPNQuery *RPNquery;
1948                 YAZ_PQF_Parser pqf_parser;
1949
1950                 pqf_parser = yaz_pqf_create ();
1951
1952                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
1953                                           srw_req->query.pqf);
1954                 if (!RPNquery)
1955                 {
1956                     const char *pqf_msg;
1957                     size_t off;
1958                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
1959                     yaz_log(LOG_LOG, "%*s^\n", off+4, "");
1960                     yaz_log(LOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
1961
1962                     send_to_srw_client_error(10, 0);
1963                     return;
1964                 }
1965                 query->which = Z_Query_type_1;
1966                 query->u.type_1 =  RPNquery;
1967
1968                 yaz_pqf_destroy (pqf_parser);
1969             }
1970             else
1971             {
1972                 send_to_srw_client_error(7, "query");
1973                 return;
1974             }
1975
1976             // present
1977             m_s2z_present_apdu = 0;
1978             int max = 0;
1979             if (srw_req->maximumRecords)
1980                 max = *srw_req->maximumRecords;
1981             int start = 1;
1982             if (srw_req->startRecord)
1983                 start = *srw_req->startRecord;
1984             if (max > 0)
1985             {
1986                 // Some backend, such as Voyager doesn't honor piggyback
1987                 // So we use present always (0 &&).
1988                 if (0 && start <= 1)  // Z39.50 piggyback
1989                 {
1990                     *z_searchRequest->smallSetUpperBound = max;
1991                     *z_searchRequest->mediumSetPresentNumber = max;
1992                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
1993
1994                     z_searchRequest->preferredRecordSyntax =
1995                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1996                                                VAL_TEXT_XML);
1997                     if (srw_req->recordSchema)
1998                     {
1999                         z_searchRequest->smallSetElementSetNames =
2000                             z_searchRequest->mediumSetElementSetNames =
2001                             mk_esn_from_schema(m_s2z_odr_search,
2002                                                srw_req->recordSchema);
2003                     }
2004                 }
2005                 else   // Z39.50 present
2006                 {
2007                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2008                                                    Z_APDU_presentRequest);
2009                     Z_PresentRequest *z_presentRequest =
2010                         m_s2z_present_apdu->u.presentRequest;
2011                     *z_presentRequest->resultSetStartPoint = start;
2012                     *z_presentRequest->numberOfRecordsRequested = max;
2013                     z_presentRequest->preferredRecordSyntax =
2014                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2015                                                VAL_TEXT_XML);
2016                     if (srw_req->recordSchema)
2017                     {
2018                         z_presentRequest->recordComposition =
2019                             (Z_RecordComposition *)
2020                             odr_malloc(m_s2z_odr_search,
2021                                        sizeof(Z_RecordComposition));
2022                         z_presentRequest->recordComposition->which =
2023                             Z_RecordComp_simple;
2024                         z_presentRequest->recordComposition->u.simple =
2025                             mk_esn_from_schema(m_s2z_odr_search,
2026                                                srw_req->recordSchema);
2027                     }
2028                 }
2029             }
2030             if (!m_client)
2031             {
2032                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2033                                             Z_APDU_initRequest);
2034
2035                 // prevent m_initRequest_apdu memory from being grabbed
2036                 // in Yaz_Proxy::handle_incoming_Z_PDU
2037                 m_initRequest_apdu = m_s2z_init_apdu;
2038                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2039                 return;
2040             }
2041             else
2042             {
2043                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2044                 return;
2045             }
2046         }
2047         else if (srw_pdu->which == Z_SRW_explain_request)
2048         {
2049             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2050
2051             const char *backend_db = srw_req->database;
2052             srw_get_client(srw_req->database, &backend_db);
2053
2054             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2055
2056             // save stylesheet
2057             if (srw_req->stylesheet)
2058                 m_s2z_stylesheet =
2059                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2060
2061             if (srw_req->recordPacking &&
2062                 !strcmp(srw_req->recordPacking, "xml"))
2063                 m_s2z_packing = Z_SRW_recordPacking_XML;
2064             else
2065                 m_s2z_packing = Z_SRW_recordPacking_string;
2066
2067             if (num_diagnostic)
2068             {
2069                 send_srw_explain_response(diagnostic, num_diagnostic);
2070                 return;
2071             }
2072
2073             if (!m_client)
2074             {
2075                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2076                                             Z_APDU_initRequest);
2077
2078                 // prevent m_initRequest_apdu memory from being grabbed
2079                 // in Yaz_Proxy::handle_incoming_Z_PDU
2080                 m_initRequest_apdu = m_s2z_init_apdu;
2081                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2082             }
2083             else
2084                 send_srw_explain_response(0, 0);
2085             return;
2086         }
2087         else if (srw_pdu->which == Z_SRW_scan_request)
2088         {
2089             m_s2z_database = odr_strdup(m_s2z_odr_init,
2090                                         srw_pdu->u.scan_request->database);
2091
2092             yaz_add_srw_diagnostic(odr_decode(),
2093                                    &diagnostic, &num_diagnostic,
2094                                    4, "scan");
2095             Z_SRW_PDU *srw_pdu =
2096                 yaz_srw_get(odr_encode(),
2097                             Z_SRW_scan_response);
2098             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2099
2100             srw_res->diagnostics = diagnostic;
2101             srw_res->num_diagnostics = num_diagnostic;
2102             send_srw_response(srw_pdu);
2103             return;
2104         }
2105         else
2106         {
2107             m_s2z_database = 0;
2108
2109             send_to_srw_client_error(4, 0);
2110         }
2111     }
2112     int len = 0;
2113     Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
2114     timeout(0);
2115     send_GDU(p, &len);
2116 }
2117
2118 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2119 {
2120     Z_ReferenceId **refid = get_referenceIdP(apdu);
2121     nmem_reset(m_referenceId_mem);
2122     if (refid && *refid)
2123     {
2124         m_referenceId = (Z_ReferenceId *)
2125             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2126         m_referenceId->len = m_referenceId->size = (*refid)->len;
2127         m_referenceId->buf = (unsigned char *)
2128             nmem_malloc(m_referenceId_mem, (*refid)->len);
2129         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2130     }
2131     else
2132         m_referenceId = 0;
2133
2134     if (!m_client && m_invalid_session)
2135     {
2136         m_apdu_invalid_session = apdu;
2137         m_mem_invalid_session = odr_extract_mem(odr_decode());
2138         apdu = m_initRequest_apdu;
2139     }
2140
2141     // Determine our client.
2142     Z_OtherInformation **oi;
2143     get_otherInfoAPDU(apdu, &oi);
2144     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2145     if (!m_client)
2146     {
2147         delete this;
2148         return;
2149     }
2150     m_client->m_server = this;
2151
2152     if (apdu->which == Z_APDU_initRequest)
2153     {
2154         if (apdu->u.initRequest->implementationId)
2155             yaz_log(LOG_LOG, "%simplementationId: %s",
2156                     m_session_str, apdu->u.initRequest->implementationId);
2157         if (apdu->u.initRequest->implementationName)
2158             yaz_log(LOG_LOG, "%simplementationName: %s",
2159                     m_session_str, apdu->u.initRequest->implementationName);
2160         if (apdu->u.initRequest->implementationVersion)
2161             yaz_log(LOG_LOG, "%simplementationVersion: %s",
2162                     m_session_str, apdu->u.initRequest->implementationVersion);
2163         if (m_initRequest_apdu == 0)
2164         {
2165             if (m_initRequest_mem)
2166                 nmem_destroy(m_initRequest_mem);
2167             m_initRequest_apdu = apdu;
2168             m_initRequest_mem = odr_extract_mem(odr_decode());
2169
2170             m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2171                 preferredMessageSize;
2172             *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2173             m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2174                 maximumRecordSize;
2175             *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2176
2177             // save init options for the response..
2178             m_initRequest_options = apdu->u.initRequest->options;
2179
2180             apdu->u.initRequest->options =
2181                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2182                                            sizeof(Odr_bitmask));
2183             ODR_MASK_ZERO(apdu->u.initRequest->options);
2184             int i;
2185             for (i = 0; i<= 24; i++)
2186                 ODR_MASK_SET(apdu->u.initRequest->options, i);
2187             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2188                            Z_Options_negotiationModel);
2189             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2190                            Z_Options_concurrentOperations);
2191
2192             // make new version
2193             m_initRequest_version = apdu->u.initRequest->protocolVersion;
2194             apdu->u.initRequest->protocolVersion =
2195                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2196                                            sizeof(Odr_bitmask));
2197             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2198
2199             for (i = 0; i<= 8; i++)
2200                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2201         }
2202         if (m_client->m_init_flag)
2203         {
2204             if (handle_init_response_for_invalid_session(apdu))
2205                 return;
2206             if (m_client->m_initResponse)
2207             {
2208                 Z_APDU *apdu2 = m_client->m_initResponse;
2209                 apdu2->u.initResponse->otherInfo = 0;
2210                 if (m_client->m_cookie && *m_client->m_cookie)
2211                     set_otherInformationString(apdu2, VAL_COOKIE, 1,
2212                                                m_client->m_cookie);
2213                 apdu2->u.initResponse->referenceId =
2214                     apdu->u.initRequest->referenceId;
2215                 apdu2->u.initResponse->options = m_client->m_initResponse_options;
2216                 apdu2->u.initResponse->protocolVersion =
2217                     m_client->m_initResponse_version;
2218
2219                 send_to_client(apdu2);
2220                 return;
2221             }
2222         }
2223         m_client->m_init_flag = 1;
2224     }
2225     handle_max_record_retrieve(apdu);
2226
2227     if (apdu)
2228         apdu = handle_syntax_validation(apdu);
2229
2230     if (apdu)
2231         apdu = handle_query_transformation(apdu);
2232
2233     if (apdu)
2234         apdu = handle_query_validation(apdu);
2235
2236     if (apdu)
2237         apdu = result_set_optimize(apdu);
2238     if (!apdu)
2239     {
2240         m_client->timeout(m_target_idletime);  // mark it active even
2241         // though we didn't use it
2242         return;
2243     }
2244
2245     // delete other info part from PDU before sending to target
2246     get_otherInfoAPDU(apdu, &oi);
2247     if (oi)
2248         *oi = 0;
2249
2250     if (apdu->which == Z_APDU_presentRequest &&
2251         m_client->m_resultSetStartPoint == 0)
2252     {
2253         Z_PresentRequest *pr = apdu->u.presentRequest;
2254         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2255         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2256     } else {
2257         m_client->m_resultSetStartPoint = 0;
2258     }
2259     if (m_client->send_to_target(apdu) < 0)
2260     {
2261         delete m_client;
2262         m_client = 0;
2263         delete this;
2264     }
2265     else
2266         m_client->m_waiting = 1;
2267 }
2268
2269 void Yaz_Proxy::connectNotify()
2270 {
2271 }
2272
2273 void Yaz_Proxy::releaseClient()
2274 {
2275     xfree(m_proxyTarget);
2276     m_proxyTarget = 0;
2277     m_invalid_session = 0;
2278     // only keep if keep_alive flag is set...
2279     if (m_client &&
2280         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2281         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2282         m_client->m_waiting == 0)
2283     {
2284         yaz_log(LOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2285                  m_session_str,
2286                  m_client->get_hostname());
2287         yaz_log(LOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2288                 m_session_str, m_client->m_pdu_recv,
2289                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2290                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2291         assert (m_client->m_waiting != 2);
2292         // Tell client (if any) that no server connection is there..
2293         m_client->m_server = 0;
2294         m_invalid_session = 0;
2295         m_client = 0;
2296     }
2297     else if (m_client)
2298     {
2299         yaz_log (LOG_LOG, "%sShutdown (client to proxy) close %s",
2300                  m_session_str,
2301                  m_client->get_hostname());
2302         assert (m_client->m_waiting != 2);
2303         delete m_client;
2304         m_client = 0;
2305     }
2306     else if (!m_parent)
2307     {
2308         yaz_log (LOG_LOG, "%sshutdown (client to proxy) bad state",
2309                  m_session_str);
2310         assert (m_parent);
2311     }
2312     else
2313     {
2314         yaz_log (LOG_LOG, "%sShutdown (client to proxy)",
2315                  m_session_str);
2316     }
2317     if (m_parent)
2318         m_parent->pre_init();
2319 }
2320
2321 void Yaz_Proxy::shutdown()
2322 {
2323     releaseClient();
2324     delete this;
2325 }
2326
2327 const char *Yaz_ProxyClient::get_session_str()
2328 {
2329     if (!m_server)
2330         return "0 ";
2331     return m_server->get_session_str();
2332 }
2333
2334 void Yaz_ProxyClient::shutdown()
2335 {
2336     yaz_log (LOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2337              get_hostname());
2338     delete m_server;
2339     delete this;
2340 }
2341
2342 void Yaz_Proxy::failNotify()
2343 {
2344     inc_request_no();
2345     yaz_log (LOG_LOG, "%sConnection closed by client",
2346              get_session_str());
2347     shutdown();
2348 }
2349
2350 void Yaz_ProxyClient::failNotify()
2351 {
2352     if (m_server)
2353         m_server->inc_request_no();
2354     yaz_log (LOG_LOG, "%sConnection closed by target %s",
2355              get_session_str(), get_hostname());
2356     shutdown();
2357 }
2358
2359 void Yaz_ProxyClient::connectNotify()
2360 {
2361     const char *s = get_session_str();
2362     const char *h = get_hostname();
2363     yaz_log (LOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2364              m_target_idletime);
2365     timeout(m_target_idletime);
2366     if (!m_server)
2367         pre_init_client();
2368 }
2369
2370 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2371                                                   *the_PDU_Observable, int fd)
2372 {
2373     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2374 }
2375
2376 Yaz_ProxyClient::~Yaz_ProxyClient()
2377 {
2378     if (m_prev)
2379         *m_prev = m_next;
2380     if (m_next)
2381         m_next->m_prev = m_prev;
2382     m_waiting = 2;     // for debugging purposes only.
2383     odr_destroy(m_init_odr);
2384     delete m_last_query;
2385     xfree (m_last_resultSetId);
2386     xfree (m_cookie);
2387 }
2388
2389 void Yaz_ProxyClient::pre_init_client()
2390 {
2391     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2392     Z_InitRequest *req = apdu->u.initRequest;
2393
2394     int i;
2395     for (i = 0; i<= 24; i++)
2396         ODR_MASK_SET(req->options, i);
2397     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2398                    Z_Options_negotiationModel);
2399     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2400                    Z_Options_concurrentOperations);
2401     for (i = 0; i<= 10; i++)
2402         ODR_MASK_SET(req->protocolVersion, i);
2403
2404     if (send_to_target(apdu) < 0)
2405     {
2406         delete this;
2407     }
2408     else
2409     {
2410         m_waiting = 1;
2411         m_init_flag = 1;
2412     }
2413 }
2414
2415 void Yaz_Proxy::pre_init()
2416 {
2417     int i;
2418     const char *name = 0;
2419     const char *zurl_in_use[MAX_ZURL_PLEX];
2420     int limit_bw, limit_pdu, limit_req;
2421     int target_idletime, client_idletime;
2422     int max_clients;
2423     int keepalive_limit_bw, keepalive_limit_pdu;
2424     int pre_init;
2425     const char *cql2rpn = 0;
2426
2427     Yaz_ProxyConfig *cfg = check_reconfigure();
2428
2429     zurl_in_use[0] = 0;
2430
2431     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2432         set_APDU_yazlog(1);
2433     else
2434         set_APDU_yazlog(0);
2435
2436     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2437                                           &limit_bw, &limit_pdu, &limit_req,
2438                                           &target_idletime, &client_idletime,
2439                                           &max_clients,
2440                                           &keepalive_limit_bw,
2441                                           &keepalive_limit_pdu,
2442                                           &pre_init,
2443                                           &cql2rpn) ; i++)
2444     {
2445         if (pre_init)
2446         {
2447             int j;
2448             for (j = 0; zurl_in_use[j]; j++)
2449             {
2450                 Yaz_ProxyClient *c;
2451                 int spare = 0;
2452                 int spare_waiting = 0;
2453                 int in_use = 0;
2454                 int other = 0;
2455                 for (c = m_clientPool; c; c = c->m_next)
2456                 {
2457                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
2458                     {
2459                         if (c->m_cookie == 0)
2460                         {
2461                             if (c->m_server == 0)
2462                                 if (c->m_waiting)
2463                                     spare_waiting++;
2464                                 else
2465                                     spare++;
2466                             else
2467                                 in_use++;
2468                         }
2469                         else
2470                             other++;
2471                     }
2472                 }
2473                 yaz_log(LOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2474                         "sparew=%d preinit=%d",m_session_str,
2475                         name, zurl_in_use[j], in_use, other,
2476                         spare, spare_waiting, pre_init);
2477                 if (spare + spare_waiting < pre_init)
2478                 {
2479                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2480                     c->m_next = m_clientPool;
2481                     if (c->m_next)
2482                         c->m_next->m_prev = &c->m_next;
2483                     m_clientPool = c;
2484                     c->m_prev = &m_clientPool;
2485
2486                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
2487                         c->set_APDU_yazlog(1);
2488                     else
2489                         c->set_APDU_yazlog(0);
2490
2491                     if (c->client(zurl_in_use[j]))
2492                     {
2493                         timeout(60);
2494                         delete c;
2495                         return;
2496                     }
2497                     c->timeout(30);
2498                     c->m_waiting = 1;
2499                     c->m_target_idletime = target_idletime;
2500                     c->m_seqno = m_seqno++;
2501                 }
2502             }
2503         }
2504     }
2505 }
2506
2507 void Yaz_Proxy::timeoutNotify()
2508 {
2509     if (m_parent)
2510     {
2511         if (m_bw_hold_PDU)
2512         {
2513             timeout(m_client_idletime);
2514             Z_GDU *apdu = m_bw_hold_PDU;
2515             m_bw_hold_PDU = 0;
2516
2517             if (apdu->which == Z_GDU_Z3950)
2518                 handle_incoming_Z_PDU(apdu->u.z3950);
2519             else if (apdu->which == Z_GDU_HTTP_Request)
2520                 handle_incoming_HTTP(apdu->u.HTTP_Request);
2521         }
2522         else if (m_stylesheet_nprl)
2523             convert_xsl_delay();
2524         else
2525         {
2526             inc_request_no();
2527
2528             yaz_log (LOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2529             shutdown();
2530         }
2531     }
2532     else
2533     {
2534         timeout(600);
2535         pre_init();
2536     }
2537 }
2538
2539 void Yaz_Proxy::markInvalid()
2540 {
2541     m_client = 0;
2542     m_invalid_session = 1;
2543 }
2544
2545 void Yaz_ProxyClient::timeoutNotify()
2546 {
2547     if (m_server)
2548         m_server->inc_request_no();
2549
2550     yaz_log (LOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2551              get_hostname());
2552     m_waiting = 1;
2553     m_root->pre_init();
2554     if (m_server && m_init_flag)
2555     {
2556         // target timed out in a session that was properly initialized
2557         // server object stay alive but we mark it as invalid so it
2558         // gets initialized again
2559         m_server->markInvalid();
2560         m_server = 0;
2561     }
2562     shutdown();
2563 }
2564
2565 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2566                                  Yaz_Proxy *parent) :
2567     Yaz_Z_Assoc (the_PDU_Observable)
2568 {
2569     m_cookie = 0;
2570     m_next = 0;
2571     m_prev = 0;
2572     m_init_flag = 0;
2573     m_last_query = 0;
2574     m_last_resultSetId = 0;
2575     m_last_resultCount = 0;
2576     m_last_ok = 0;
2577     m_sr_transform = 0;
2578     m_waiting = 0;
2579     m_init_odr = odr_createmem (ODR_DECODE);
2580     m_initResponse = 0;
2581     m_initResponse_options = 0;
2582     m_initResponse_version = 0;
2583     m_initResponse_preferredMessageSize = 0;
2584     m_initResponse_maximumRecordSize = 0;
2585     m_resultSetStartPoint = 0;
2586     m_bytes_sent = m_bytes_recv = 0;
2587     m_pdu_recv = 0;
2588     m_server = 0;
2589     m_seqno = 0;
2590     m_target_idletime = 600;
2591     m_root = parent;
2592 }
2593
2594 const char *Yaz_Proxy::option(const char *name, const char *value)
2595 {
2596     if (!strcmp (name, "optimize")) {
2597         if (value) {
2598             xfree (m_optimize);
2599             m_optimize = xstrdup (value);
2600         }
2601         return m_optimize;
2602     }
2603     return 0;
2604 }
2605
2606 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2607 {
2608
2609 }
2610
2611 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2612 {
2613     if (apdu->which == Z_GDU_Z3950)
2614         recv_Z_PDU(apdu->u.z3950, len);
2615     else if (apdu->which == Z_GDU_HTTP_Response)
2616         recv_HTTP_response(apdu->u.HTTP_Response, len);
2617     else
2618         shutdown();
2619 }
2620
2621 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2622 {
2623     if (!m_invalid_session)
2624         return 0;
2625     m_invalid_session = 0;
2626     handle_incoming_Z_PDU(m_apdu_invalid_session);
2627     assert (m_mem_invalid_session);
2628     nmem_destroy(m_mem_invalid_session);
2629     m_mem_invalid_session = 0;
2630     return 1;
2631 }
2632
2633 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2634 {
2635     m_bytes_recv += len;
2636
2637     m_pdu_recv++;
2638     m_waiting = 0;
2639     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2640         yaz_log (LOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2641                  apdu_name(apdu), get_hostname(), len);
2642     if (apdu->which == Z_APDU_initResponse)
2643     {
2644         if (!m_server)  // if this is a pre init session , check for more
2645             m_root->pre_init();
2646         NMEM nmem = odr_extract_mem (odr_decode());
2647         odr_reset (m_init_odr);
2648         nmem_transfer (m_init_odr->mem, nmem);
2649         m_initResponse = apdu;
2650         m_initResponse_options = apdu->u.initResponse->options;
2651         m_initResponse_version = apdu->u.initResponse->protocolVersion;
2652         m_initResponse_preferredMessageSize =
2653             *apdu->u.initResponse->preferredMessageSize;
2654         m_initResponse_maximumRecordSize =
2655             *apdu->u.initResponse->maximumRecordSize;
2656
2657         Z_InitResponse *ir = apdu->u.initResponse;
2658         char *im0 = ir->implementationName;
2659
2660         char *im1 = (char*)
2661             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2662         *im1 = '\0';
2663         if (im0)
2664         {
2665             strcat(im1, im0);
2666             strcat(im1, " ");
2667         }
2668         strcat(im1, "(YAZ Proxy)");
2669         ir->implementationName = im1;
2670
2671         nmem_destroy (nmem);
2672
2673         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2674             return;
2675     }
2676     if (apdu->which == Z_APDU_searchResponse)
2677     {
2678         Z_SearchResponse *sr = apdu->u.searchResponse;
2679         m_last_resultCount = *sr->resultCount;
2680         int status = *sr->searchStatus;
2681         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2682         {
2683             m_last_ok = 1;
2684
2685             if (sr->records && sr->records->which == Z_Records_DBOSD)
2686             {
2687                 m_cache.add(odr_decode(),
2688                             sr->records->u.databaseOrSurDiagnostics, 1,
2689                             *sr->resultCount);
2690             }
2691         }
2692     }
2693     if (apdu->which == Z_APDU_presentResponse)
2694     {
2695         Z_PresentResponse *pr = apdu->u.presentResponse;
2696         if (m_sr_transform)
2697         {
2698             m_sr_transform = 0;
2699             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2700             Z_SearchResponse *sr = new_apdu->u.searchResponse;
2701             sr->referenceId = pr->referenceId;
2702             *sr->resultCount = m_last_resultCount;
2703             sr->records = pr->records;
2704             sr->nextResultSetPosition = pr->nextResultSetPosition;
2705             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2706             apdu = new_apdu;
2707         }
2708         if (pr->records &&
2709             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2710         {
2711             m_cache.add(odr_decode(),
2712                         pr->records->u.databaseOrSurDiagnostics,
2713                         m_resultSetStartPoint, -1);
2714             m_resultSetStartPoint = 0;
2715         }
2716     }
2717     if (m_cookie)
2718         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2719     if (m_server)
2720     {
2721         m_server->send_to_client(apdu);
2722     }
2723     if (apdu->which == Z_APDU_close)
2724     {
2725         shutdown();
2726     }
2727 }
2728
2729 void Yaz_Proxy::low_socket_close()
2730 {
2731 #if WIN32
2732 #else
2733     int i;
2734     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
2735         if  (m_lo_fd[i] >= 0)
2736             ::close(m_lo_fd[i]);
2737 #endif
2738 }
2739
2740 void Yaz_Proxy::low_socket_open()
2741 {
2742 #if WIN32
2743 #else
2744     int i;
2745     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
2746         m_lo_fd[i] = open("/dev/null", O_RDONLY);
2747 #endif
2748 }
2749
2750 int Yaz_Proxy::server(const char *addr)
2751 {
2752     int r = Yaz_Z_Assoc::server(addr);
2753     if (!r)
2754     {
2755         yaz_log(LOG_LOG, "%sStarted proxy "
2756 #ifdef VERSION
2757             VERSION
2758 #endif
2759             " on %s", m_session_str, addr);
2760         timeout(1);
2761     }
2762     return r;
2763 }
2764