src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.13 2004-12-03 14:28:18 adam Exp $
   2    Copyright (c) 1998-2004, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #else
  24 #include <unistd.h>
  25 #endif
  26
  27 #if HAVE_GETTIMEOFDAY
  28 #include <sys/time.h>
  29 #endif
  30
  31 #include <assert.h>
  32 #include <time.h>
  33 #include <sys/types.h>
  34 #include <fcntl.h>
  35
  36 #include <yaz/srw.h>
  37 #include <yaz/marcdisp.h>
  38 #include <yaz/yaz-iconv.h>
  39 #include <yaz/ylog.h>
  40 #include <yaz/diagbib1.h>
  41 #include "proxyp.h"
  42 #include <yaz/pquery.h>
  43 #include <yaz/otherinfo.h>
  44
  45 static const char *apdu_name(Z_APDU *apdu)
  46 {
  47     switch (apdu->which)
  48     {
  49     case Z_APDU_initRequest:
  50         return "initRequest";
  51     case Z_APDU_initResponse:
  52         return "initResponse";
  53     case Z_APDU_searchRequest:
  54         return "searchRequest";
  55     case Z_APDU_searchResponse:
  56         return "searchResponse";
  57     case Z_APDU_presentRequest:
  58         return "presentRequest";
  59     case Z_APDU_presentResponse:
  60         return "presentResponse";
  61     case Z_APDU_deleteResultSetRequest:
  62         return "deleteResultSetRequest";
  63     case Z_APDU_deleteResultSetResponse:
  64         return "deleteResultSetResponse";
  65     case Z_APDU_scanRequest:
  66         return "scanRequest";
  67     case Z_APDU_scanResponse:
  68         return "scanResponse";
  69     case Z_APDU_sortRequest:
  70         return "sortRequest";
  71     case Z_APDU_sortResponse:
  72         return "sortResponse";
  73     case Z_APDU_extendedServicesRequest:
  74         return "extendedServicesRequest";
  75     case Z_APDU_extendedServicesResponse:
  76         return "extendedServicesResponse";
  77     case Z_APDU_close:
  78         return "close";
  79     }
  80     return "other";
  81 }
  82
  83 static const char *gdu_name(Z_GDU *gdu)
  84 {
  85     switch(gdu->which)
  86     {
  87     case Z_GDU_Z3950:
  88         return apdu_name(gdu->u.z3950);
  89     case Z_GDU_HTTP_Request:
  90         return "HTTP Request";
  91     case Z_GDU_HTTP_Response:
  92         return "HTTP Response";
  93     }
  94     return "Unknown request/response";
  95 }
  96
  97 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
  98                      Yaz_Proxy *parent) :
  99     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 100 {
 101     m_PDU_Observable = the_PDU_Observable;
 102     m_client = 0;
 103     m_parent = parent;
 104     m_clientPool = 0;
 105     m_seqno = 1;
 106     m_keepalive_limit_bw = 500000;
 107     m_keepalive_limit_pdu = 1000;
 108     m_proxyTarget = 0;
 109     m_default_target = 0;
 110     m_proxy_authentication = 0;
 111     m_max_clients = 150;
 112     m_log_mask = 0;
 113     m_seed = time(0);
 114     m_client_idletime = 600;
 115     m_target_idletime = 600;
 116     m_optimize = xstrdup ("1");
 117     strcpy(m_session_str, "0 ");
 118     m_session_no=0;
 119     m_bytes_sent = 0;
 120     m_bytes_recv = 0;
 121     m_bw_hold_PDU = 0;
 122     m_bw_max = 0;
 123     m_pdu_max = 0;
 124     m_max_record_retrieve = 0;
 125     m_reconfig_flag = 0;
 126     m_config_fname = 0;
 127     m_request_no = 0;
 128     m_invalid_session = 0;
 129     m_referenceId = 0;
 130     m_referenceId_mem = nmem_create();
 131     m_config = 0;
 132     m_marcxml_flag = 0;
 133     m_stylesheet_xsp = 0;
 134     m_stylesheet_nprl = 0;
 135     m_s2z_stylesheet = 0;
 136     m_s2z_database = 0;
 137     m_schema = 0;
 138     m_backend_type = 0;
 139     m_backend_charset = 0;
 140     m_frontend_type = 0;
 141     m_initRequest_apdu = 0;
 142     m_initRequest_mem = 0;
 143     m_initRequest_preferredMessageSize = 0;
 144     m_initRequest_maximumRecordSize = 0;
 145     m_initRequest_options = 0;
 146     m_initRequest_version = 0;
 147     m_apdu_invalid_session = 0;
 148     m_mem_invalid_session = 0;
 149     m_s2z_odr_init = 0;
 150     m_s2z_odr_search = 0;
 151     m_s2z_init_apdu = 0;
 152     m_s2z_search_apdu = 0;
 153     m_s2z_present_apdu = 0;
 154     m_http_keepalive = 0;
 155     m_http_version = 0;
 156     m_soap_ns = 0;
 157     m_s2z_packing = Z_SRW_recordPacking_string;
 158 #if HAVE_GETTIMEOFDAY
 159     m_time_tv = xmalloc(sizeof(struct timeval));
 160     struct timeval *tv = (struct timeval *) m_time_tv;
 161     tv->tv_sec = 0;
 162     tv->tv_usec = 0;
 163 #else
 164     m_time_tv = 0;
 165 #endif
 166     m_usemarcon_ini_stage1 = 0;
 167     m_usemarcon_ini_stage2 = 0;
 168     m_usemarcon = new Yaz_usemarcon();
 169     if (!m_parent)
 170         low_socket_open();
 171 }
 172
 173 Yaz_Proxy::~Yaz_Proxy()
 174 {
 175     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 176             m_bytes_sent, m_bytes_recv);
 177     nmem_destroy(m_initRequest_mem);
 178     nmem_destroy(m_mem_invalid_session);
 179     nmem_destroy(m_referenceId_mem);
 180
 181     xfree(m_proxyTarget);
 182     xfree(m_default_target);
 183     xfree(m_proxy_authentication);
 184     xfree(m_optimize);
 185
 186 #if HAVE_XSLT
 187     if (m_stylesheet_xsp)
 188         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 189 #endif
 190     xfree (m_time_tv);
 191
 192     xfree (m_schema);
 193     xfree (m_backend_type);
 194     xfree (m_backend_charset);
 195     xfree (m_usemarcon_ini_stage1);
 196     xfree (m_usemarcon_ini_stage2);
 197     delete m_usemarcon;
 198     if (m_s2z_odr_init)
 199         odr_destroy(m_s2z_odr_init);
 200     if (m_s2z_odr_search)
 201         odr_destroy(m_s2z_odr_search);
 202     if (!m_parent)
 203         low_socket_close();
 204     delete m_config;
 205 }
 206
 207 int Yaz_Proxy::set_config(const char *config)
 208 {
 209     delete m_config;
 210     m_config = new Yaz_ProxyConfig();
 211     xfree(m_config_fname);
 212     m_config_fname = xstrdup(config);
 213     int r = m_config->read_xml(config);
 214     if (!r)
 215         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 216     return r;
 217 }
 218
 219 void Yaz_Proxy::set_default_target(const char *target)
 220 {
 221     xfree (m_default_target);
 222     m_default_target = 0;
 223     if (target)
 224         m_default_target = (char *) xstrdup (target);
 225 }
 226
 227 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 228 {
 229     xfree (m_proxy_authentication);
 230     m_proxy_authentication = 0;
 231     if (auth)
 232         m_proxy_authentication = (char *) xstrdup (auth);
 233 }
 234
 235 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 236 {
 237     if (m_parent)
 238         return m_parent->check_reconfigure();
 239
 240     Yaz_ProxyConfig *cfg = m_config;
 241     if (m_reconfig_flag)
 242     {
 243         yaz_log(YLOG_LOG, "reconfigure");
 244         yaz_log_reopen();
 245         if (m_config_fname && cfg)
 246         {
 247             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 248             int r = cfg->read_xml(m_config_fname);
 249             if (r)
 250                 yaz_log(YLOG_WARN, "reconfigure failed");
 251             else
 252             {
 253                 m_log_mask = 0;
 254                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 255             }
 256         }
 257         else
 258             yaz_log(YLOG_LOG, "reconfigure");
 259         m_reconfig_flag = 0;
 260     }
 261     return cfg;
 262 }
 263
 264 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 265                                             *the_PDU_Observable, int fd)
 266 {
 267     check_reconfigure();
 268     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 269     new_proxy->m_config = 0;
 270     new_proxy->m_config_fname = 0;
 271     new_proxy->timeout(m_client_idletime);
 272     new_proxy->m_target_idletime = m_target_idletime;
 273     new_proxy->set_default_target(m_default_target);
 274     new_proxy->m_max_clients = m_max_clients;
 275     new_proxy->m_log_mask = m_log_mask;
 276     new_proxy->set_APDU_log(get_APDU_log());
 277     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 278         new_proxy->set_APDU_yazlog(1);
 279     else
 280         new_proxy->set_APDU_yazlog(0);
 281     new_proxy->set_proxy_authentication(m_proxy_authentication);
 282     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 283     m_session_no++;
 284     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 285              the_PDU_Observable->getpeername());
 286     return new_proxy;
 287 }
 288
 289 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 290 {
 291     int oid[OID_SIZE];
 292     Z_OtherInformationUnit *oi;
 293     struct oident ent;
 294     ent.proto = PROTO_Z3950;
 295     ent.oclass = CLASS_USERINFO;
 296     ent.value = (oid_value) VAL_COOKIE;
 297     assert (oid_ent_to_oid (&ent, oid));
 298
 299     if (oid_ent_to_oid (&ent, oid) &&
 300         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 301         oi->which == Z_OtherInfo_characterInfo)
 302         return oi->information.characterInfo;
 303     return 0;
 304 }
 305
 306 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 307 {
 308     int oid[OID_SIZE];
 309     Z_OtherInformationUnit *oi;
 310     struct oident ent;
 311     ent.proto = PROTO_Z3950;
 312     ent.oclass = CLASS_USERINFO;
 313     ent.value = (oid_value) VAL_PROXY;
 314     if (oid_ent_to_oid (&ent, oid) &&
 315         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 316         oi->which == Z_OtherInfo_characterInfo)
 317         return oi->information.characterInfo;
 318     return 0;
 319 }
 320
 321 const char *Yaz_Proxy::load_balance(const char **url)
 322 {
 323     int zurl_in_use[MAX_ZURL_PLEX];
 324     int zurl_in_spare[MAX_ZURL_PLEX];
 325     Yaz_ProxyClient *c;
 326     int i;
 327
 328     for (i = 0; i<MAX_ZURL_PLEX; i++)
 329     {
 330         zurl_in_use[i] = 0;
 331         zurl_in_spare[i] = 0;
 332     }
 333     for (c = m_parent->m_clientPool; c; c = c->m_next)
 334     {
 335         for (i = 0; url[i]; i++)
 336             if (!strcmp(url[i], c->get_hostname()))
 337             {
 338                 zurl_in_use[i]++;
 339                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 340                     zurl_in_spare[i]++;
 341             }
 342     }
 343     int min_use = 100000;
 344     int spare_for_min = 0;
 345     int max_spare = 0;
 346     const char *ret_min = 0;
 347     const char *ret_spare = 0;
 348     for (i = 0; url[i]; i++)
 349     {
 350         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 351                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 352         if (min_use > zurl_in_use[i])
 353         {
 354             ret_min = url[i];
 355             min_use = zurl_in_use[i];
 356             spare_for_min = zurl_in_spare[i];
 357         }
 358         if (max_spare < zurl_in_spare[i])
 359         {
 360             ret_spare = url[i];
 361             max_spare = zurl_in_spare[i];
 362         }
 363     }
 364     // use the one with minimum connections if spare is > 3
 365     if (spare_for_min > 3)
 366         return ret_min;
 367     // use one with most spares (if any)
 368     if (max_spare > 0)
 369         return ret_spare;
 370     return ret_min;
 371 }
 372
 373 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 374                                        const char *proxy_host)
 375 {
 376     assert (m_parent);
 377     Yaz_Proxy *parent = m_parent;
 378     Yaz_ProxyClient *c = m_client;
 379
 380     if (!m_proxyTarget)
 381     {
 382         const char *url[MAX_ZURL_PLEX];
 383         Yaz_ProxyConfig *cfg = check_reconfigure();
 384         if (proxy_host)
 385         {
 386 #if 0
 387 /* only to be enabled for debugging... */
 388             if (!strcmp(proxy_host, "stop"))
 389                 exit(0);
 390 #endif
 391             xfree(m_default_target);
 392             m_default_target = xstrdup(proxy_host);
 393         }
 394         proxy_host = m_default_target;
 395         int client_idletime = -1;
 396         const char *cql2rpn_fname = 0;
 397         const char *authentication = 0;
 398         url[0] = m_default_target;
 399         url[1] = 0;
 400         if (cfg)
 401         {
 402             int pre_init = 0;
 403             cfg->get_target_info(proxy_host, url, &m_bw_max,
 404                                  &m_pdu_max, &m_max_record_retrieve,
 405                                  &m_target_idletime, &client_idletime,
 406                                  &parent->m_max_clients,
 407                                  &m_keepalive_limit_bw,
 408                                  &m_keepalive_limit_pdu,
 409                                  &pre_init,
 410                                  &cql2rpn_fname,
 411                                  &authentication);
 412         }
 413         if (client_idletime != -1)
 414         {
 415             m_client_idletime = client_idletime;
 416             timeout(m_client_idletime);
 417         }
 418         if (cql2rpn_fname)
 419             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 420         if (authentication)
 421             set_proxy_authentication(authentication);
 422         if (!url[0])
 423         {
 424             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 425             return 0;
 426         }
 427         // we don't handle multiplexing for cookie session, so we just
 428         // pick the first one in this case (anonymous users will be able
 429         // to use any backend)
 430         if (cookie && *cookie)
 431             m_proxyTarget = (char*) xstrdup(url[0]);
 432         else
 433             m_proxyTarget = (char*) xstrdup(load_balance(url));
 434     }
 435     if (cookie && *cookie)
 436     {   // search in sessions with a cookie
 437         for (c = parent->m_clientPool; c; c = c->m_next)
 438         {
 439             assert (c->m_prev);
 440             assert (*c->m_prev == c);
 441             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 442                 !strcmp(m_proxyTarget, c->get_hostname()))
 443             {
 444                 // Found it in cache
 445                 // The following handles "cancel"
 446                 // If connection is busy (waiting for PDU) and
 447                 // we have an initRequest we can safely do re-open
 448                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 449                 {
 450                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 451                              c->get_hostname());
 452                     c->close();
 453                     c->m_init_flag = 0;
 454
 455                     c->m_last_ok = 0;
 456                     c->m_cache.clear();
 457                     c->m_last_resultCount = 0;
 458                     c->m_sr_transform = 0;
 459                     c->m_waiting = 0;
 460                     c->m_resultSetStartPoint = 0;
 461                     c->m_target_idletime = m_target_idletime;
 462                     if (c->client(m_proxyTarget))
 463                     {
 464                         delete c;
 465                         return 0;
 466                     }
 467                     c->timeout(30);
 468                 }
 469                 c->m_seqno = parent->m_seqno;
 470                 if (c->m_server && c->m_server != this)
 471                     c->m_server->m_client = 0;
 472                 c->m_server = this;
 473                 (parent->m_seqno)++;
 474                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 475                 return c;
 476             }
 477         }
 478     }
 479     else if (!c && apdu->which == Z_APDU_initRequest
 480              && apdu->u.initRequest->idAuthentication == 0)
 481     {
 482         // anonymous sessions without cookie.
 483         // if authentication is set it is NOT anonymous se we can't share them.
 484         for (c = parent->m_clientPool; c; c = c->m_next)
 485         {
 486             assert(c->m_prev);
 487             assert(*c->m_prev == c);
 488             if (c->m_server == 0 && c->m_cookie == 0 &&
 489                 c->m_waiting == 0 &&
 490                 !strcmp(m_proxyTarget, c->get_hostname()))
 491             {
 492                 // found it in cache
 493                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 494                          m_session_str, parent->m_seqno, c->get_hostname());
 495
 496                 c->m_seqno = parent->m_seqno;
 497                 assert(c->m_server == 0);
 498                 c->m_server = this;
 499
 500                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 501                     c->set_APDU_yazlog(1);
 502                 else
 503                     c->set_APDU_yazlog(0);
 504
 505                 (parent->m_seqno)++;
 506
 507                 parent->pre_init();
 508
 509                 return c;
 510             }
 511         }
 512     }
 513     if (!m_client)
 514     {
 515         if (apdu->which != Z_APDU_initRequest)
 516         {
 517             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 518             return 0;
 519         }
 520         Z_InitRequest *initRequest = apdu->u.initRequest;
 521
 522         if (!initRequest->idAuthentication)
 523         {
 524             if (m_proxy_authentication)
 525             {
 526                 initRequest->idAuthentication =
 527                     (Z_IdAuthentication *)
 528                     odr_malloc (odr_encode(),
 529                                 sizeof(*initRequest->idAuthentication));
 530                 initRequest->idAuthentication->which =
 531                     Z_IdAuthentication_open;
 532                 initRequest->idAuthentication->u.open =
 533                     odr_strdup (odr_encode(), m_proxy_authentication);
 534             }
 535         }
 536         else
 537         {
 538             // the client use authentication. We set the keepalive PDU
 539             // to 0 so we don't cache it in releaseClient
 540             m_keepalive_limit_pdu = 0;
 541         }
 542         // go through list of clients - and find the lowest/oldest one.
 543         Yaz_ProxyClient *c_min = 0;
 544         int min_seq = -1;
 545         int no_of_clients = 0;
 546         if (parent->m_clientPool)
 547             yaz_log (YLOG_DEBUG, "Existing sessions");
 548         for (c = parent->m_clientPool; c; c = c->m_next)
 549         {
 550             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 551                                c->m_waiting, c->get_hostname(),
 552                                c->m_cookie ? c->m_cookie : "");
 553             no_of_clients++;
 554             if (min_seq < 0 || c->m_seqno < min_seq)
 555             {
 556                 min_seq = c->m_seqno;
 557                 c_min = c;
 558             }
 559         }
 560         if (no_of_clients >= parent->m_max_clients)
 561         {
 562             c = c_min;
 563             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 564             {
 565                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 566                          m_session_str, parent->m_max_clients, c->m_seqno);
 567                 if (c->m_server && c->m_server != this)
 568                     delete c->m_server;
 569                 c->m_server = 0;
 570             }
 571             else
 572             {
 573                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 574                          m_session_str, parent->m_max_clients,
 575                          c->m_seqno, parent->m_seqno, c->get_hostname());
 576                 xfree (c->m_cookie);
 577                 c->m_cookie = 0;
 578                 if (cookie)
 579                     c->m_cookie = xstrdup(cookie);
 580                 c->m_seqno = parent->m_seqno;
 581                 if (c->m_server && c->m_server != this)
 582                 {
 583                     c->m_server->m_client = 0;
 584                     delete c->m_server;
 585                 }
 586                 (parent->m_seqno)++;
 587                 c->m_target_idletime = m_target_idletime;
 588                 c->timeout(m_target_idletime);
 589
 590                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 591                     c->set_APDU_yazlog(1);
 592                 else
 593                     c->set_APDU_yazlog(0);
 594
 595                 return c;
 596             }
 597         }
 598         else
 599         {
 600             yaz_log (YLOG_LOG, "%sNEW %d %s",
 601                      m_session_str, parent->m_seqno, m_proxyTarget);
 602             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 603             c->m_next = parent->m_clientPool;
 604             if (c->m_next)
 605                 c->m_next->m_prev = &c->m_next;
 606             parent->m_clientPool = c;
 607             c->m_prev = &parent->m_clientPool;
 608         }
 609
 610         xfree (c->m_cookie);
 611         c->m_cookie = 0;
 612         if (cookie)
 613             c->m_cookie = xstrdup(cookie);
 614
 615         c->m_seqno = parent->m_seqno;
 616         c->m_init_flag = 0;
 617         c->m_last_resultCount = 0;
 618         c->m_last_ok = 0;
 619         c->m_cache.clear();
 620         c->m_sr_transform = 0;
 621         c->m_waiting = 0;
 622         c->m_resultSetStartPoint = 0;
 623         (parent->m_seqno)++;
 624         if (c->client(m_proxyTarget))
 625         {
 626             delete c;
 627             return 0;
 628         }
 629         c->m_target_idletime = m_target_idletime;
 630         c->timeout(30);
 631
 632         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 633             c->set_APDU_yazlog(1);
 634         else
 635             c->set_APDU_yazlog(0);
 636     }
 637     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 638     return c;
 639 }
 640
 641 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 642 {
 643     int i;
 644     for (i = 0; i<num; i++)
 645     {
 646         oident *ent;
 647         Z_DefaultDiagFormat *r;
 648         Z_DiagRec *p = pp[i];
 649         if (p->which != Z_DiagRec_defaultFormat)
 650         {
 651             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 652             return;
 653         }
 654         else
 655             r = p->u.defaultFormat;
 656         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 657             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 658             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 659         switch (r->which)
 660         {
 661         case Z_DefaultDiagFormat_v2Addinfo:
 662             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 663                     m_session_str,
 664                     *r->condition, diagbib1_str(*r->condition),
 665                     r->u.v2Addinfo);
 666             break;
 667         case Z_DefaultDiagFormat_v3Addinfo:
 668             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 669                     m_session_str,
 670                     *r->condition, diagbib1_str(*r->condition),
 671                     r->u.v3Addinfo);
 672             break;
 673         }
 674     }
 675 }
 676
 677 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 678 {
 679     if (!m_stylesheet_xsp || p->num_records <= 0)
 680         return 0;  /* no XSLT to be done ... */
 681
 682     m_stylesheet_offset = 0;
 683     m_stylesheet_nprl = p;
 684     m_stylesheet_apdu = apdu;
 685     timeout(0);
 686     return 1;
 687 }
 688
 689 void Yaz_Proxy::convert_xsl_delay()
 690 {
 691     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 692 #if HAVE_XSLT
 693     if (npr->which == Z_NamePlusRecord_databaseRecord)
 694     {
 695         Z_External *r = npr->u.databaseRecord;
 696         if (r->which == Z_External_octet)
 697         {
 698 #if 0
 699             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 700 #endif
 701             xmlDocPtr res, doc = xmlParseMemory(
 702                 (char*) r->u.octet_aligned->buf,
 703                 r->u.octet_aligned->len);
 704
 705
 706             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 707                     m_session_str, m_stylesheet_offset);
 708             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 709                                       doc, 0);
 710
 711             if (res)
 712             {
 713                 xmlChar *out_buf;
 714                 int out_len;
 715                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 716
 717                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 718                     u.databaseRecord =
 719                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 720                                  (char*) out_buf, out_len);
 721                 xmlFree(out_buf);
 722                 xmlFreeDoc(res);
 723             }
 724
 725             xmlFreeDoc(doc);
 726         }
 727     }
 728 #endif
 729     m_stylesheet_offset++;
 730     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 731     {
 732         m_stylesheet_nprl = 0;
 733 #if HAVE_XSLT
 734         if (m_stylesheet_xsp)
 735             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 736 #endif
 737         m_stylesheet_xsp = 0;
 738         timeout(m_client_idletime);
 739         send_PDU_convert(m_stylesheet_apdu);
 740     }
 741     else
 742         timeout(0);
 743 }
 744
 745 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 746 {
 747     if (m_frontend_type != VAL_NONE)
 748     {
 749         int i;
 750         for (i = 0; i < p->num_records; i++)
 751         {
 752             Z_NamePlusRecord *npr = p->records[i];
 753             if (npr->which == Z_NamePlusRecord_databaseRecord)
 754             {
 755                 Z_External *r = npr->u.databaseRecord;
 756                 if (r->which == Z_External_octet)
 757                 {
 758 #if HAVE_USEMARCON
 759                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 760                     {
 761                         if (!m_usemarcon->m_stage1)
 762                         {
 763                             m_usemarcon->m_stage1 = new CDetails();
 764                         }
 765                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 766                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 767                         int res = m_usemarcon->m_stage1->Start();
 768                         if (res == 0)
 769                         {
 770                             char *converted;
 771                             int convlen;
 772                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 773                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 774                             {
 775                                 if (!m_usemarcon->m_stage2)
 776                                 {
 777                                     m_usemarcon->m_stage2 = new CDetails();
 778                                 }
 779                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 780                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 781                                 res = m_usemarcon->m_stage2->Start();
 782                                 if (res == 0)
 783                                 {
 784                                     free(converted);
 785                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 786                                 }
 787                                 else
 788                                 {
 789                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 790                                 }
 791                             }
 792                             npr->u.databaseRecord =
 793                                 z_ext_record(odr_encode(),
 794                                              m_frontend_type,
 795                                              converted,
 796                                              strlen(converted));
 797                             free(converted);
 798                         }
 799                         else
 800                         {
 801                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 802                         }
 803                         continue;
 804                     }
 805 #endif
 806                     npr->u.databaseRecord =
 807                         z_ext_record(odr_encode(),
 808                                      m_frontend_type,
 809                                      (char*) r->u.octet_aligned->buf,
 810                                      r->u.octet_aligned->len);
 811                 }
 812             }
 813         }
 814     }
 815 }
 816
 817 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
 818                                    const char *backend_charset)
 819 {
 820     int i;
 821
 822     if (!backend_charset)
 823         backend_charset = "MARC-8";
 824     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
 825     yaz_marc_t mt = yaz_marc_create();
 826     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 827     yaz_marc_iconv(mt, cd);
 828     for (i = 0; i < p->num_records; i++)
 829     {
 830         Z_NamePlusRecord *npr = p->records[i];
 831         if (npr->which == Z_NamePlusRecord_databaseRecord)
 832         {
 833             Z_External *r = npr->u.databaseRecord;
 834             if (r->which == Z_External_octet)
 835             {
 836                 int rlen;
 837                 char *result;
 838                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 839                                         r->u.octet_aligned->len,
 840                                         &result, &rlen))
 841                 {
 842                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 843                                                          VAL_TEXT_XML,
 844                                                          result, rlen);
 845                 }
 846             }
 847         }
 848     }
 849     if (cd)
 850         yaz_iconv_close(cd);
 851     yaz_marc_destroy(mt);
 852 }
 853
 854 void Yaz_Proxy::logtime()
 855 {
 856 #if HAVE_GETTIMEOFDAY
 857     struct timeval *tv = (struct timeval*) m_time_tv;
 858     if (tv->tv_sec)
 859     {
 860         struct timeval tv1;
 861         gettimeofday(&tv1, 0);
 862         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
 863             (tv1.tv_usec - tv->tv_usec);
 864         if (diff >= 0)
 865             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 866                     diff/1000000, (diff/1000)%1000);
 867     }
 868     tv->tv_sec = 0;
 869     tv->tv_usec = 0;
 870 #endif
 871 }
 872
 873 int Yaz_Proxy::send_http_response(int code)
 874 {
 875     ODR o = odr_encode();
 876     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 877     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 878     if (m_http_version)
 879         hres->version = odr_strdup(o, m_http_version);
 880     if (m_http_keepalive)
 881         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 882     else
 883         timeout(0);
 884
 885     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 886     {
 887         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
 888                  gdu_name(gdu));
 889     }
 890     int len;
 891     int r = send_GDU(gdu, &len);
 892     m_bytes_sent += len;
 893     m_bw_stat.add_bytes(len);
 894     logtime();
 895     return r;
 896 }
 897
 898 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 899 {
 900     ODR o = odr_encode();
 901     const char *ctype = "text/xml";
 902     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 903     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 904     if (m_http_version)
 905         hres->version = odr_strdup(o, m_http_version);
 906     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 907     if (m_http_keepalive)
 908         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 909     else
 910         timeout(0);
 911
 912     static Z_SOAP_Handler soap_handlers[2] = {
 913 #if HAVE_XSLT
 914         {"http://www.loc.gov/zing/srw/", 0,
 915          (Z_SOAP_fun) yaz_srw_codec},
 916 #endif
 917         {0, 0, 0}
 918     };
 919
 920     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 921     soap_package->which = Z_SOAP_generic;
 922     soap_package->u.generic =
 923         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 924     soap_package->u.generic->no = 0;
 925     soap_package->u.generic->ns = soap_handlers[0].ns;
 926     soap_package->u.generic->p = (void *) srw_pdu;
 927     soap_package->ns = m_soap_ns;
 928     z_soap_codec_enc_xsl(o, &soap_package,
 929                          &hres->content_buf, &hres->content_len,
 930                          soap_handlers, 0, m_s2z_stylesheet);
 931     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 932     {
 933         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
 934                  gdu_name(gdu));
 935     }
 936     int len;
 937     int r = send_GDU(gdu, &len);
 938     m_bytes_sent += len;
 939     m_bw_stat.add_bytes(len);
 940     logtime();
 941     return r;
 942 }
 943
 944 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
 945 {
 946     ODR o = odr_encode();
 947     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 948     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 949
 950     srw_res->num_diagnostics = 1;
 951     srw_res->diagnostics = (Z_SRW_diagnostic *)
 952         odr_malloc(o, sizeof(*srw_res->diagnostics));
 953     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
 954     return send_srw_response(srw_pdu);
 955 }
 956
 957 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 958                              Z_DefaultDiagFormat *ddf)
 959 {
 960     int bib1_code = *ddf->condition;
 961     if (bib1_code == 109)
 962         return 404;
 963     srw_res->num_diagnostics = 1;
 964     srw_res->diagnostics = (Z_SRW_diagnostic *)
 965         odr_malloc(o, sizeof(*srw_res->diagnostics));
 966     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
 967                           yaz_diag_bib1_to_srw(*ddf->condition),
 968                           ddf->u.v2Addinfo);
 969     return 0;
 970 }
 971
 972 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
 973 {
 974     ODR o = odr_encode();
 975     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 976     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 977
 978     srw_res->numberOfRecords = odr_intdup (o, hits);
 979     if (records && records->which == Z_Records_DBOSD)
 980     {
 981         srw_res->num_records =
 982             records->u.databaseOrSurDiagnostics->num_records;
 983         int i;
 984         srw_res->records = (Z_SRW_record *)
 985             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
 986         for (i = 0; i < srw_res->num_records; i++)
 987         {
 988             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
 989             if (npr->which != Z_NamePlusRecord_databaseRecord)
 990             {
 991                 srw_res->records[i].recordSchema = "diagnostic";
 992                 srw_res->records[i].recordPacking = m_s2z_packing;
 993                 srw_res->records[i].recordData_buf = "67";
 994                 srw_res->records[i].recordData_len = 2;
 995                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 996                 continue;
 997             }
 998             Z_External *r = npr->u.databaseRecord;
 999             oident *ent = oid_getentbyoid(r->direct_reference);
1000             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1001             {
1002                 srw_res->records[i].recordSchema = m_schema;
1003                 srw_res->records[i].recordPacking = m_s2z_packing;
1004                 srw_res->records[i].recordData_buf = (char*)
1005                     r->u.octet_aligned->buf;
1006                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1007                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1008             }
1009             else
1010             {
1011                 srw_res->records[i].recordSchema = "diagnostic";
1012                 srw_res->records[i].recordPacking = m_s2z_packing;
1013                 srw_res->records[i].recordData_buf = "67";
1014                 srw_res->records[i].recordData_len = 2;
1015                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1016             }
1017         }
1018     }
1019     if (records && records->which == Z_Records_NSD)
1020     {
1021         int http_code;
1022         http_code = z_to_srw_diag(odr_encode(), srw_res,
1023                                    records->u.nonSurrogateDiagnostic);
1024         if (http_code)
1025             return send_http_response(http_code);
1026     }
1027     return send_srw_response(srw_pdu);
1028
1029 }
1030
1031 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1032                                         int num_diagnostics)
1033 {
1034     Yaz_ProxyConfig *cfg = check_reconfigure();
1035     if (cfg)
1036     {
1037         int len;
1038         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1039                                        m_s2z_database, &len);
1040         if (b)
1041         {
1042             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1043             Z_SRW_explainResponse *er = res->u.explain_response;
1044
1045             er->record.recordData_buf = b;
1046             er->record.recordData_len = len;
1047             er->record.recordPacking = m_s2z_packing;
1048             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1049
1050             er->diagnostics = diagnostics;
1051             er->num_diagnostics = num_diagnostics;
1052             return send_srw_response(res);
1053         }
1054     }
1055     return send_http_response(404);
1056 }
1057
1058 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1059 {
1060     if (m_http_version)
1061     {
1062         if (apdu->which == Z_APDU_initResponse)
1063         {
1064             Z_InitResponse *res = apdu->u.initResponse;
1065             if (*res->result == 0)
1066             {
1067                 send_to_srw_client_error(3, 0);
1068             }
1069             else if (!m_s2z_search_apdu)
1070             {
1071                 send_srw_explain_response(0, 0);
1072             }
1073             else
1074             {
1075                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1076             }
1077         }
1078         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1079         {
1080             m_s2z_search_apdu = 0;
1081             Z_SearchResponse *res = apdu->u.searchResponse;
1082             m_s2z_hit_count = *res->resultCount;
1083             if (res->records && res->records->which == Z_Records_NSD)
1084             {
1085                 send_to_srw_client_ok(0, res->records, 1);
1086             }
1087             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1088             {
1089                 // adjust
1090                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1091
1092                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1093                 {
1094                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1095                         > m_s2z_hit_count)
1096                         *pr->numberOfRecordsRequested =
1097                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1098                 }
1099                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1100             }
1101             else
1102             {
1103                 m_s2z_present_apdu = 0;
1104                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1105             }
1106         }
1107         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1108         {
1109             int start =
1110                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1111
1112             m_s2z_present_apdu = 0;
1113             Z_PresentResponse *res = apdu->u.presentResponse;
1114             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1115         }
1116     }
1117     else
1118     {
1119         int len = 0;
1120         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1121             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1122                      apdu_name(apdu));
1123         int r = send_Z_PDU(apdu, &len);
1124         m_bytes_sent += len;
1125         m_bw_stat.add_bytes(len);
1126         logtime();
1127         return r;
1128     }
1129     return 0;
1130 }
1131
1132 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1133 {
1134     int kill_session = 0;
1135     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1136
1137     if (new_id)
1138         *new_id = m_referenceId;
1139
1140     if (apdu->which == Z_APDU_searchResponse)
1141     {
1142         Z_SearchResponse *sr = apdu->u.searchResponse;
1143         Z_Records *p = sr->records;
1144         if (p && p->which == Z_Records_NSD)
1145         {
1146             Z_DiagRec dr, *dr_p = &dr;
1147             dr.which = Z_DiagRec_defaultFormat;
1148             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1149
1150             *sr->searchStatus = 0;
1151             display_diagrecs(&dr_p, 1);
1152         }
1153         else
1154         {
1155             if (p && p->which == Z_Records_DBOSD)
1156             {
1157                 if (m_backend_type
1158 #if HAVE_USEMARCON
1159                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1160 #endif
1161                     )
1162                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1163                 if (m_marcxml_flag)
1164                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1165                                        m_backend_charset);
1166                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1167                     return 0;
1168
1169             }
1170             if (sr->resultCount)
1171             {
1172                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1173                         *sr->resultCount);
1174                 if (*sr->resultCount < 0)
1175                 {
1176                     m_invalid_session = 1;
1177                     kill_session = 1;
1178
1179                     *sr->searchStatus = 0;
1180                     sr->records =
1181                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1182                     *sr->resultCount = 0;
1183                 }
1184             }
1185         }
1186     }
1187     else if (apdu->which == Z_APDU_presentResponse)
1188     {
1189         Z_PresentResponse *sr = apdu->u.presentResponse;
1190         Z_Records *p = sr->records;
1191         if (p && p->which == Z_Records_NSD)
1192         {
1193             Z_DiagRec dr, *dr_p = &dr;
1194             dr.which = Z_DiagRec_defaultFormat;
1195             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1196             if (*sr->presentStatus == Z_PresentStatus_success)
1197                 *sr->presentStatus = Z_PresentStatus_failure;
1198             display_diagrecs(&dr_p, 1);
1199         }
1200         if (p && p->which == Z_Records_DBOSD)
1201         {
1202             if (m_backend_type
1203 #if HAVE_USEMARCON
1204                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1205 #endif
1206                 )
1207                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1208             if (m_marcxml_flag)
1209                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1210                                    m_backend_charset);
1211             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1212                 return 0;
1213         }
1214     }
1215     else if (apdu->which == Z_APDU_initResponse)
1216     {
1217         if (m_initRequest_options)
1218         {
1219             Z_Options *nopt =
1220                 (Odr_bitmask *)odr_malloc(odr_encode(),
1221                                           sizeof(Odr_bitmask));
1222             ODR_MASK_ZERO(nopt);
1223
1224             int i;
1225             for (i = 0; i<24; i++)
1226                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1227                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1228                     ODR_MASK_SET(nopt, i);
1229             apdu->u.initResponse->options = nopt;
1230         }
1231         if (m_initRequest_version)
1232         {
1233             Z_ProtocolVersion *nopt =
1234                 (Odr_bitmask *)odr_malloc(odr_encode(),
1235                                           sizeof(Odr_bitmask));
1236             ODR_MASK_ZERO(nopt);
1237
1238             int i;
1239             for (i = 0; i<8; i++)
1240                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1241                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1242                     ODR_MASK_SET(nopt, i);
1243             apdu->u.initResponse->protocolVersion = nopt;
1244         }
1245         apdu->u.initResponse->preferredMessageSize =
1246             odr_intdup(odr_encode(),
1247                        m_client->m_initResponse_preferredMessageSize >
1248                        m_initRequest_preferredMessageSize ?
1249                        m_initRequest_preferredMessageSize :
1250                        m_client->m_initResponse_preferredMessageSize);
1251         apdu->u.initResponse->maximumRecordSize =
1252             odr_intdup(odr_encode(),
1253                        m_client->m_initResponse_maximumRecordSize >
1254                        m_initRequest_maximumRecordSize ?
1255                        m_initRequest_maximumRecordSize :
1256                        m_client->m_initResponse_maximumRecordSize);
1257     }
1258     int r = send_PDU_convert(apdu);
1259     if (r)
1260         return r;
1261     if (kill_session)
1262     {
1263         delete m_client;
1264         m_client = 0;
1265         m_parent->pre_init();
1266     }
1267     return r;
1268 }
1269
1270 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1271 {
1272     int len = 0;
1273     const char *apdu_name_tmp = apdu_name(apdu);
1274     int r = send_Z_PDU(apdu, &len);
1275     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1276         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1277                  get_session_str(),
1278                  apdu_name_tmp, get_hostname(), len);
1279     m_bytes_sent += len;
1280     return r;
1281 }
1282
1283 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1284 {
1285     if (apdu->which == Z_APDU_presentRequest)
1286     {
1287         Z_PresentRequest *pr = apdu->u.presentRequest;
1288         int toget = *pr->numberOfRecordsRequested;
1289         int start = *pr->resultSetStartPoint;
1290
1291         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1292                 pr->resultSetId, start, toget);
1293
1294         if (*m_parent->m_optimize == '0')
1295             return apdu;
1296
1297         if (!m_client->m_last_resultSetId)
1298         {
1299             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1300             new_apdu->u.presentResponse->records =
1301                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1302                                                pr->resultSetId);
1303             send_to_client(new_apdu);
1304             return 0;
1305         }
1306         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1307         {
1308             if (start+toget-1 > m_client->m_last_resultCount)
1309             {
1310                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1311                 new_apdu->u.presentResponse->records =
1312                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1313                 send_to_client(new_apdu);
1314                 return 0;
1315             }
1316             Z_NamePlusRecordList *npr;
1317 #if 0
1318             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1319                     m_session_str, start, toget, yaz_z3950oid_to_str(
1320                         pr->preferredRecordSyntax, &oclass));
1321 #endif
1322             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1323                                           pr->preferredRecordSyntax,
1324                                           pr->recordComposition))
1325             {
1326                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1327                          m_session_str);
1328                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1329                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1330
1331                 new_apdu->u.presentResponse->numberOfRecordsReturned
1332                     = odr_intdup(odr_encode(), toget);
1333
1334                 new_apdu->u.presentResponse->records = (Z_Records*)
1335                     odr_malloc(odr_encode(), sizeof(Z_Records));
1336                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1337                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1338                 new_apdu->u.presentResponse->nextResultSetPosition =
1339                     odr_intdup(odr_encode(), start+toget);
1340
1341                 send_to_client(new_apdu);
1342                 return 0;
1343             }
1344         }
1345     }
1346
1347     if (apdu->which != Z_APDU_searchRequest)
1348         return apdu;
1349     Z_SearchRequest *sr = apdu->u.searchRequest;
1350     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1351     Yaz_Z_Databases this_databases;
1352
1353     this_databases.set(sr->num_databaseNames, (const char **)
1354                        sr->databaseNames);
1355
1356     this_query->set_Z_Query(sr->query);
1357
1358     char query_str[120];
1359     this_query->print(query_str, sizeof(query_str)-1);
1360     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1361
1362     if (*m_parent->m_optimize != '0' &&
1363         m_client->m_last_ok && m_client->m_last_query &&
1364         m_client->m_last_query->match(this_query) &&
1365         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1366         m_client->m_last_databases.match(this_databases))
1367     {
1368         delete this_query;
1369         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1370             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1371         {
1372             Z_NamePlusRecordList *npr;
1373             int toget = *sr->mediumSetPresentNumber;
1374             Z_RecordComposition *comp = 0;
1375
1376             if (toget > m_client->m_last_resultCount)
1377                 toget = m_client->m_last_resultCount;
1378
1379             if (sr->mediumSetElementSetNames)
1380             {
1381                 comp = (Z_RecordComposition *)
1382                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1383                 comp->which = Z_RecordComp_simple;
1384                 comp->u.simple = sr->mediumSetElementSetNames;
1385             }
1386
1387             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1388                                           sr->preferredRecordSyntax, comp))
1389             {
1390                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1391                          m_session_str);
1392                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1393                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1394                 new_apdu->u.searchResponse->resultCount =
1395                     &m_client->m_last_resultCount;
1396
1397                 new_apdu->u.searchResponse->numberOfRecordsReturned
1398                     = odr_intdup(odr_encode(), toget);
1399
1400                 new_apdu->u.searchResponse->presentStatus =
1401                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1402                 new_apdu->u.searchResponse->records = (Z_Records*)
1403                     odr_malloc(odr_encode(), sizeof(Z_Records));
1404                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1405                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1406                 new_apdu->u.searchResponse->nextResultSetPosition =
1407                     odr_intdup(odr_encode(), toget+1);
1408                 send_to_client(new_apdu);
1409                 return 0;
1410             }
1411             else
1412             {
1413                 // medium Set
1414                 // send present request (medium size)
1415                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1416                          m_session_str);
1417
1418                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1419                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1420                 pr->referenceId = sr->referenceId;
1421                 pr->resultSetId = sr->resultSetName;
1422                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1423                 *pr->numberOfRecordsRequested = toget;
1424                 pr->recordComposition = comp;
1425                 m_client->m_sr_transform = 1;
1426                 return new_apdu;
1427             }
1428         }
1429         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1430             m_client->m_last_resultCount <= 0)
1431         {
1432             // large set. Return pseudo-search response immediately
1433             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1434                      m_session_str);
1435             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1436             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1437             new_apdu->u.searchResponse->resultCount =
1438                 &m_client->m_last_resultCount;
1439             send_to_client(new_apdu);
1440             return 0;
1441         }
1442         else
1443         {
1444             Z_NamePlusRecordList *npr;
1445             int toget = m_client->m_last_resultCount;
1446             Z_RecordComposition *comp = 0;
1447             // small set
1448             // send a present request (small set)
1449
1450             if (sr->smallSetElementSetNames)
1451             {
1452                 comp = (Z_RecordComposition *)
1453                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1454                 comp->which = Z_RecordComp_simple;
1455                 comp->u.simple = sr->smallSetElementSetNames;
1456             }
1457
1458             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1459                                           sr->preferredRecordSyntax, comp))
1460             {
1461                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1462                          m_session_str);
1463                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1464                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1465                 new_apdu->u.searchResponse->resultCount =
1466                     &m_client->m_last_resultCount;
1467
1468                 new_apdu->u.searchResponse->numberOfRecordsReturned
1469                     = odr_intdup(odr_encode(), toget);
1470
1471                 new_apdu->u.searchResponse->presentStatus =
1472                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1473                 new_apdu->u.searchResponse->records = (Z_Records*)
1474                     odr_malloc(odr_encode(), sizeof(Z_Records));
1475                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1476                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1477                 new_apdu->u.searchResponse->nextResultSetPosition =
1478                     odr_intdup(odr_encode(), toget+1);
1479                 send_to_client(new_apdu);
1480                 return 0;
1481             }
1482             else
1483             {
1484                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1485                          m_session_str);
1486                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1487                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1488                 pr->referenceId = sr->referenceId;
1489                 pr->resultSetId = sr->resultSetName;
1490                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1491                 *pr->numberOfRecordsRequested = toget;
1492                 pr->recordComposition = comp;
1493                 m_client->m_sr_transform = 1;
1494                 return new_apdu;
1495             }
1496         }
1497     }
1498     else  // query doesn't match
1499     {
1500         delete m_client->m_last_query;
1501         m_client->m_last_query = this_query;
1502         m_client->m_last_ok = 0;
1503         m_client->m_cache.clear();
1504         m_client->m_resultSetStartPoint = 0;
1505
1506         xfree (m_client->m_last_resultSetId);
1507         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1508
1509         m_client->m_last_databases.set(sr->num_databaseNames,
1510                                        (const char **) sr->databaseNames);
1511     }
1512     return apdu;
1513 }
1514
1515
1516 void Yaz_Proxy::inc_request_no()
1517 {
1518     char *cp = strchr(m_session_str, ' ');
1519     m_request_no++;
1520     if (cp)
1521         sprintf(cp+1, "%d ", m_request_no);
1522 }
1523
1524 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1525 {
1526     inc_request_no();
1527
1528     m_bytes_recv += len;
1529
1530     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1531         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1532                  m_session_str, gdu_name(apdu), len);
1533
1534     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1535         shutdown();
1536
1537     m_bw_stat.add_bytes(len);
1538     m_pdu_stat.add_bytes(1);
1539
1540 #if HAVE_GETTIMEOFDAY
1541     gettimeofday((struct timeval *) m_time_tv, 0);
1542 #endif
1543
1544     int bw_total = m_bw_stat.get_total();
1545     int pdu_total = m_pdu_stat.get_total();
1546
1547     int reduce = 0;
1548     if (m_bw_max)
1549     {
1550         if (bw_total > m_bw_max)
1551         {
1552             reduce = (bw_total/m_bw_max);
1553         }
1554     }
1555     if (m_pdu_max)
1556     {
1557         if (pdu_total > m_pdu_max)
1558         {
1559             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1560             reduce = (reduce > nreduce) ? reduce : nreduce;
1561         }
1562     }
1563     if (reduce)
1564     {
1565         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1566                 m_session_str, reduce, bw_total, pdu_total,
1567                 m_bw_max, m_pdu_max);
1568
1569         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1570         timeout(reduce);       // call us reduce seconds later
1571     }
1572     else if (apdu->which == Z_GDU_Z3950)
1573         handle_incoming_Z_PDU(apdu->u.z3950);
1574     else if (apdu->which == Z_GDU_HTTP_Request)
1575         handle_incoming_HTTP(apdu->u.HTTP_Request);
1576 }
1577
1578 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1579 {
1580     if (m_max_record_retrieve)
1581     {
1582         if (apdu->which == Z_APDU_presentRequest)
1583         {
1584             Z_PresentRequest *pr = apdu->u.presentRequest;
1585             if (pr->numberOfRecordsRequested &&
1586                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1587                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1588         }
1589     }
1590 }
1591
1592 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1593                                                      int error,
1594                                                      const char *addinfo)
1595 {
1596     Z_Records *rec = (Z_Records *)
1597         odr_malloc (odr, sizeof(*rec));
1598     int *err = (int *)
1599         odr_malloc (odr, sizeof(*err));
1600     Z_DiagRec *drec = (Z_DiagRec *)
1601         odr_malloc (odr, sizeof(*drec));
1602     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1603         odr_malloc (odr, sizeof(*dr));
1604     *err = error;
1605     rec->which = Z_Records_NSD;
1606     rec->u.nonSurrogateDiagnostic = dr;
1607     dr->diagnosticSetId =
1608         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1609     dr->condition = err;
1610     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1611     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1612     return rec;
1613 }
1614
1615 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1616 {
1617     if (apdu->which == Z_APDU_searchRequest &&
1618         apdu->u.searchRequest->query &&
1619         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1620         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1621     {
1622         Z_RPNQuery *rpnquery = 0;
1623         Z_SearchRequest *sr = apdu->u.searchRequest;
1624         char *addinfo = 0;
1625
1626         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
1627                 sr->query->u.type_104->u.cql);
1628
1629         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1630                                           &rpnquery, odr_encode(),
1631                                           &addinfo);
1632         if (r == -3)
1633             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
1634         else if (r)
1635         {
1636             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1637             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1638
1639             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1640             new_apdu->u.searchResponse->records =
1641                 create_nonSurrogateDiagnostics(odr_encode(),
1642                                                yaz_diag_srw_to_bib1(r),
1643                                                addinfo);
1644             *new_apdu->u.searchResponse->searchStatus = 0;
1645
1646             send_to_client(new_apdu);
1647
1648             return 0;
1649         }
1650         else
1651         {
1652             sr->query->which = Z_Query_type_1;
1653             sr->query->u.type_1 = rpnquery;
1654         }
1655         return apdu;
1656     }
1657     return apdu;
1658 }
1659
1660 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1661 {
1662     if (apdu->which == Z_APDU_searchRequest)
1663     {
1664         Z_SearchRequest *sr = apdu->u.searchRequest;
1665         int err = 0;
1666         char *addinfo = 0;
1667
1668         Yaz_ProxyConfig *cfg = check_reconfigure();
1669         if (cfg)
1670             err = cfg->check_query(odr_encode(), m_default_target,
1671                                    sr->query, &addinfo);
1672         if (err)
1673         {
1674             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1675
1676             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1677             new_apdu->u.searchResponse->records =
1678                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1679             *new_apdu->u.searchResponse->searchStatus = 0;
1680
1681             send_to_client(new_apdu);
1682
1683             return 0;
1684         }
1685     }
1686     return apdu;
1687 }
1688
1689 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1690 {
1691     m_marcxml_flag = 0;
1692     if (apdu->which == Z_APDU_searchRequest)
1693     {
1694         Z_SearchRequest *sr = apdu->u.searchRequest;
1695         int err = 0;
1696         char *addinfo = 0;
1697         Yaz_ProxyConfig *cfg = check_reconfigure();
1698
1699         Z_RecordComposition rc_temp, *rc = 0;
1700         if (sr->smallSetElementSetNames)
1701         {
1702             rc_temp.which = Z_RecordComp_simple;
1703             rc_temp.u.simple = sr->smallSetElementSetNames;
1704             rc = &rc_temp;
1705         }
1706
1707         if (sr->preferredRecordSyntax)
1708         {
1709             struct oident *ent;
1710             ent = oid_getentbyoid(sr->preferredRecordSyntax);
1711             m_frontend_type = ent->value;
1712         }
1713         else
1714             m_frontend_type = VAL_NONE;
1715
1716         char *stylesheet_name = 0;
1717         if (cfg)
1718             err = cfg->check_syntax(odr_encode(),
1719                                     m_default_target,
1720                                     sr->preferredRecordSyntax, rc,
1721                                     &addinfo, &stylesheet_name, &m_schema,
1722                                     &m_backend_type, &m_backend_charset,
1723                                     &m_usemarcon_ini_stage1,
1724                                     &m_usemarcon_ini_stage2);
1725         if (stylesheet_name)
1726         {
1727             m_parent->low_socket_close();
1728
1729 #if HAVE_XSLT
1730             if (m_stylesheet_xsp)
1731                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
1732             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1733                                                        stylesheet_name);
1734 #endif
1735             m_stylesheet_offset = 0;
1736             xfree(stylesheet_name);
1737
1738             m_parent->low_socket_open();
1739         }
1740         if (err == -1)
1741         {
1742             sr->smallSetElementSetNames = 0;
1743             sr->mediumSetElementSetNames = 0;
1744             if (m_backend_type)
1745             {
1746
1747                 sr->preferredRecordSyntax =
1748                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
1749                                         m_backend_type);
1750             }
1751             else
1752                 sr->preferredRecordSyntax =
1753                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
1754                                            VAL_USMARC);
1755             m_marcxml_flag = 1;
1756         }
1757         else if (err)
1758         {
1759             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1760
1761             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1762             new_apdu->u.searchResponse->records =
1763                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1764             *new_apdu->u.searchResponse->searchStatus = 0;
1765
1766             send_to_client(new_apdu);
1767
1768             return 0;
1769         }
1770         else if (m_backend_type)
1771         {
1772             sr->preferredRecordSyntax =
1773                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
1774         }
1775     }
1776     else if (apdu->which == Z_APDU_presentRequest)
1777     {
1778         Z_PresentRequest *pr = apdu->u.presentRequest;
1779         int err = 0;
1780         char *addinfo = 0;
1781         Yaz_ProxyConfig *cfg = check_reconfigure();
1782
1783         if (pr->preferredRecordSyntax)
1784         {
1785             struct oident *ent;
1786             ent = oid_getentbyoid(pr->preferredRecordSyntax);
1787             m_frontend_type = ent->value;
1788         }
1789         else
1790             m_frontend_type = VAL_NONE;
1791
1792         char *stylesheet_name = 0;
1793         if (cfg)
1794             err = cfg->check_syntax(odr_encode(), m_default_target,
1795                                     pr->preferredRecordSyntax,
1796                                     pr->recordComposition,
1797                                     &addinfo, &stylesheet_name, &m_schema,
1798                                     &m_backend_type, &m_backend_charset,
1799                                     &m_usemarcon_ini_stage1,
1800                                     &m_usemarcon_ini_stage2
1801                                     );
1802         if (stylesheet_name)
1803         {
1804             m_parent->low_socket_close();
1805
1806 #if HAVE_XSLT
1807             if (m_stylesheet_xsp)
1808                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
1809             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1810                                                        stylesheet_name);
1811 #endif
1812             m_stylesheet_offset = 0;
1813             xfree(stylesheet_name);
1814
1815             m_parent->low_socket_open();
1816         }
1817         if (err == -1)
1818         {
1819             pr->recordComposition = 0;
1820             if (m_backend_type)
1821             {
1822
1823                 pr->preferredRecordSyntax =
1824                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
1825                                         m_backend_type);
1826             }
1827             else
1828                 pr->preferredRecordSyntax =
1829                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
1830                                            VAL_USMARC);
1831             m_marcxml_flag = 1;
1832         }
1833         else if (err)
1834         {
1835             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1836
1837             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1838             new_apdu->u.presentResponse->records =
1839                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1840             *new_apdu->u.presentResponse->presentStatus =
1841                 Z_PresentStatus_failure;
1842
1843             send_to_client(new_apdu);
1844
1845             return 0;
1846         }
1847         else if (m_backend_type)
1848         {
1849             pr->preferredRecordSyntax =
1850                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
1851         }
1852     }
1853     return apdu;
1854 }
1855
1856 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
1857 {
1858     if (!schema)
1859         return 0;
1860     Z_ElementSetNames *esn = (Z_ElementSetNames *)
1861         odr_malloc(o, sizeof(Z_ElementSetNames));
1862     esn->which = Z_ElementSetNames_generic;
1863     esn->u.generic = odr_strdup(o, schema);
1864     return esn;
1865 }
1866
1867 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
1868 {
1869     const char *t = 0;
1870     Yaz_ProxyConfig *cfg = check_reconfigure();
1871     if (cfg)
1872         t = cfg->get_explain_name(db, backend_db);
1873
1874     if (m_client && m_default_target && t && strcmp(m_default_target, t))
1875     {
1876         releaseClient();
1877     }
1878
1879     if (t)
1880     {
1881         xfree(m_default_target);
1882         m_default_target = xstrdup(t);
1883     }
1884 }
1885
1886 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1887 {
1888     if (m_s2z_odr_init)
1889     {
1890         odr_destroy(m_s2z_odr_init);
1891         m_s2z_odr_init = 0;
1892     }
1893     if (m_s2z_odr_search)
1894     {
1895         odr_destroy(m_s2z_odr_search);
1896         m_s2z_odr_search = 0;
1897     }
1898
1899     m_http_keepalive = 0;
1900     m_http_version = 0;
1901     if (!strcmp(hreq->version, "1.0"))
1902     {
1903         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1904         if (v && !strcmp(v, "Keep-Alive"))
1905             m_http_keepalive = 1;
1906         else
1907             m_http_keepalive = 0;
1908         m_http_version = "1.0";
1909     }
1910     else
1911     {
1912         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1913         if (v && !strcmp(v, "close"))
1914             m_http_keepalive = 0;
1915         else
1916             m_http_keepalive = 1;
1917         m_http_version = "1.1";
1918     }
1919
1920     Z_SRW_PDU *srw_pdu = 0;
1921     Z_SOAP *soap_package = 0;
1922     char *charset = 0;
1923     Z_SRW_diagnostic *diagnostic = 0;
1924     int num_diagnostic = 0;
1925     if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1926                        &charset) == 0
1927         || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1928                           &charset, &diagnostic, &num_diagnostic) == 0)
1929     {
1930         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
1931         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
1932         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
1933         m_s2z_init_apdu = 0;
1934         m_s2z_search_apdu = 0;
1935         m_s2z_present_apdu = 0;
1936
1937         m_s2z_stylesheet = 0;
1938
1939         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
1940         {
1941             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
1942
1943             const char *backend_db = srw_req->database;
1944             srw_get_client(srw_req->database, &backend_db);
1945
1946             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1947             // recordXPath unsupported.
1948             if (srw_req->recordXPath)
1949             {
1950                 yaz_add_srw_diagnostic(odr_decode(),
1951                                        &diagnostic, &num_diagnostic,
1952                                        72, 0);
1953             }
1954             // sort unsupported
1955             if (srw_req->sort_type != Z_SRW_sort_type_none)
1956             {
1957                 yaz_add_srw_diagnostic(odr_decode(),
1958                                        &diagnostic, &num_diagnostic,
1959                                        80, 0);
1960             }
1961             // save stylesheet
1962             if (srw_req->stylesheet)
1963                 m_s2z_stylesheet =
1964                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1965
1966             // set packing for response records ..
1967             if (srw_req->recordPacking &&
1968                 !strcmp(srw_req->recordPacking, "xml"))
1969                 m_s2z_packing = Z_SRW_recordPacking_XML;
1970             else
1971                 m_s2z_packing = Z_SRW_recordPacking_string;
1972
1973             if (num_diagnostic)
1974             {
1975                 Z_SRW_PDU *srw_pdu =
1976                     yaz_srw_get(odr_encode(),
1977                                 Z_SRW_searchRetrieve_response);
1978                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1979
1980                 srw_res->diagnostics = diagnostic;
1981                 srw_res->num_diagnostics = num_diagnostic;
1982                 send_srw_response(srw_pdu);
1983                 return;
1984             }
1985
1986             // prepare search PDU
1987             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
1988                                           Z_APDU_searchRequest);
1989             Z_SearchRequest *z_searchRequest =
1990                 m_s2z_search_apdu->u.searchRequest;
1991
1992             z_searchRequest->num_databaseNames = 1;
1993             z_searchRequest->databaseNames = (char**)
1994                 odr_malloc(m_s2z_odr_search, sizeof(char *));
1995             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
1996                                                            backend_db);
1997
1998             // query transformation
1999             Z_Query *query = (Z_Query *)
2000                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2001             z_searchRequest->query = query;
2002
2003             if (srw_req->query_type == Z_SRW_query_type_cql)
2004             {
2005                 Z_External *ext = (Z_External *)
2006                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2007                 ext->direct_reference =
2008                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2009                 ext->indirect_reference = 0;
2010                 ext->descriptor = 0;
2011                 ext->which = Z_External_CQL;
2012                 ext->u.cql = srw_req->query.cql;
2013
2014                 query->which = Z_Query_type_104;
2015                 query->u.type_104 =  ext;
2016             }
2017             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2018             {
2019                 Z_RPNQuery *RPNquery;
2020                 YAZ_PQF_Parser pqf_parser;
2021
2022                 pqf_parser = yaz_pqf_create ();
2023
2024                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2025                                           srw_req->query.pqf);
2026                 if (!RPNquery)
2027                 {
2028                     const char *pqf_msg;
2029                     size_t off;
2030                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2031                     yaz_log(YLOG_LOG, "%*s^\n", off+4, "");
2032                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2033
2034                     send_to_srw_client_error(10, 0);
2035                     return;
2036                 }
2037                 query->which = Z_Query_type_1;
2038                 query->u.type_1 =  RPNquery;
2039
2040                 yaz_pqf_destroy (pqf_parser);
2041             }
2042             else
2043             {
2044                 send_to_srw_client_error(7, "query");
2045                 return;
2046             }
2047
2048             // present
2049             m_s2z_present_apdu = 0;
2050             int max = 0;
2051             if (srw_req->maximumRecords)
2052                 max = *srw_req->maximumRecords;
2053             int start = 1;
2054             if (srw_req->startRecord)
2055                 start = *srw_req->startRecord;
2056             if (max > 0)
2057             {
2058                 // Some backend, such as Voyager doesn't honor piggyback
2059                 // So we use present always (0 &&).
2060                 if (0 && start <= 1)  // Z39.50 piggyback
2061                 {
2062                     *z_searchRequest->smallSetUpperBound = max;
2063                     *z_searchRequest->mediumSetPresentNumber = max;
2064                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2065
2066                     z_searchRequest->preferredRecordSyntax =
2067                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2068                                                VAL_TEXT_XML);
2069                     if (srw_req->recordSchema)
2070                     {
2071                         z_searchRequest->smallSetElementSetNames =
2072                             z_searchRequest->mediumSetElementSetNames =
2073                             mk_esn_from_schema(m_s2z_odr_search,
2074                                                srw_req->recordSchema);
2075                     }
2076                 }
2077                 else   // Z39.50 present
2078                 {
2079                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2080                                                    Z_APDU_presentRequest);
2081                     Z_PresentRequest *z_presentRequest =
2082                         m_s2z_present_apdu->u.presentRequest;
2083                     *z_presentRequest->resultSetStartPoint = start;
2084                     *z_presentRequest->numberOfRecordsRequested = max;
2085                     z_presentRequest->preferredRecordSyntax =
2086                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2087                                                VAL_TEXT_XML);
2088                     if (srw_req->recordSchema)
2089                     {
2090                         z_presentRequest->recordComposition =
2091                             (Z_RecordComposition *)
2092                             odr_malloc(m_s2z_odr_search,
2093                                        sizeof(Z_RecordComposition));
2094                         z_presentRequest->recordComposition->which =
2095                             Z_RecordComp_simple;
2096                         z_presentRequest->recordComposition->u.simple =
2097                             mk_esn_from_schema(m_s2z_odr_search,
2098                                                srw_req->recordSchema);
2099                     }
2100                 }
2101             }
2102             if (!m_client)
2103             {
2104                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2105                                             Z_APDU_initRequest);
2106
2107                 // prevent m_initRequest_apdu memory from being grabbed
2108                 // in Yaz_Proxy::handle_incoming_Z_PDU
2109                 m_initRequest_apdu = m_s2z_init_apdu;
2110                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2111                 return;
2112             }
2113             else
2114             {
2115                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2116                 return;
2117             }
2118         }
2119         else if (srw_pdu->which == Z_SRW_explain_request)
2120         {
2121             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2122
2123             const char *backend_db = srw_req->database;
2124             srw_get_client(srw_req->database, &backend_db);
2125
2126             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2127
2128             // save stylesheet
2129             if (srw_req->stylesheet)
2130                 m_s2z_stylesheet =
2131                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2132
2133             if (srw_req->recordPacking &&
2134                 !strcmp(srw_req->recordPacking, "xml"))
2135                 m_s2z_packing = Z_SRW_recordPacking_XML;
2136             else
2137                 m_s2z_packing = Z_SRW_recordPacking_string;
2138
2139             if (num_diagnostic)
2140             {
2141                 send_srw_explain_response(diagnostic, num_diagnostic);
2142                 return;
2143             }
2144
2145             if (!m_client)
2146             {
2147                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2148                                             Z_APDU_initRequest);
2149
2150                 // prevent m_initRequest_apdu memory from being grabbed
2151                 // in Yaz_Proxy::handle_incoming_Z_PDU
2152                 m_initRequest_apdu = m_s2z_init_apdu;
2153                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2154             }
2155             else
2156                 send_srw_explain_response(0, 0);
2157             return;
2158         }
2159         else if (srw_pdu->which == Z_SRW_scan_request)
2160         {
2161             m_s2z_database = odr_strdup(m_s2z_odr_init,
2162                                         srw_pdu->u.scan_request->database);
2163
2164             yaz_add_srw_diagnostic(odr_decode(),
2165                                    &diagnostic, &num_diagnostic,
2166                                    4, "scan");
2167             Z_SRW_PDU *srw_pdu =
2168                 yaz_srw_get(odr_encode(),
2169                             Z_SRW_scan_response);
2170             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2171
2172             srw_res->diagnostics = diagnostic;
2173             srw_res->num_diagnostics = num_diagnostic;
2174             send_srw_response(srw_pdu);
2175             return;
2176         }
2177         else
2178         {
2179             m_s2z_database = 0;
2180
2181             send_to_srw_client_error(4, 0);
2182         }
2183     }
2184     int len = 0;
2185     Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
2186     timeout(0);
2187     send_GDU(p, &len);
2188 }
2189
2190 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2191 {
2192     Z_ReferenceId **refid = get_referenceIdP(apdu);
2193     nmem_reset(m_referenceId_mem);
2194     if (refid && *refid)
2195     {
2196         m_referenceId = (Z_ReferenceId *)
2197             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2198         m_referenceId->len = m_referenceId->size = (*refid)->len;
2199         m_referenceId->buf = (unsigned char *)
2200             nmem_malloc(m_referenceId_mem, (*refid)->len);
2201         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2202     }
2203     else
2204         m_referenceId = 0;
2205
2206     if (!m_client && m_invalid_session)
2207     {
2208         m_apdu_invalid_session = apdu;
2209         m_mem_invalid_session = odr_extract_mem(odr_decode());
2210         apdu = m_initRequest_apdu;
2211     }
2212
2213     // Determine our client.
2214     Z_OtherInformation **oi;
2215     get_otherInfoAPDU(apdu, &oi);
2216     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2217     if (!m_client)
2218     {
2219         delete this;
2220         return;
2221     }
2222     m_client->m_server = this;
2223
2224     if (apdu->which == Z_APDU_initRequest)
2225     {
2226         if (apdu->u.initRequest->implementationId)
2227             yaz_log(YLOG_LOG, "%simplementationId: %s",
2228                     m_session_str, apdu->u.initRequest->implementationId);
2229         if (apdu->u.initRequest->implementationName)
2230             yaz_log(YLOG_LOG, "%simplementationName: %s",
2231                     m_session_str, apdu->u.initRequest->implementationName);
2232         if (apdu->u.initRequest->implementationVersion)
2233             yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2234                     m_session_str, apdu->u.initRequest->implementationVersion);
2235         if (m_initRequest_apdu == 0)
2236         {
2237             if (m_initRequest_mem)
2238                 nmem_destroy(m_initRequest_mem);
2239             m_initRequest_apdu = apdu;
2240             m_initRequest_mem = odr_extract_mem(odr_decode());
2241
2242             m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2243                 preferredMessageSize;
2244             *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2245             m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2246                 maximumRecordSize;
2247             *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2248
2249             // save init options for the response..
2250             m_initRequest_options = apdu->u.initRequest->options;
2251
2252             apdu->u.initRequest->options =
2253                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2254                                            sizeof(Odr_bitmask));
2255             ODR_MASK_ZERO(apdu->u.initRequest->options);
2256             int i;
2257             for (i = 0; i<= 24; i++)
2258                 ODR_MASK_SET(apdu->u.initRequest->options, i);
2259             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2260                            Z_Options_negotiationModel);
2261             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2262                            Z_Options_concurrentOperations);
2263
2264             // make new version
2265             m_initRequest_version = apdu->u.initRequest->protocolVersion;
2266             apdu->u.initRequest->protocolVersion =
2267                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2268                                            sizeof(Odr_bitmask));
2269             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2270
2271             for (i = 0; i<= 8; i++)
2272                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2273         }
2274         if (m_client->m_init_flag)
2275         {
2276             if (handle_init_response_for_invalid_session(apdu))
2277                 return;
2278             if (m_client->m_initResponse)
2279             {
2280                 Z_APDU *apdu2 = m_client->m_initResponse;
2281                 apdu2->u.initResponse->otherInfo = 0;
2282                 if (m_client->m_cookie && *m_client->m_cookie)
2283                     set_otherInformationString(apdu2, VAL_COOKIE, 1,
2284                                                m_client->m_cookie);
2285                 apdu2->u.initResponse->referenceId =
2286                     apdu->u.initRequest->referenceId;
2287                 apdu2->u.initResponse->options = m_client->m_initResponse_options;
2288                 apdu2->u.initResponse->protocolVersion =
2289                     m_client->m_initResponse_version;
2290
2291                 send_to_client(apdu2);
2292                 return;
2293             }
2294         }
2295         m_client->m_init_flag = 1;
2296     }
2297     handle_max_record_retrieve(apdu);
2298
2299     if (apdu)
2300         apdu = handle_syntax_validation(apdu);
2301
2302     if (apdu)
2303         apdu = handle_query_transformation(apdu);
2304
2305     if (apdu)
2306         apdu = handle_query_validation(apdu);
2307
2308     if (apdu)
2309         apdu = result_set_optimize(apdu);
2310     if (!apdu)
2311     {
2312         m_client->timeout(m_target_idletime);  // mark it active even
2313         // though we didn't use it
2314         return;
2315     }
2316
2317     // delete other info construct completely if 0 elements
2318     get_otherInfoAPDU(apdu, &oi);
2319     if (*oi && (*oi)->num_elements == 0)
2320         *oi = 0;
2321
2322     if (apdu->which == Z_APDU_presentRequest &&
2323         m_client->m_resultSetStartPoint == 0)
2324     {
2325         Z_PresentRequest *pr = apdu->u.presentRequest;
2326         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2327         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2328     } else {
2329         m_client->m_resultSetStartPoint = 0;
2330     }
2331     if (m_client->send_to_target(apdu) < 0)
2332     {
2333         delete m_client;
2334         m_client = 0;
2335         delete this;
2336     }
2337     else
2338         m_client->m_waiting = 1;
2339 }
2340
2341 void Yaz_Proxy::connectNotify()
2342 {
2343 }
2344
2345 void Yaz_Proxy::releaseClient()
2346 {
2347     xfree(m_proxyTarget);
2348     m_proxyTarget = 0;
2349     m_invalid_session = 0;
2350     // only keep if keep_alive flag is set...
2351     if (m_client &&
2352         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2353         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2354         m_client->m_waiting == 0)
2355     {
2356         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2357                  m_session_str,
2358                  m_client->get_hostname());
2359         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2360                 m_session_str, m_client->m_pdu_recv,
2361                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2362                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2363         assert (m_client->m_waiting != 2);
2364         // Tell client (if any) that no server connection is there..
2365         m_client->m_server = 0;
2366         m_client = 0;
2367     }
2368     else if (m_client)
2369     {
2370         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
2371                  m_session_str,
2372                  m_client->get_hostname());
2373         assert (m_client->m_waiting != 2);
2374         delete m_client;
2375         m_client = 0;
2376     }
2377     else if (!m_parent)
2378     {
2379         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
2380                  m_session_str);
2381         assert (m_parent);
2382     }
2383     else
2384     {
2385         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
2386                  m_session_str);
2387     }
2388     if (m_parent)
2389         m_parent->pre_init();
2390 }
2391
2392 void Yaz_Proxy::shutdown()
2393 {
2394     releaseClient();
2395     delete this;
2396 }
2397
2398 const char *Yaz_ProxyClient::get_session_str()
2399 {
2400     if (!m_server)
2401         return "0 ";
2402     return m_server->get_session_str();
2403 }
2404
2405 void Yaz_ProxyClient::shutdown()
2406 {
2407     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2408              get_hostname());
2409     delete m_server;
2410     delete this;
2411 }
2412
2413 void Yaz_Proxy::failNotify()
2414 {
2415     inc_request_no();
2416     yaz_log (YLOG_LOG, "%sConnection closed by client",
2417              get_session_str());
2418     shutdown();
2419 }
2420
2421 void Yaz_ProxyClient::failNotify()
2422 {
2423     if (m_server)
2424         m_server->inc_request_no();
2425     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
2426              get_session_str(), get_hostname());
2427     shutdown();
2428 }
2429
2430 void Yaz_ProxyClient::connectNotify()
2431 {
2432     const char *s = get_session_str();
2433     const char *h = get_hostname();
2434     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2435              m_target_idletime);
2436     timeout(m_target_idletime);
2437     if (!m_server)
2438         pre_init_client();
2439 }
2440
2441 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2442                                                   *the_PDU_Observable, int fd)
2443 {
2444     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2445 }
2446
2447 Yaz_ProxyClient::~Yaz_ProxyClient()
2448 {
2449     if (m_prev)
2450         *m_prev = m_next;
2451     if (m_next)
2452         m_next->m_prev = m_prev;
2453     m_waiting = 2;     // for debugging purposes only.
2454     odr_destroy(m_init_odr);
2455     delete m_last_query;
2456     xfree (m_last_resultSetId);
2457     xfree (m_cookie);
2458 }
2459
2460 void Yaz_ProxyClient::pre_init_client()
2461 {
2462     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2463     Z_InitRequest *req = apdu->u.initRequest;
2464
2465     int i;
2466     for (i = 0; i<= 24; i++)
2467         ODR_MASK_SET(req->options, i);
2468     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2469                    Z_Options_negotiationModel);
2470     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2471                    Z_Options_concurrentOperations);
2472     for (i = 0; i<= 10; i++)
2473         ODR_MASK_SET(req->protocolVersion, i);
2474
2475     if (send_to_target(apdu) < 0)
2476     {
2477         delete this;
2478     }
2479     else
2480     {
2481         m_waiting = 1;
2482         m_init_flag = 1;
2483     }
2484 }
2485
2486 void Yaz_Proxy::pre_init()
2487 {
2488     int i;
2489     const char *name = 0;
2490     const char *zurl_in_use[MAX_ZURL_PLEX];
2491     int limit_bw, limit_pdu, limit_req;
2492     int target_idletime, client_idletime;
2493     int max_clients;
2494     int keepalive_limit_bw, keepalive_limit_pdu;
2495     int pre_init;
2496     const char *cql2rpn = 0;
2497     const char *authentication = 0;
2498
2499     Yaz_ProxyConfig *cfg = check_reconfigure();
2500
2501     zurl_in_use[0] = 0;
2502
2503     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2504         set_APDU_yazlog(1);
2505     else
2506         set_APDU_yazlog(0);
2507
2508     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2509                                           &limit_bw, &limit_pdu, &limit_req,
2510                                           &target_idletime, &client_idletime,
2511                                           &max_clients,
2512                                           &keepalive_limit_bw,
2513                                           &keepalive_limit_pdu,
2514                                           &pre_init,
2515                                           &cql2rpn,
2516                                           &authentication) ; i++)
2517     {
2518         if (pre_init)
2519         {
2520             int j;
2521             for (j = 0; zurl_in_use[j]; j++)
2522             {
2523                 Yaz_ProxyClient *c;
2524                 int spare = 0;
2525                 int spare_waiting = 0;
2526                 int in_use = 0;
2527                 int other = 0;
2528                 for (c = m_clientPool; c; c = c->m_next)
2529                 {
2530                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
2531                     {
2532                         if (c->m_cookie == 0)
2533                         {
2534                             if (c->m_server == 0)
2535                                 if (c->m_waiting)
2536                                     spare_waiting++;
2537                                 else
2538                                     spare++;
2539                             else
2540                                 in_use++;
2541                         }
2542                         else
2543                             other++;
2544                     }
2545                 }
2546                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2547                         "sparew=%d preinit=%d",m_session_str,
2548                         name, zurl_in_use[j], in_use, other,
2549                         spare, spare_waiting, pre_init);
2550                 if (spare + spare_waiting < pre_init)
2551                 {
2552                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2553                     c->m_next = m_clientPool;
2554                     if (c->m_next)
2555                         c->m_next->m_prev = &c->m_next;
2556                     m_clientPool = c;
2557                     c->m_prev = &m_clientPool;
2558
2559                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
2560                         c->set_APDU_yazlog(1);
2561                     else
2562                         c->set_APDU_yazlog(0);
2563
2564                     if (c->client(zurl_in_use[j]))
2565                     {
2566                         timeout(60);
2567                         delete c;
2568                         return;
2569                     }
2570                     c->timeout(30);
2571                     c->m_waiting = 1;
2572                     c->m_target_idletime = target_idletime;
2573                     c->m_seqno = m_seqno++;
2574                 }
2575             }
2576         }
2577     }
2578 }
2579
2580 void Yaz_Proxy::timeoutNotify()
2581 {
2582     if (m_parent)
2583     {
2584         if (m_bw_hold_PDU)
2585         {
2586             timeout(m_client_idletime);
2587             Z_GDU *apdu = m_bw_hold_PDU;
2588             m_bw_hold_PDU = 0;
2589
2590             if (apdu->which == Z_GDU_Z3950)
2591                 handle_incoming_Z_PDU(apdu->u.z3950);
2592             else if (apdu->which == Z_GDU_HTTP_Request)
2593                 handle_incoming_HTTP(apdu->u.HTTP_Request);
2594         }
2595         else if (m_stylesheet_nprl)
2596             convert_xsl_delay();
2597         else
2598         {
2599             inc_request_no();
2600
2601             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2602             shutdown();
2603         }
2604     }
2605     else
2606     {
2607         timeout(600);
2608         pre_init();
2609     }
2610 }
2611
2612 void Yaz_Proxy::markInvalid()
2613 {
2614     m_client = 0;
2615     m_invalid_session = 1;
2616 }
2617
2618 void Yaz_ProxyClient::timeoutNotify()
2619 {
2620     if (m_server)
2621         m_server->inc_request_no();
2622
2623     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2624              get_hostname());
2625     m_waiting = 1;
2626     m_root->pre_init();
2627     if (m_server && m_init_flag)
2628     {
2629         // target timed out in a session that was properly initialized
2630         // server object stay alive but we mark it as invalid so it
2631         // gets initialized again
2632         m_server->markInvalid();
2633         m_server = 0;
2634     }
2635     shutdown();
2636 }
2637
2638 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2639                                  Yaz_Proxy *parent) :
2640     Yaz_Z_Assoc (the_PDU_Observable)
2641 {
2642     m_cookie = 0;
2643     m_next = 0;
2644     m_prev = 0;
2645     m_init_flag = 0;
2646     m_last_query = 0;
2647     m_last_resultSetId = 0;
2648     m_last_resultCount = 0;
2649     m_last_ok = 0;
2650     m_sr_transform = 0;
2651     m_waiting = 0;
2652     m_init_odr = odr_createmem (ODR_DECODE);
2653     m_initResponse = 0;
2654     m_initResponse_options = 0;
2655     m_initResponse_version = 0;
2656     m_initResponse_preferredMessageSize = 0;
2657     m_initResponse_maximumRecordSize = 0;
2658     m_resultSetStartPoint = 0;
2659     m_bytes_sent = m_bytes_recv = 0;
2660     m_pdu_recv = 0;
2661     m_server = 0;
2662     m_seqno = 0;
2663     m_target_idletime = 600;
2664     m_root = parent;
2665 }
2666
2667 const char *Yaz_Proxy::option(const char *name, const char *value)
2668 {
2669     if (!strcmp (name, "optimize")) {
2670         if (value) {
2671             xfree (m_optimize);
2672             m_optimize = xstrdup (value);
2673         }
2674         return m_optimize;
2675     }
2676     return 0;
2677 }
2678
2679 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2680 {
2681
2682 }
2683
2684 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2685 {
2686     if (apdu->which == Z_GDU_Z3950)
2687         recv_Z_PDU(apdu->u.z3950, len);
2688     else if (apdu->which == Z_GDU_HTTP_Response)
2689         recv_HTTP_response(apdu->u.HTTP_Response, len);
2690     else
2691         shutdown();
2692 }
2693
2694 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2695 {
2696     if (!m_invalid_session)
2697         return 0;
2698     m_invalid_session = 0;
2699     handle_incoming_Z_PDU(m_apdu_invalid_session);
2700     assert (m_mem_invalid_session);
2701     nmem_destroy(m_mem_invalid_session);
2702     m_mem_invalid_session = 0;
2703     return 1;
2704 }
2705
2706 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2707 {
2708     m_bytes_recv += len;
2709
2710     m_pdu_recv++;
2711     m_waiting = 0;
2712     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2713         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2714                  apdu_name(apdu), get_hostname(), len);
2715     if (apdu->which == Z_APDU_initResponse)
2716     {
2717         if (!m_server)  // if this is a pre init session , check for more
2718             m_root->pre_init();
2719         NMEM nmem = odr_extract_mem (odr_decode());
2720         odr_reset (m_init_odr);
2721         nmem_transfer (m_init_odr->mem, nmem);
2722         m_initResponse = apdu;
2723         m_initResponse_options = apdu->u.initResponse->options;
2724         m_initResponse_version = apdu->u.initResponse->protocolVersion;
2725         m_initResponse_preferredMessageSize =
2726             *apdu->u.initResponse->preferredMessageSize;
2727         m_initResponse_maximumRecordSize =
2728             *apdu->u.initResponse->maximumRecordSize;
2729
2730         Z_InitResponse *ir = apdu->u.initResponse;
2731         char *im0 = ir->implementationName;
2732
2733         char *im1 = (char*)
2734             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2735         *im1 = '\0';
2736         if (im0)
2737         {
2738             strcat(im1, im0);
2739             strcat(im1, " ");
2740         }
2741         strcat(im1, "(YAZ Proxy)");
2742         ir->implementationName = im1;
2743
2744         nmem_destroy (nmem);
2745
2746         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2747             return;
2748     }
2749     if (apdu->which == Z_APDU_searchResponse)
2750     {
2751         Z_SearchResponse *sr = apdu->u.searchResponse;
2752         m_last_resultCount = *sr->resultCount;
2753         int status = *sr->searchStatus;
2754         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2755         {
2756             m_last_ok = 1;
2757
2758             if (sr->records && sr->records->which == Z_Records_DBOSD)
2759             {
2760                 m_cache.add(odr_decode(),
2761                             sr->records->u.databaseOrSurDiagnostics, 1,
2762                             *sr->resultCount);
2763             }
2764         }
2765     }
2766     if (apdu->which == Z_APDU_presentResponse)
2767     {
2768         Z_PresentResponse *pr = apdu->u.presentResponse;
2769         if (m_sr_transform)
2770         {
2771             m_sr_transform = 0;
2772             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2773             Z_SearchResponse *sr = new_apdu->u.searchResponse;
2774             sr->referenceId = pr->referenceId;
2775             *sr->resultCount = m_last_resultCount;
2776             sr->records = pr->records;
2777             sr->nextResultSetPosition = pr->nextResultSetPosition;
2778             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2779             apdu = new_apdu;
2780         }
2781         if (pr->records &&
2782             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2783         {
2784             m_cache.add(odr_decode(),
2785                         pr->records->u.databaseOrSurDiagnostics,
2786                         m_resultSetStartPoint, -1);
2787             m_resultSetStartPoint = 0;
2788         }
2789     }
2790     if (m_cookie)
2791         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2792     if (m_server)
2793     {
2794         m_server->send_to_client(apdu);
2795     }
2796     if (apdu->which == Z_APDU_close)
2797     {
2798         shutdown();
2799     }
2800 }
2801
2802 void Yaz_Proxy::low_socket_close()
2803 {
2804 #if WIN32
2805 #else
2806     int i;
2807     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
2808         if  (m_lo_fd[i] >= 0)
2809             ::close(m_lo_fd[i]);
2810 #endif
2811 }
2812
2813 void Yaz_Proxy::low_socket_open()
2814 {
2815 #if WIN32
2816 #else
2817     int i;
2818     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
2819         m_lo_fd[i] = open("/dev/null", O_RDONLY);
2820 #endif
2821 }
2822
2823 int Yaz_Proxy::server(const char *addr)
2824 {
2825     int r = Yaz_Z_Assoc::server(addr);
2826     if (!r)
2827     {
2828         yaz_log(YLOG_LOG, "%sStarted proxy "
2829 #ifdef VERSION
2830             VERSION
2831 #endif
2832             " on %s", m_session_str, addr);
2833         timeout(1);
2834     }
2835     return r;
2836 }
2837