src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.18 2005-01-18 10:49:45 adam Exp $
   2    Copyright (c) 1998-2005, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #if HAVE_UNISTD_H
  23 #include <unistd.h>
  24 #endif
  25 #if HAVE_SYS_TIME_H
  26 #include <sys/time.h>
  27 #endif
  28 #if HAVE_SYS_TYPES_H
  29 #include <sys/types.h>
  30 #endif
  31 #if HAVE_SYS_STAT_H
  32 #include <sys/stat.h>
  33 #endif
  34
  35 #include <assert.h>
  36 #include <time.h>
  37 #include <fcntl.h>
  38
  39 #include <yaz/srw.h>
  40 #include <yaz/marcdisp.h>
  41 #include <yaz/yaz-iconv.h>
  42 #include <yaz/log.h>
  43 #include <yaz/diagbib1.h>
  44 #include "proxyp.h"
  45 #include <yaz/pquery.h>
  46 #include <yaz/otherinfo.h>
  47
  48 static const char *apdu_name(Z_APDU *apdu)
  49 {
  50     switch (apdu->which)
  51     {
  52     case Z_APDU_initRequest:
  53         return "initRequest";
  54     case Z_APDU_initResponse:
  55         return "initResponse";
  56     case Z_APDU_searchRequest:
  57         return "searchRequest";
  58     case Z_APDU_searchResponse:
  59         return "searchResponse";
  60     case Z_APDU_presentRequest:
  61         return "presentRequest";
  62     case Z_APDU_presentResponse:
  63         return "presentResponse";
  64     case Z_APDU_deleteResultSetRequest:
  65         return "deleteResultSetRequest";
  66     case Z_APDU_deleteResultSetResponse:
  67         return "deleteResultSetResponse";
  68     case Z_APDU_scanRequest:
  69         return "scanRequest";
  70     case Z_APDU_scanResponse:
  71         return "scanResponse";
  72     case Z_APDU_sortRequest:
  73         return "sortRequest";
  74     case Z_APDU_sortResponse:
  75         return "sortResponse";
  76     case Z_APDU_extendedServicesRequest:
  77         return "extendedServicesRequest";
  78     case Z_APDU_extendedServicesResponse:
  79         return "extendedServicesResponse";
  80     case Z_APDU_close:
  81         return "close";
  82     }
  83     return "other";
  84 }
  85
  86 static const char *gdu_name(Z_GDU *gdu)
  87 {
  88     switch(gdu->which)
  89     {
  90     case Z_GDU_Z3950:
  91         return apdu_name(gdu->u.z3950);
  92     case Z_GDU_HTTP_Request:
  93         return "HTTP Request";
  94     case Z_GDU_HTTP_Response:
  95         return "HTTP Response";
  96     }
  97     return "Unknown request/response";
  98 }
  99
 100 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
 101                      Yaz_Proxy *parent) :
 102     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 103 {
 104     m_PDU_Observable = the_PDU_Observable;
 105     m_client = 0;
 106     m_parent = parent;
 107     m_clientPool = 0;
 108     m_seqno = 1;
 109     m_keepalive_limit_bw = 500000;
 110     m_keepalive_limit_pdu = 1000;
 111     m_proxyTarget = 0;
 112     m_default_target = 0;
 113     m_proxy_authentication = 0;
 114     m_max_clients = 150;
 115     m_log_mask = 0;
 116     m_seed = time(0);
 117     m_client_idletime = 600;
 118     m_target_idletime = 600;
 119     m_optimize = xstrdup ("1");
 120     strcpy(m_session_str, "0 ");
 121     m_session_no=0;
 122     m_bytes_sent = 0;
 123     m_bytes_recv = 0;
 124     m_bw_hold_PDU = 0;
 125     m_bw_max = 0;
 126     m_pdu_max = 0;
 127     m_max_record_retrieve = 0;
 128     m_reconfig_flag = 0;
 129     m_config_fname = 0;
 130     m_request_no = 0;
 131     m_invalid_session = 0;
 132     m_referenceId = 0;
 133     m_referenceId_mem = nmem_create();
 134     m_config = 0;
 135     m_marcxml_flag = 0;
 136     m_stylesheet_xsp = 0;
 137     m_stylesheet_nprl = 0;
 138     m_s2z_stylesheet = 0;
 139     m_s2z_database = 0;
 140     m_schema = 0;
 141     m_backend_type = 0;
 142     m_backend_charset = 0;
 143     m_frontend_type = 0;
 144     m_initRequest_apdu = 0;
 145     m_initRequest_mem = 0;
 146     m_initRequest_preferredMessageSize = 0;
 147     m_initRequest_maximumRecordSize = 0;
 148     m_initRequest_options = 0;
 149     m_initRequest_version = 0;
 150     m_apdu_invalid_session = 0;
 151     m_mem_invalid_session = 0;
 152     m_s2z_odr_init = 0;
 153     m_s2z_odr_search = 0;
 154     m_s2z_init_apdu = 0;
 155     m_s2z_search_apdu = 0;
 156     m_s2z_present_apdu = 0;
 157     m_http_keepalive = 0;
 158     m_http_version = 0;
 159     m_soap_ns = 0;
 160     m_s2z_packing = Z_SRW_recordPacking_string;
 161 #if HAVE_GETTIMEOFDAY
 162     m_time_tv = xmalloc(sizeof(struct timeval));
 163     struct timeval *tv = (struct timeval *) m_time_tv;
 164     tv->tv_sec = 0;
 165     tv->tv_usec = 0;
 166 #else
 167     m_time_tv = 0;
 168 #endif
 169     m_usemarcon_ini_stage1 = 0;
 170     m_usemarcon_ini_stage2 = 0;
 171     m_usemarcon = new Yaz_usemarcon();
 172     if (!m_parent)
 173         low_socket_open();
 174 }
 175
 176 Yaz_Proxy::~Yaz_Proxy()
 177 {
 178     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 179             m_bytes_sent, m_bytes_recv);
 180     nmem_destroy(m_initRequest_mem);
 181     nmem_destroy(m_mem_invalid_session);
 182     nmem_destroy(m_referenceId_mem);
 183
 184     xfree(m_proxyTarget);
 185     xfree(m_default_target);
 186     xfree(m_proxy_authentication);
 187     xfree(m_optimize);
 188
 189 #if HAVE_XSLT
 190     if (m_stylesheet_xsp)
 191         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 192 #endif
 193     xfree (m_time_tv);
 194
 195     xfree (m_schema);
 196     xfree (m_backend_type);
 197     xfree (m_backend_charset);
 198     xfree (m_usemarcon_ini_stage1);
 199     xfree (m_usemarcon_ini_stage2);
 200     delete m_usemarcon;
 201     if (m_s2z_odr_init)
 202         odr_destroy(m_s2z_odr_init);
 203     if (m_s2z_odr_search)
 204         odr_destroy(m_s2z_odr_search);
 205     if (!m_parent)
 206         low_socket_close();
 207     delete m_config;
 208 }
 209
 210 int Yaz_Proxy::set_config(const char *config)
 211 {
 212     delete m_config;
 213     m_config = new Yaz_ProxyConfig();
 214     xfree(m_config_fname);
 215     m_config_fname = xstrdup(config);
 216     int r = m_config->read_xml(config);
 217     if (!r)
 218         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 219     return r;
 220 }
 221
 222 void Yaz_Proxy::set_default_target(const char *target)
 223 {
 224     xfree (m_default_target);
 225     m_default_target = 0;
 226     if (target)
 227         m_default_target = (char *) xstrdup (target);
 228 }
 229
 230 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 231 {
 232     xfree (m_proxy_authentication);
 233     m_proxy_authentication = 0;
 234     if (auth)
 235         m_proxy_authentication = (char *) xstrdup (auth);
 236 }
 237
 238 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 239 {
 240     if (m_parent)
 241         return m_parent->check_reconfigure();
 242
 243     Yaz_ProxyConfig *cfg = m_config;
 244     if (m_reconfig_flag)
 245     {
 246         yaz_log(YLOG_LOG, "reconfigure");
 247         yaz_log_reopen();
 248         if (m_config_fname && cfg)
 249         {
 250             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 251             int r = cfg->read_xml(m_config_fname);
 252             if (r)
 253                 yaz_log(YLOG_WARN, "reconfigure failed");
 254             else
 255             {
 256                 m_log_mask = 0;
 257                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 258             }
 259         }
 260         else
 261             yaz_log(YLOG_LOG, "reconfigure");
 262         m_reconfig_flag = 0;
 263     }
 264     return cfg;
 265 }
 266
 267 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 268                                             *the_PDU_Observable, int fd)
 269 {
 270     check_reconfigure();
 271     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 272     new_proxy->m_config = 0;
 273     new_proxy->m_config_fname = 0;
 274     new_proxy->timeout(m_client_idletime);
 275     new_proxy->m_target_idletime = m_target_idletime;
 276     new_proxy->set_default_target(m_default_target);
 277     new_proxy->m_max_clients = m_max_clients;
 278     new_proxy->m_log_mask = m_log_mask;
 279     new_proxy->set_APDU_log(get_APDU_log());
 280     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 281         new_proxy->set_APDU_yazlog(1);
 282     else
 283         new_proxy->set_APDU_yazlog(0);
 284     new_proxy->set_proxy_authentication(m_proxy_authentication);
 285     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 286     m_session_no++;
 287     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 288              the_PDU_Observable->getpeername());
 289     return new_proxy;
 290 }
 291
 292 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 293 {
 294     int oid[OID_SIZE];
 295     Z_OtherInformationUnit *oi;
 296     struct oident ent;
 297     ent.proto = PROTO_Z3950;
 298     ent.oclass = CLASS_USERINFO;
 299     ent.value = (oid_value) VAL_COOKIE;
 300     assert (oid_ent_to_oid (&ent, oid));
 301
 302     if (oid_ent_to_oid (&ent, oid) &&
 303         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 304         oi->which == Z_OtherInfo_characterInfo)
 305         return oi->information.characterInfo;
 306     return 0;
 307 }
 308
 309 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 310 {
 311     int oid[OID_SIZE];
 312     Z_OtherInformationUnit *oi;
 313     struct oident ent;
 314     ent.proto = PROTO_Z3950;
 315     ent.oclass = CLASS_USERINFO;
 316     ent.value = (oid_value) VAL_PROXY;
 317     if (oid_ent_to_oid (&ent, oid) &&
 318         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 319         oi->which == Z_OtherInfo_characterInfo)
 320         return oi->information.characterInfo;
 321     return 0;
 322 }
 323
 324 const char *Yaz_Proxy::load_balance(const char **url)
 325 {
 326     int zurl_in_use[MAX_ZURL_PLEX];
 327     int zurl_in_spare[MAX_ZURL_PLEX];
 328     Yaz_ProxyClient *c;
 329     int i;
 330
 331     for (i = 0; i<MAX_ZURL_PLEX; i++)
 332     {
 333         zurl_in_use[i] = 0;
 334         zurl_in_spare[i] = 0;
 335     }
 336     for (c = m_parent->m_clientPool; c; c = c->m_next)
 337     {
 338         for (i = 0; url[i]; i++)
 339             if (!strcmp(url[i], c->get_hostname()))
 340             {
 341                 zurl_in_use[i]++;
 342                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 343                     zurl_in_spare[i]++;
 344             }
 345     }
 346     int min_use = 100000;
 347     int spare_for_min = 0;
 348     int max_spare = 0;
 349     const char *ret_min = 0;
 350     const char *ret_spare = 0;
 351     for (i = 0; url[i]; i++)
 352     {
 353         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 354                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 355         if (min_use > zurl_in_use[i])
 356         {
 357             ret_min = url[i];
 358             min_use = zurl_in_use[i];
 359             spare_for_min = zurl_in_spare[i];
 360         }
 361         if (max_spare < zurl_in_spare[i])
 362         {
 363             ret_spare = url[i];
 364             max_spare = zurl_in_spare[i];
 365         }
 366     }
 367     // use the one with minimum connections if spare is > 3
 368     if (spare_for_min > 3)
 369         return ret_min;
 370     // use one with most spares (if any)
 371     if (max_spare > 0)
 372         return ret_spare;
 373     return ret_min;
 374 }
 375
 376 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 377                                        const char *proxy_host)
 378 {
 379     assert (m_parent);
 380     Yaz_Proxy *parent = m_parent;
 381     Yaz_ProxyClient *c = m_client;
 382
 383     if (!m_proxyTarget)
 384     {
 385         const char *url[MAX_ZURL_PLEX];
 386         Yaz_ProxyConfig *cfg = check_reconfigure();
 387         if (proxy_host)
 388         {
 389 #if 0
 390 /* only to be enabled for debugging... */
 391             if (!strcmp(proxy_host, "stop"))
 392                 exit(0);
 393 #endif
 394             xfree(m_default_target);
 395             m_default_target = xstrdup(proxy_host);
 396         }
 397         proxy_host = m_default_target;
 398         int client_idletime = -1;
 399         const char *cql2rpn_fname = 0;
 400         const char *authentication = 0;
 401         url[0] = m_default_target;
 402         url[1] = 0;
 403         if (cfg)
 404         {
 405             int pre_init = 0;
 406             cfg->get_target_info(proxy_host, url, &m_bw_max,
 407                                  &m_pdu_max, &m_max_record_retrieve,
 408                                  &m_target_idletime, &client_idletime,
 409                                  &parent->m_max_clients,
 410                                  &m_keepalive_limit_bw,
 411                                  &m_keepalive_limit_pdu,
 412                                  &pre_init,
 413                                  &cql2rpn_fname,
 414                                  &authentication);
 415         }
 416         if (client_idletime != -1)
 417         {
 418             m_client_idletime = client_idletime;
 419             timeout(m_client_idletime);
 420         }
 421         if (cql2rpn_fname)
 422             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 423         if (authentication)
 424             set_proxy_authentication(authentication);
 425         if (!url[0])
 426         {
 427             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 428             return 0;
 429         }
 430         // we don't handle multiplexing for cookie session, so we just
 431         // pick the first one in this case (anonymous users will be able
 432         // to use any backend)
 433         if (cookie && *cookie)
 434             m_proxyTarget = (char*) xstrdup(url[0]);
 435         else
 436             m_proxyTarget = (char*) xstrdup(load_balance(url));
 437     }
 438     if (cookie && *cookie)
 439     {   // search in sessions with a cookie
 440         for (c = parent->m_clientPool; c; c = c->m_next)
 441         {
 442             assert (c->m_prev);
 443             assert (*c->m_prev == c);
 444             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 445                 !strcmp(m_proxyTarget, c->get_hostname()))
 446             {
 447                 // Found it in cache
 448                 // The following handles "cancel"
 449                 // If connection is busy (waiting for PDU) and
 450                 // we have an initRequest we can safely do re-open
 451                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 452                 {
 453                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 454                              c->get_hostname());
 455                     c->close();
 456                     c->m_init_flag = 0;
 457
 458                     c->m_last_ok = 0;
 459                     c->m_cache.clear();
 460                     c->m_last_resultCount = 0;
 461                     c->m_sr_transform = 0;
 462                     c->m_waiting = 0;
 463                     c->m_resultSetStartPoint = 0;
 464                     c->m_target_idletime = m_target_idletime;
 465                     if (c->client(m_proxyTarget))
 466                     {
 467                         delete c;
 468                         return 0;
 469                     }
 470                     c->timeout(30);
 471                 }
 472                 c->m_seqno = parent->m_seqno;
 473                 if (c->m_server && c->m_server != this)
 474                     c->m_server->m_client = 0;
 475                 c->m_server = this;
 476                 (parent->m_seqno)++;
 477                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 478                 return c;
 479             }
 480         }
 481     }
 482     else if (!c && apdu->which == Z_APDU_initRequest
 483              && apdu->u.initRequest->idAuthentication == 0)
 484     {
 485         // anonymous sessions without cookie.
 486         // if authentication is set it is NOT anonymous se we can't share them.
 487         for (c = parent->m_clientPool; c; c = c->m_next)
 488         {
 489             assert(c->m_prev);
 490             assert(*c->m_prev == c);
 491             if (c->m_server == 0 && c->m_cookie == 0 &&
 492                 c->m_waiting == 0 &&
 493                 !strcmp(m_proxyTarget, c->get_hostname()))
 494             {
 495                 // found it in cache
 496                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 497                          m_session_str, parent->m_seqno, c->get_hostname());
 498
 499                 c->m_seqno = parent->m_seqno;
 500                 assert(c->m_server == 0);
 501                 c->m_server = this;
 502
 503                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 504                     c->set_APDU_yazlog(1);
 505                 else
 506                     c->set_APDU_yazlog(0);
 507
 508                 (parent->m_seqno)++;
 509
 510                 parent->pre_init();
 511
 512                 return c;
 513             }
 514         }
 515     }
 516     if (!m_client)
 517     {
 518         if (apdu->which != Z_APDU_initRequest)
 519         {
 520             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 521             return 0;
 522         }
 523         Z_InitRequest *initRequest = apdu->u.initRequest;
 524
 525         if (!initRequest->idAuthentication)
 526         {
 527             if (m_proxy_authentication)
 528             {
 529                 initRequest->idAuthentication =
 530                     (Z_IdAuthentication *)
 531                     odr_malloc (odr_encode(),
 532                                 sizeof(*initRequest->idAuthentication));
 533                 initRequest->idAuthentication->which =
 534                     Z_IdAuthentication_open;
 535                 initRequest->idAuthentication->u.open =
 536                     odr_strdup (odr_encode(), m_proxy_authentication);
 537             }
 538         }
 539         else
 540         {
 541             // the client use authentication. We set the keepalive PDU
 542             // to 0 so we don't cache it in releaseClient
 543             m_keepalive_limit_pdu = 0;
 544         }
 545         // go through list of clients - and find the lowest/oldest one.
 546         Yaz_ProxyClient *c_min = 0;
 547         int min_seq = -1;
 548         int no_of_clients = 0;
 549         if (parent->m_clientPool)
 550             yaz_log (YLOG_DEBUG, "Existing sessions");
 551         for (c = parent->m_clientPool; c; c = c->m_next)
 552         {
 553             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 554                                c->m_waiting, c->get_hostname(),
 555                                c->m_cookie ? c->m_cookie : "");
 556             no_of_clients++;
 557             if (min_seq < 0 || c->m_seqno < min_seq)
 558             {
 559                 min_seq = c->m_seqno;
 560                 c_min = c;
 561             }
 562         }
 563         if (no_of_clients >= parent->m_max_clients)
 564         {
 565             c = c_min;
 566             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 567             {
 568                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 569                          m_session_str, parent->m_max_clients, c->m_seqno);
 570                 if (c->m_server && c->m_server != this)
 571                     delete c->m_server;
 572                 c->m_server = 0;
 573             }
 574             else
 575             {
 576                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 577                          m_session_str, parent->m_max_clients,
 578                          c->m_seqno, parent->m_seqno, c->get_hostname());
 579                 xfree (c->m_cookie);
 580                 c->m_cookie = 0;
 581                 if (cookie)
 582                     c->m_cookie = xstrdup(cookie);
 583                 c->m_seqno = parent->m_seqno;
 584                 if (c->m_server && c->m_server != this)
 585                 {
 586                     c->m_server->m_client = 0;
 587                     delete c->m_server;
 588                 }
 589                 (parent->m_seqno)++;
 590                 c->m_target_idletime = m_target_idletime;
 591                 c->timeout(m_target_idletime);
 592
 593                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 594                     c->set_APDU_yazlog(1);
 595                 else
 596                     c->set_APDU_yazlog(0);
 597
 598                 return c;
 599             }
 600         }
 601         else
 602         {
 603             yaz_log (YLOG_LOG, "%sNEW %d %s",
 604                      m_session_str, parent->m_seqno, m_proxyTarget);
 605             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 606             c->m_next = parent->m_clientPool;
 607             if (c->m_next)
 608                 c->m_next->m_prev = &c->m_next;
 609             parent->m_clientPool = c;
 610             c->m_prev = &parent->m_clientPool;
 611         }
 612
 613         xfree (c->m_cookie);
 614         c->m_cookie = 0;
 615         if (cookie)
 616             c->m_cookie = xstrdup(cookie);
 617
 618         c->m_seqno = parent->m_seqno;
 619         c->m_init_flag = 0;
 620         c->m_last_resultCount = 0;
 621         c->m_last_ok = 0;
 622         c->m_cache.clear();
 623         c->m_sr_transform = 0;
 624         c->m_waiting = 0;
 625         c->m_resultSetStartPoint = 0;
 626         (parent->m_seqno)++;
 627         if (c->client(m_proxyTarget))
 628         {
 629             delete c;
 630             return 0;
 631         }
 632         c->m_target_idletime = m_target_idletime;
 633         c->timeout(30);
 634
 635         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 636             c->set_APDU_yazlog(1);
 637         else
 638             c->set_APDU_yazlog(0);
 639     }
 640     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 641     return c;
 642 }
 643
 644 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 645 {
 646     int i;
 647     for (i = 0; i<num; i++)
 648     {
 649         oident *ent;
 650         Z_DefaultDiagFormat *r;
 651         Z_DiagRec *p = pp[i];
 652         if (p->which != Z_DiagRec_defaultFormat)
 653         {
 654             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 655             return;
 656         }
 657         else
 658             r = p->u.defaultFormat;
 659         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 660             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 661             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 662         switch (r->which)
 663         {
 664         case Z_DefaultDiagFormat_v2Addinfo:
 665             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 666                     m_session_str,
 667                     *r->condition, diagbib1_str(*r->condition),
 668                     r->u.v2Addinfo);
 669             break;
 670         case Z_DefaultDiagFormat_v3Addinfo:
 671             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 672                     m_session_str,
 673                     *r->condition, diagbib1_str(*r->condition),
 674                     r->u.v3Addinfo);
 675             break;
 676         }
 677     }
 678 }
 679
 680 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 681 {
 682     if (!m_stylesheet_xsp || p->num_records <= 0)
 683         return 0;  /* no XSLT to be done ... */
 684
 685     m_stylesheet_offset = 0;
 686     m_stylesheet_nprl = p;
 687     m_stylesheet_apdu = apdu;
 688     timeout(0);
 689     return 1;
 690 }
 691
 692 void Yaz_Proxy::convert_xsl_delay()
 693 {
 694 #if HAVE_XSLT
 695     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 696     if (npr->which == Z_NamePlusRecord_databaseRecord)
 697     {
 698         Z_External *r = npr->u.databaseRecord;
 699         if (r->which == Z_External_octet)
 700         {
 701 #if 0
 702             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 703 #endif
 704             xmlDocPtr res, doc = xmlParseMemory(
 705                 (char*) r->u.octet_aligned->buf,
 706                 r->u.octet_aligned->len);
 707
 708
 709             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 710                     m_session_str, m_stylesheet_offset);
 711             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 712                                       doc, 0);
 713
 714             if (res)
 715             {
 716                 xmlChar *out_buf;
 717                 int out_len;
 718                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 719
 720                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 721                     u.databaseRecord =
 722                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 723                                  (char*) out_buf, out_len);
 724                 xmlFree(out_buf);
 725                 xmlFreeDoc(res);
 726             }
 727
 728             xmlFreeDoc(doc);
 729         }
 730     }
 731 #endif
 732     m_stylesheet_offset++;
 733     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 734     {
 735         m_stylesheet_nprl = 0;
 736 #if HAVE_XSLT
 737         if (m_stylesheet_xsp)
 738             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 739 #endif
 740         m_stylesheet_xsp = 0;
 741         timeout(m_client_idletime);
 742         send_PDU_convert(m_stylesheet_apdu);
 743     }
 744     else
 745         timeout(0);
 746 }
 747
 748 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 749 {
 750     if (m_frontend_type != VAL_NONE)
 751     {
 752         int i;
 753         for (i = 0; i < p->num_records; i++)
 754         {
 755             Z_NamePlusRecord *npr = p->records[i];
 756             if (npr->which == Z_NamePlusRecord_databaseRecord)
 757             {
 758                 Z_External *r = npr->u.databaseRecord;
 759                 if (r->which == Z_External_octet)
 760                 {
 761 #if HAVE_USEMARCON
 762                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 763                     {
 764                         if (!m_usemarcon->m_stage1)
 765                         {
 766                             m_usemarcon->m_stage1 = new CDetails();
 767                         }
 768                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 769                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 770                         int res = m_usemarcon->m_stage1->Start();
 771                         if (res == 0)
 772                         {
 773                             char *converted;
 774                             int convlen;
 775                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 776                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 777                             {
 778                                 if (!m_usemarcon->m_stage2)
 779                                 {
 780                                     m_usemarcon->m_stage2 = new CDetails();
 781                                 }
 782                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 783                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 784                                 res = m_usemarcon->m_stage2->Start();
 785                                 if (res == 0)
 786                                 {
 787                                     free(converted);
 788                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 789                                 }
 790                                 else
 791                                 {
 792                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 793                                 }
 794                             }
 795                             npr->u.databaseRecord =
 796                                 z_ext_record(odr_encode(),
 797                                              m_frontend_type,
 798                                              converted,
 799                                              strlen(converted));
 800                             free(converted);
 801                         }
 802                         else
 803                         {
 804                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 805                         }
 806                         continue;
 807                     }
 808 #endif
 809                     npr->u.databaseRecord =
 810                         z_ext_record(odr_encode(),
 811                                      m_frontend_type,
 812                                      (char*) r->u.octet_aligned->buf,
 813                                      r->u.octet_aligned->len);
 814                 }
 815             }
 816         }
 817     }
 818 }
 819
 820 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
 821                                    const char *backend_charset)
 822 {
 823     int i;
 824
 825     if (!backend_charset)
 826         backend_charset = "MARC-8";
 827     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
 828     yaz_marc_t mt = yaz_marc_create();
 829     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 830     yaz_marc_iconv(mt, cd);
 831     for (i = 0; i < p->num_records; i++)
 832     {
 833         Z_NamePlusRecord *npr = p->records[i];
 834         if (npr->which == Z_NamePlusRecord_databaseRecord)
 835         {
 836             Z_External *r = npr->u.databaseRecord;
 837             if (r->which == Z_External_octet)
 838             {
 839                 int rlen;
 840                 char *result;
 841                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 842                                         r->u.octet_aligned->len,
 843                                         &result, &rlen))
 844                 {
 845                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 846                                                          VAL_TEXT_XML,
 847                                                          result, rlen);
 848                 }
 849             }
 850         }
 851     }
 852     if (cd)
 853         yaz_iconv_close(cd);
 854     yaz_marc_destroy(mt);
 855 }
 856
 857 void Yaz_Proxy::logtime()
 858 {
 859 #if HAVE_GETTIMEOFDAY
 860     struct timeval *tv = (struct timeval*) m_time_tv;
 861     if (tv->tv_sec)
 862     {
 863         struct timeval tv1;
 864         gettimeofday(&tv1, 0);
 865         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
 866             (tv1.tv_usec - tv->tv_usec);
 867         if (diff >= 0)
 868             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 869                     diff/1000000, (diff/1000)%1000);
 870     }
 871     tv->tv_sec = 0;
 872     tv->tv_usec = 0;
 873 #endif
 874 }
 875
 876 int Yaz_Proxy::send_http_response(int code)
 877 {
 878     ODR o = odr_encode();
 879     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 880     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 881     if (m_http_version)
 882         hres->version = odr_strdup(o, m_http_version);
 883     if (m_http_keepalive)
 884         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 885     else
 886         timeout(0);
 887
 888     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 889     {
 890         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
 891                  gdu_name(gdu));
 892     }
 893     int len;
 894     int r = send_GDU(gdu, &len);
 895     m_bytes_sent += len;
 896     m_bw_stat.add_bytes(len);
 897     logtime();
 898     return r;
 899 }
 900
 901 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 902 {
 903     ODR o = odr_encode();
 904     const char *ctype = "text/xml";
 905     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 906     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 907     if (m_http_version)
 908         hres->version = odr_strdup(o, m_http_version);
 909     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 910     if (m_http_keepalive)
 911         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 912     else
 913         timeout(0);
 914
 915     static Z_SOAP_Handler soap_handlers[2] = {
 916 #if HAVE_XSLT
 917         {"http://www.loc.gov/zing/srw/", 0,
 918          (Z_SOAP_fun) yaz_srw_codec},
 919 #endif
 920         {0, 0, 0}
 921     };
 922
 923     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 924     soap_package->which = Z_SOAP_generic;
 925     soap_package->u.generic =
 926         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 927     soap_package->u.generic->no = 0;
 928     soap_package->u.generic->ns = soap_handlers[0].ns;
 929     soap_package->u.generic->p = (void *) srw_pdu;
 930     soap_package->ns = m_soap_ns;
 931     z_soap_codec_enc_xsl(o, &soap_package,
 932                          &hres->content_buf, &hres->content_len,
 933                          soap_handlers, 0, m_s2z_stylesheet);
 934     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 935     {
 936         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
 937                  gdu_name(gdu));
 938     }
 939     int len;
 940     int r = send_GDU(gdu, &len);
 941     m_bytes_sent += len;
 942     m_bw_stat.add_bytes(len);
 943     logtime();
 944     return r;
 945 }
 946
 947 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
 948 {
 949     ODR o = odr_encode();
 950     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 951     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 952
 953     srw_res->num_diagnostics = 1;
 954     srw_res->diagnostics = (Z_SRW_diagnostic *)
 955         odr_malloc(o, sizeof(*srw_res->diagnostics));
 956     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
 957     return send_srw_response(srw_pdu);
 958 }
 959
 960 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 961                              Z_DefaultDiagFormat *ddf)
 962 {
 963     int bib1_code = *ddf->condition;
 964     if (bib1_code == 109)
 965         return 404;
 966     srw_res->num_diagnostics = 1;
 967     srw_res->diagnostics = (Z_SRW_diagnostic *)
 968         odr_malloc(o, sizeof(*srw_res->diagnostics));
 969     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
 970                           yaz_diag_bib1_to_srw(*ddf->condition),
 971                           ddf->u.v2Addinfo);
 972     return 0;
 973 }
 974
 975 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
 976 {
 977     ODR o = odr_encode();
 978     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 979     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 980
 981     srw_res->numberOfRecords = odr_intdup (o, hits);
 982     if (records && records->which == Z_Records_DBOSD)
 983     {
 984         srw_res->num_records =
 985             records->u.databaseOrSurDiagnostics->num_records;
 986         int i;
 987         srw_res->records = (Z_SRW_record *)
 988             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
 989         for (i = 0; i < srw_res->num_records; i++)
 990         {
 991             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
 992             if (npr->which != Z_NamePlusRecord_databaseRecord)
 993             {
 994                 srw_res->records[i].recordSchema = "diagnostic";
 995                 srw_res->records[i].recordPacking = m_s2z_packing;
 996                 srw_res->records[i].recordData_buf = "67";
 997                 srw_res->records[i].recordData_len = 2;
 998                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 999                 continue;
1000             }
1001             Z_External *r = npr->u.databaseRecord;
1002             oident *ent = oid_getentbyoid(r->direct_reference);
1003             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1004             {
1005                 srw_res->records[i].recordSchema = m_schema;
1006                 srw_res->records[i].recordPacking = m_s2z_packing;
1007                 srw_res->records[i].recordData_buf = (char*)
1008                     r->u.octet_aligned->buf;
1009                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1010                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1011             }
1012             else
1013             {
1014                 srw_res->records[i].recordSchema = "diagnostic";
1015                 srw_res->records[i].recordPacking = m_s2z_packing;
1016                 srw_res->records[i].recordData_buf = "67";
1017                 srw_res->records[i].recordData_len = 2;
1018                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1019             }
1020         }
1021     }
1022     if (records && records->which == Z_Records_NSD)
1023     {
1024         int http_code;
1025         http_code = z_to_srw_diag(odr_encode(), srw_res,
1026                                    records->u.nonSurrogateDiagnostic);
1027         if (http_code)
1028             return send_http_response(http_code);
1029     }
1030     return send_srw_response(srw_pdu);
1031
1032 }
1033
1034 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1035                                         int num_diagnostics)
1036 {
1037     Yaz_ProxyConfig *cfg = check_reconfigure();
1038     if (cfg)
1039     {
1040         int len;
1041         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1042                                        m_s2z_database, &len);
1043         if (b)
1044         {
1045             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1046             Z_SRW_explainResponse *er = res->u.explain_response;
1047
1048             er->record.recordData_buf = b;
1049             er->record.recordData_len = len;
1050             er->record.recordPacking = m_s2z_packing;
1051             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1052
1053             er->diagnostics = diagnostics;
1054             er->num_diagnostics = num_diagnostics;
1055             return send_srw_response(res);
1056         }
1057     }
1058     return send_http_response(404);
1059 }
1060
1061 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1062 {
1063     if (m_http_version)
1064     {
1065         if (apdu->which == Z_APDU_initResponse)
1066         {
1067             Z_InitResponse *res = apdu->u.initResponse;
1068             if (*res->result == 0)
1069             {
1070                 send_to_srw_client_error(3, 0);
1071             }
1072             else if (!m_s2z_search_apdu)
1073             {
1074                 send_srw_explain_response(0, 0);
1075             }
1076             else
1077             {
1078                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1079             }
1080         }
1081         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1082         {
1083             m_s2z_search_apdu = 0;
1084             Z_SearchResponse *res = apdu->u.searchResponse;
1085             m_s2z_hit_count = *res->resultCount;
1086             if (res->records && res->records->which == Z_Records_NSD)
1087             {
1088                 send_to_srw_client_ok(0, res->records, 1);
1089             }
1090             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1091             {
1092                 // adjust
1093                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1094
1095                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1096                 {
1097                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1098                         > m_s2z_hit_count)
1099                         *pr->numberOfRecordsRequested =
1100                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1101                 }
1102                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1103             }
1104             else
1105             {
1106                 m_s2z_present_apdu = 0;
1107                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1108             }
1109         }
1110         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1111         {
1112             int start =
1113                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1114
1115             m_s2z_present_apdu = 0;
1116             Z_PresentResponse *res = apdu->u.presentResponse;
1117             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1118         }
1119     }
1120     else
1121     {
1122         int len = 0;
1123         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1124             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1125                      apdu_name(apdu));
1126         int r = send_Z_PDU(apdu, &len);
1127         m_bytes_sent += len;
1128         m_bw_stat.add_bytes(len);
1129         logtime();
1130         return r;
1131     }
1132     return 0;
1133 }
1134
1135 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1136 {
1137     int kill_session = 0;
1138     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1139
1140     if (new_id)
1141         *new_id = m_referenceId;
1142
1143     if (apdu->which == Z_APDU_searchResponse)
1144     {
1145         Z_SearchResponse *sr = apdu->u.searchResponse;
1146         Z_Records *p = sr->records;
1147         if (p && p->which == Z_Records_NSD)
1148         {
1149             Z_DiagRec dr, *dr_p = &dr;
1150             dr.which = Z_DiagRec_defaultFormat;
1151             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1152
1153             *sr->searchStatus = 0;
1154             display_diagrecs(&dr_p, 1);
1155         }
1156         else
1157         {
1158             if (p && p->which == Z_Records_DBOSD)
1159             {
1160                 if (m_backend_type
1161 #if HAVE_USEMARCON
1162                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1163 #endif
1164                     )
1165                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1166                 if (m_marcxml_flag)
1167                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1168                                        m_backend_charset);
1169                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1170                     return 0;
1171
1172             }
1173             if (sr->resultCount)
1174             {
1175                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1176                         *sr->resultCount);
1177                 if (*sr->resultCount < 0)
1178                 {
1179                     m_invalid_session = 1;
1180                     kill_session = 1;
1181
1182                     *sr->searchStatus = 0;
1183                     sr->records =
1184                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1185                     *sr->resultCount = 0;
1186                 }
1187             }
1188         }
1189     }
1190     else if (apdu->which == Z_APDU_presentResponse)
1191     {
1192         Z_PresentResponse *sr = apdu->u.presentResponse;
1193         Z_Records *p = sr->records;
1194         if (p && p->which == Z_Records_NSD)
1195         {
1196             Z_DiagRec dr, *dr_p = &dr;
1197             dr.which = Z_DiagRec_defaultFormat;
1198             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1199             if (*sr->presentStatus == Z_PresentStatus_success)
1200                 *sr->presentStatus = Z_PresentStatus_failure;
1201             display_diagrecs(&dr_p, 1);
1202         }
1203         if (p && p->which == Z_Records_DBOSD)
1204         {
1205             if (m_backend_type
1206 #if HAVE_USEMARCON
1207                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1208 #endif
1209                 )
1210                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1211             if (m_marcxml_flag)
1212                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1213                                    m_backend_charset);
1214             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1215                 return 0;
1216         }
1217     }
1218     else if (apdu->which == Z_APDU_initResponse)
1219     {
1220         if (m_initRequest_options)
1221         {
1222             Z_Options *nopt =
1223                 (Odr_bitmask *)odr_malloc(odr_encode(),
1224                                           sizeof(Odr_bitmask));
1225             ODR_MASK_ZERO(nopt);
1226
1227             int i;
1228             for (i = 0; i<24; i++)
1229                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1230                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1231                     ODR_MASK_SET(nopt, i);
1232             apdu->u.initResponse->options = nopt;
1233         }
1234         if (m_initRequest_version)
1235         {
1236             Z_ProtocolVersion *nopt =
1237                 (Odr_bitmask *)odr_malloc(odr_encode(),
1238                                           sizeof(Odr_bitmask));
1239             ODR_MASK_ZERO(nopt);
1240
1241             int i;
1242             for (i = 0; i<8; i++)
1243                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1244                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1245                     ODR_MASK_SET(nopt, i);
1246             apdu->u.initResponse->protocolVersion = nopt;
1247         }
1248         apdu->u.initResponse->preferredMessageSize =
1249             odr_intdup(odr_encode(),
1250                        m_client->m_initResponse_preferredMessageSize >
1251                        m_initRequest_preferredMessageSize ?
1252                        m_initRequest_preferredMessageSize :
1253                        m_client->m_initResponse_preferredMessageSize);
1254         apdu->u.initResponse->maximumRecordSize =
1255             odr_intdup(odr_encode(),
1256                        m_client->m_initResponse_maximumRecordSize >
1257                        m_initRequest_maximumRecordSize ?
1258                        m_initRequest_maximumRecordSize :
1259                        m_client->m_initResponse_maximumRecordSize);
1260     }
1261     int r = send_PDU_convert(apdu);
1262     if (r)
1263         return r;
1264     if (kill_session)
1265     {
1266         delete m_client;
1267         m_client = 0;
1268         m_parent->pre_init();
1269     }
1270     return r;
1271 }
1272
1273 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1274 {
1275     int len = 0;
1276     const char *apdu_name_tmp = apdu_name(apdu);
1277     int r = send_Z_PDU(apdu, &len);
1278     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1279         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1280                  get_session_str(),
1281                  apdu_name_tmp, get_hostname(), len);
1282     m_bytes_sent += len;
1283     return r;
1284 }
1285
1286 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1287 {
1288     if (apdu->which == Z_APDU_presentRequest)
1289     {
1290         Z_PresentRequest *pr = apdu->u.presentRequest;
1291         int toget = *pr->numberOfRecordsRequested;
1292         int start = *pr->resultSetStartPoint;
1293
1294         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1295                 pr->resultSetId, start, toget);
1296
1297         if (*m_parent->m_optimize == '0')
1298             return apdu;
1299
1300         if (!m_client->m_last_resultSetId)
1301         {
1302             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1303             new_apdu->u.presentResponse->records =
1304                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1305                                                pr->resultSetId);
1306             send_to_client(new_apdu);
1307             return 0;
1308         }
1309         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1310         {
1311             if (start+toget-1 > m_client->m_last_resultCount)
1312             {
1313                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1314                 new_apdu->u.presentResponse->records =
1315                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1316                 send_to_client(new_apdu);
1317                 return 0;
1318             }
1319             Z_NamePlusRecordList *npr;
1320 #if 0
1321             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1322                     m_session_str, start, toget, yaz_z3950oid_to_str(
1323                         pr->preferredRecordSyntax, &oclass));
1324 #endif
1325             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1326                                           pr->preferredRecordSyntax,
1327                                           pr->recordComposition))
1328             {
1329                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1330                          m_session_str);
1331                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1332                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1333
1334                 new_apdu->u.presentResponse->numberOfRecordsReturned
1335                     = odr_intdup(odr_encode(), toget);
1336
1337                 new_apdu->u.presentResponse->records = (Z_Records*)
1338                     odr_malloc(odr_encode(), sizeof(Z_Records));
1339                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1340                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1341                 new_apdu->u.presentResponse->nextResultSetPosition =
1342                     odr_intdup(odr_encode(), start+toget);
1343
1344                 send_to_client(new_apdu);
1345                 return 0;
1346             }
1347         }
1348     }
1349
1350     if (apdu->which != Z_APDU_searchRequest)
1351         return apdu;
1352     Z_SearchRequest *sr = apdu->u.searchRequest;
1353     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1354     Yaz_Z_Databases this_databases;
1355
1356     this_databases.set(sr->num_databaseNames, (const char **)
1357                        sr->databaseNames);
1358
1359     this_query->set_Z_Query(sr->query);
1360
1361     char query_str[120];
1362     this_query->print(query_str, sizeof(query_str)-1);
1363     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1364
1365     if (*m_parent->m_optimize != '0' &&
1366         m_client->m_last_ok && m_client->m_last_query &&
1367         m_client->m_last_query->match(this_query) &&
1368         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1369         m_client->m_last_databases.match(this_databases))
1370     {
1371         delete this_query;
1372         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1373             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1374         {
1375             Z_NamePlusRecordList *npr;
1376             int toget = *sr->mediumSetPresentNumber;
1377             Z_RecordComposition *comp = 0;
1378
1379             if (toget > m_client->m_last_resultCount)
1380                 toget = m_client->m_last_resultCount;
1381
1382             if (sr->mediumSetElementSetNames)
1383             {
1384                 comp = (Z_RecordComposition *)
1385                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1386                 comp->which = Z_RecordComp_simple;
1387                 comp->u.simple = sr->mediumSetElementSetNames;
1388             }
1389
1390             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1391                                           sr->preferredRecordSyntax, comp))
1392             {
1393                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1394                          m_session_str);
1395                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1396                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1397                 new_apdu->u.searchResponse->resultCount =
1398                     &m_client->m_last_resultCount;
1399
1400                 new_apdu->u.searchResponse->numberOfRecordsReturned
1401                     = odr_intdup(odr_encode(), toget);
1402
1403                 new_apdu->u.searchResponse->presentStatus =
1404                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1405                 new_apdu->u.searchResponse->records = (Z_Records*)
1406                     odr_malloc(odr_encode(), sizeof(Z_Records));
1407                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1408                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1409                 new_apdu->u.searchResponse->nextResultSetPosition =
1410                     odr_intdup(odr_encode(), toget+1);
1411                 send_to_client(new_apdu);
1412                 return 0;
1413             }
1414             else
1415             {
1416                 // medium Set
1417                 // send present request (medium size)
1418                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1419                          m_session_str);
1420
1421                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1422                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1423                 pr->referenceId = sr->referenceId;
1424                 pr->resultSetId = sr->resultSetName;
1425                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1426                 *pr->numberOfRecordsRequested = toget;
1427                 pr->recordComposition = comp;
1428                 m_client->m_sr_transform = 1;
1429                 return new_apdu;
1430             }
1431         }
1432         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1433             m_client->m_last_resultCount <= 0)
1434         {
1435             // large set. Return pseudo-search response immediately
1436             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1437                      m_session_str);
1438             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1439             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1440             new_apdu->u.searchResponse->resultCount =
1441                 &m_client->m_last_resultCount;
1442             send_to_client(new_apdu);
1443             return 0;
1444         }
1445         else
1446         {
1447             Z_NamePlusRecordList *npr;
1448             int toget = m_client->m_last_resultCount;
1449             Z_RecordComposition *comp = 0;
1450             // small set
1451             // send a present request (small set)
1452
1453             if (sr->smallSetElementSetNames)
1454             {
1455                 comp = (Z_RecordComposition *)
1456                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1457                 comp->which = Z_RecordComp_simple;
1458                 comp->u.simple = sr->smallSetElementSetNames;
1459             }
1460
1461             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1462                                           sr->preferredRecordSyntax, comp))
1463             {
1464                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1465                          m_session_str);
1466                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1467                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1468                 new_apdu->u.searchResponse->resultCount =
1469                     &m_client->m_last_resultCount;
1470
1471                 new_apdu->u.searchResponse->numberOfRecordsReturned
1472                     = odr_intdup(odr_encode(), toget);
1473
1474                 new_apdu->u.searchResponse->presentStatus =
1475                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1476                 new_apdu->u.searchResponse->records = (Z_Records*)
1477                     odr_malloc(odr_encode(), sizeof(Z_Records));
1478                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1479                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1480                 new_apdu->u.searchResponse->nextResultSetPosition =
1481                     odr_intdup(odr_encode(), toget+1);
1482                 send_to_client(new_apdu);
1483                 return 0;
1484             }
1485             else
1486             {
1487                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1488                          m_session_str);
1489                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1490                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1491                 pr->referenceId = sr->referenceId;
1492                 pr->resultSetId = sr->resultSetName;
1493                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1494                 *pr->numberOfRecordsRequested = toget;
1495                 pr->recordComposition = comp;
1496                 m_client->m_sr_transform = 1;
1497                 return new_apdu;
1498             }
1499         }
1500     }
1501     else  // query doesn't match
1502     {
1503         delete m_client->m_last_query;
1504         m_client->m_last_query = this_query;
1505         m_client->m_last_ok = 0;
1506         m_client->m_cache.clear();
1507         m_client->m_resultSetStartPoint = 0;
1508
1509         xfree (m_client->m_last_resultSetId);
1510         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1511
1512         m_client->m_last_databases.set(sr->num_databaseNames,
1513                                        (const char **) sr->databaseNames);
1514     }
1515     return apdu;
1516 }
1517
1518
1519 void Yaz_Proxy::inc_request_no()
1520 {
1521     char *cp = strchr(m_session_str, ' ');
1522     m_request_no++;
1523     if (cp)
1524         sprintf(cp+1, "%d ", m_request_no);
1525 }
1526
1527 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1528 {
1529     inc_request_no();
1530
1531     m_bytes_recv += len;
1532
1533     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1534         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1535                  m_session_str, gdu_name(apdu), len);
1536
1537     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1538         shutdown();
1539
1540     m_bw_stat.add_bytes(len);
1541     m_pdu_stat.add_bytes(1);
1542
1543 #if HAVE_GETTIMEOFDAY
1544     gettimeofday((struct timeval *) m_time_tv, 0);
1545 #endif
1546
1547     int bw_total = m_bw_stat.get_total();
1548     int pdu_total = m_pdu_stat.get_total();
1549
1550     int reduce = 0;
1551     if (m_bw_max)
1552     {
1553         if (bw_total > m_bw_max)
1554         {
1555             reduce = (bw_total/m_bw_max);
1556         }
1557     }
1558     if (m_pdu_max)
1559     {
1560         if (pdu_total > m_pdu_max)
1561         {
1562             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1563             reduce = (reduce > nreduce) ? reduce : nreduce;
1564         }
1565     }
1566     m_http_version = 0;
1567     if (reduce)
1568     {
1569         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1570                 m_session_str, reduce, bw_total, pdu_total,
1571                 m_bw_max, m_pdu_max);
1572
1573         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1574         timeout(reduce);       // call us reduce seconds later
1575     }
1576     else if (apdu->which == Z_GDU_Z3950)
1577         handle_incoming_Z_PDU(apdu->u.z3950);
1578     else if (apdu->which == Z_GDU_HTTP_Request)
1579         handle_incoming_HTTP(apdu->u.HTTP_Request);
1580 }
1581
1582 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1583 {
1584     if (m_max_record_retrieve)
1585     {
1586         if (apdu->which == Z_APDU_presentRequest)
1587         {
1588             Z_PresentRequest *pr = apdu->u.presentRequest;
1589             if (pr->numberOfRecordsRequested &&
1590                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1591                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1592         }
1593     }
1594 }
1595
1596 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1597                                                      int error,
1598                                                      const char *addinfo)
1599 {
1600     Z_Records *rec = (Z_Records *)
1601         odr_malloc (odr, sizeof(*rec));
1602     int *err = (int *)
1603         odr_malloc (odr, sizeof(*err));
1604     Z_DiagRec *drec = (Z_DiagRec *)
1605         odr_malloc (odr, sizeof(*drec));
1606     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1607         odr_malloc (odr, sizeof(*dr));
1608     *err = error;
1609     rec->which = Z_Records_NSD;
1610     rec->u.nonSurrogateDiagnostic = dr;
1611     dr->diagnosticSetId =
1612         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1613     dr->condition = err;
1614     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1615     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1616     return rec;
1617 }
1618
1619 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1620 {
1621     if (apdu->which == Z_APDU_searchRequest &&
1622         apdu->u.searchRequest->query &&
1623         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1624         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1625     {
1626         Z_RPNQuery *rpnquery = 0;
1627         Z_SearchRequest *sr = apdu->u.searchRequest;
1628         char *addinfo = 0;
1629
1630         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
1631                 sr->query->u.type_104->u.cql);
1632
1633         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1634                                           &rpnquery, odr_encode(),
1635                                           &addinfo);
1636         if (r == -3)
1637             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
1638         else if (r)
1639         {
1640             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1641             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1642
1643             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1644             new_apdu->u.searchResponse->records =
1645                 create_nonSurrogateDiagnostics(odr_encode(),
1646                                                yaz_diag_srw_to_bib1(r),
1647                                                addinfo);
1648             *new_apdu->u.searchResponse->searchStatus = 0;
1649
1650             send_to_client(new_apdu);
1651
1652             return 0;
1653         }
1654         else
1655         {
1656             sr->query->which = Z_Query_type_1;
1657             sr->query->u.type_1 = rpnquery;
1658         }
1659         return apdu;
1660     }
1661     return apdu;
1662 }
1663
1664 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1665 {
1666     if (apdu->which == Z_APDU_searchRequest)
1667     {
1668         Z_SearchRequest *sr = apdu->u.searchRequest;
1669         int err = 0;
1670         char *addinfo = 0;
1671
1672         Yaz_ProxyConfig *cfg = check_reconfigure();
1673         if (cfg)
1674             err = cfg->check_query(odr_encode(), m_default_target,
1675                                    sr->query, &addinfo);
1676         if (err)
1677         {
1678             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1679
1680             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1681             new_apdu->u.searchResponse->records =
1682                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1683             *new_apdu->u.searchResponse->searchStatus = 0;
1684
1685             send_to_client(new_apdu);
1686
1687             return 0;
1688         }
1689     }
1690     return apdu;
1691 }
1692
1693 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1694 {
1695     m_marcxml_flag = 0;
1696     if (apdu->which == Z_APDU_searchRequest)
1697     {
1698         Z_SearchRequest *sr = apdu->u.searchRequest;
1699         int err = 0;
1700         char *addinfo = 0;
1701         Yaz_ProxyConfig *cfg = check_reconfigure();
1702
1703         Z_RecordComposition rc_temp, *rc = 0;
1704         if (sr->smallSetElementSetNames)
1705         {
1706             rc_temp.which = Z_RecordComp_simple;
1707             rc_temp.u.simple = sr->smallSetElementSetNames;
1708             rc = &rc_temp;
1709         }
1710
1711         if (sr->preferredRecordSyntax)
1712         {
1713             struct oident *ent;
1714             ent = oid_getentbyoid(sr->preferredRecordSyntax);
1715             m_frontend_type = ent->value;
1716         }
1717         else
1718             m_frontend_type = VAL_NONE;
1719
1720         char *stylesheet_name = 0;
1721         if (cfg)
1722             err = cfg->check_syntax(odr_encode(),
1723                                     m_default_target,
1724                                     sr->preferredRecordSyntax, rc,
1725                                     &addinfo, &stylesheet_name, &m_schema,
1726                                     &m_backend_type, &m_backend_charset,
1727                                     &m_usemarcon_ini_stage1,
1728                                     &m_usemarcon_ini_stage2);
1729         if (stylesheet_name)
1730         {
1731             m_parent->low_socket_close();
1732
1733 #if HAVE_XSLT
1734             if (m_stylesheet_xsp)
1735                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
1736             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1737                                                        stylesheet_name);
1738 #endif
1739             m_stylesheet_offset = 0;
1740             xfree(stylesheet_name);
1741
1742             m_parent->low_socket_open();
1743         }
1744         if (err == -1)
1745         {
1746             sr->smallSetElementSetNames = 0;
1747             sr->mediumSetElementSetNames = 0;
1748             if (m_backend_type)
1749             {
1750
1751                 sr->preferredRecordSyntax =
1752                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
1753                                         m_backend_type);
1754             }
1755             else
1756                 sr->preferredRecordSyntax =
1757                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
1758                                            VAL_USMARC);
1759             m_marcxml_flag = 1;
1760         }
1761         else if (err)
1762         {
1763             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1764
1765             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1766             new_apdu->u.searchResponse->records =
1767                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1768             *new_apdu->u.searchResponse->searchStatus = 0;
1769
1770             send_to_client(new_apdu);
1771
1772             return 0;
1773         }
1774         else if (m_backend_type)
1775         {
1776             sr->preferredRecordSyntax =
1777                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
1778         }
1779     }
1780     else if (apdu->which == Z_APDU_presentRequest)
1781     {
1782         Z_PresentRequest *pr = apdu->u.presentRequest;
1783         int err = 0;
1784         char *addinfo = 0;
1785         Yaz_ProxyConfig *cfg = check_reconfigure();
1786
1787         if (pr->preferredRecordSyntax)
1788         {
1789             struct oident *ent;
1790             ent = oid_getentbyoid(pr->preferredRecordSyntax);
1791             m_frontend_type = ent->value;
1792         }
1793         else
1794             m_frontend_type = VAL_NONE;
1795
1796         char *stylesheet_name = 0;
1797         if (cfg)
1798             err = cfg->check_syntax(odr_encode(), m_default_target,
1799                                     pr->preferredRecordSyntax,
1800                                     pr->recordComposition,
1801                                     &addinfo, &stylesheet_name, &m_schema,
1802                                     &m_backend_type, &m_backend_charset,
1803                                     &m_usemarcon_ini_stage1,
1804                                     &m_usemarcon_ini_stage2
1805                                     );
1806         if (stylesheet_name)
1807         {
1808             m_parent->low_socket_close();
1809
1810 #if HAVE_XSLT
1811             if (m_stylesheet_xsp)
1812                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
1813             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1814                                                        stylesheet_name);
1815 #endif
1816             m_stylesheet_offset = 0;
1817             xfree(stylesheet_name);
1818
1819             m_parent->low_socket_open();
1820         }
1821         if (err == -1)
1822         {
1823             pr->recordComposition = 0;
1824             if (m_backend_type)
1825             {
1826
1827                 pr->preferredRecordSyntax =
1828                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
1829                                         m_backend_type);
1830             }
1831             else
1832                 pr->preferredRecordSyntax =
1833                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
1834                                            VAL_USMARC);
1835             m_marcxml_flag = 1;
1836         }
1837         else if (err)
1838         {
1839             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1840
1841             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1842             new_apdu->u.presentResponse->records =
1843                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1844             *new_apdu->u.presentResponse->presentStatus =
1845                 Z_PresentStatus_failure;
1846
1847             send_to_client(new_apdu);
1848
1849             return 0;
1850         }
1851         else if (m_backend_type)
1852         {
1853             pr->preferredRecordSyntax =
1854                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
1855         }
1856     }
1857     return apdu;
1858 }
1859
1860 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
1861 {
1862     if (!schema)
1863         return 0;
1864     Z_ElementSetNames *esn = (Z_ElementSetNames *)
1865         odr_malloc(o, sizeof(Z_ElementSetNames));
1866     esn->which = Z_ElementSetNames_generic;
1867     esn->u.generic = odr_strdup(o, schema);
1868     return esn;
1869 }
1870
1871 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
1872 {
1873     const char *t = 0;
1874     Yaz_ProxyConfig *cfg = check_reconfigure();
1875     if (cfg)
1876         t = cfg->get_explain_name(db, backend_db);
1877
1878     if (m_client && m_default_target && t && strcmp(m_default_target, t))
1879     {
1880         releaseClient();
1881     }
1882
1883     if (t)
1884     {
1885         xfree(m_default_target);
1886         m_default_target = xstrdup(t);
1887     }
1888 }
1889
1890 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
1891 {
1892     yaz_log(YLOG_LOG, "file_access");
1893     if (strcmp(hreq->method, "GET"))
1894         return 0;
1895     struct stat sbuf;
1896     if (hreq->path[0] != '/')
1897     {
1898         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
1899         return 0;
1900     }
1901     const char *cp = hreq->path;
1902     while (*cp)
1903     {
1904         if (*cp == '/' && strchr("/.", cp[1]))
1905         {
1906             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
1907             return 0;
1908         }
1909         cp++;
1910     }
1911     const char *fname = hreq->path+1;
1912     if (stat(fname, &sbuf))
1913     {
1914         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
1915         return 0;
1916     }
1917     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
1918     {
1919         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
1920         return 0;
1921     }
1922     if (sbuf.st_size > (off_t) 1000000)
1923     {
1924         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
1925         return 0;
1926     }
1927
1928     ODR o = odr_encode();
1929     Yaz_ProxyConfig *cfg = check_reconfigure();
1930     const char *ctype = cfg->check_mime_type(fname);
1931     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
1932     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1933     if (m_http_version)
1934         hres->version = odr_strdup(o, m_http_version);
1935     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1936     if (m_http_keepalive)
1937         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1938     else
1939         timeout(0);
1940
1941     hres->content_len = sbuf.st_size;
1942     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
1943     FILE *f = fopen(fname, "rb");
1944     if (f)
1945     {
1946         fread(hres->content_buf, 1, hres->content_len, f);
1947         fclose(f);
1948     }
1949     else
1950     {
1951         return 0;
1952     }
1953     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1954     {
1955         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
1956                  fname);
1957     }
1958     int len;
1959     send_GDU(gdu, &len);
1960     return 1;
1961 }
1962
1963 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1964 {
1965     if (m_s2z_odr_init)
1966     {
1967         odr_destroy(m_s2z_odr_init);
1968         m_s2z_odr_init = 0;
1969     }
1970     if (m_s2z_odr_search)
1971     {
1972         odr_destroy(m_s2z_odr_search);
1973         m_s2z_odr_search = 0;
1974     }
1975
1976     m_http_keepalive = 0;
1977     m_http_version = 0;
1978     if (!strcmp(hreq->version, "1.0"))
1979     {
1980         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1981         if (v && !strcmp(v, "Keep-Alive"))
1982             m_http_keepalive = 1;
1983         else
1984             m_http_keepalive = 0;
1985         m_http_version = "1.0";
1986     }
1987     else
1988     {
1989         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1990         if (v && !strcmp(v, "close"))
1991             m_http_keepalive = 0;
1992         else
1993             m_http_keepalive = 1;
1994         m_http_version = "1.1";
1995     }
1996
1997     Z_SRW_PDU *srw_pdu = 0;
1998     Z_SOAP *soap_package = 0;
1999     char *charset = 0;
2000     Z_SRW_diagnostic *diagnostic = 0;
2001     int num_diagnostic = 0;
2002
2003     if (file_access(hreq))
2004     {
2005         return;
2006     }
2007     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2008                             &charset) == 0
2009              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2010                                &charset, &diagnostic, &num_diagnostic) == 0)
2011     {
2012         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2013         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2014         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2015         m_s2z_init_apdu = 0;
2016         m_s2z_search_apdu = 0;
2017         m_s2z_present_apdu = 0;
2018
2019         m_s2z_stylesheet = 0;
2020
2021         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2022         {
2023             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2024
2025             const char *backend_db = srw_req->database;
2026             srw_get_client(srw_req->database, &backend_db);
2027
2028             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2029             // recordXPath unsupported.
2030             if (srw_req->recordXPath)
2031             {
2032                 yaz_add_srw_diagnostic(odr_decode(),
2033                                        &diagnostic, &num_diagnostic,
2034                                        72, 0);
2035             }
2036             // sort unsupported
2037             if (srw_req->sort_type != Z_SRW_sort_type_none)
2038             {
2039                 yaz_add_srw_diagnostic(odr_decode(),
2040                                        &diagnostic, &num_diagnostic,
2041                                        80, 0);
2042             }
2043             // save stylesheet
2044             if (srw_req->stylesheet)
2045                 m_s2z_stylesheet =
2046                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2047
2048             // set packing for response records ..
2049             if (srw_req->recordPacking &&
2050                 !strcmp(srw_req->recordPacking, "xml"))
2051                 m_s2z_packing = Z_SRW_recordPacking_XML;
2052             else
2053                 m_s2z_packing = Z_SRW_recordPacking_string;
2054
2055             if (num_diagnostic)
2056             {
2057                 Z_SRW_PDU *srw_pdu =
2058                     yaz_srw_get(odr_encode(),
2059                                 Z_SRW_searchRetrieve_response);
2060                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2061
2062                 srw_res->diagnostics = diagnostic;
2063                 srw_res->num_diagnostics = num_diagnostic;
2064                 send_srw_response(srw_pdu);
2065                 return;
2066             }
2067
2068             // prepare search PDU
2069             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2070                                           Z_APDU_searchRequest);
2071             Z_SearchRequest *z_searchRequest =
2072                 m_s2z_search_apdu->u.searchRequest;
2073
2074             z_searchRequest->num_databaseNames = 1;
2075             z_searchRequest->databaseNames = (char**)
2076                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2077             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2078                                                            backend_db);
2079
2080             // query transformation
2081             Z_Query *query = (Z_Query *)
2082                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2083             z_searchRequest->query = query;
2084
2085             if (srw_req->query_type == Z_SRW_query_type_cql)
2086             {
2087                 Z_External *ext = (Z_External *)
2088                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2089                 ext->direct_reference =
2090                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2091                 ext->indirect_reference = 0;
2092                 ext->descriptor = 0;
2093                 ext->which = Z_External_CQL;
2094                 ext->u.cql = srw_req->query.cql;
2095
2096                 query->which = Z_Query_type_104;
2097                 query->u.type_104 =  ext;
2098             }
2099             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2100             {
2101                 Z_RPNQuery *RPNquery;
2102                 YAZ_PQF_Parser pqf_parser;
2103
2104                 pqf_parser = yaz_pqf_create ();
2105
2106                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2107                                           srw_req->query.pqf);
2108                 if (!RPNquery)
2109                 {
2110                     const char *pqf_msg;
2111                     size_t off;
2112                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2113                     yaz_log(YLOG_LOG, "%*s^\n", off+4, "");
2114                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2115
2116                     send_to_srw_client_error(10, 0);
2117                     return;
2118                 }
2119                 query->which = Z_Query_type_1;
2120                 query->u.type_1 =  RPNquery;
2121
2122                 yaz_pqf_destroy (pqf_parser);
2123             }
2124             else
2125             {
2126                 send_to_srw_client_error(7, "query");
2127                 return;
2128             }
2129
2130             // present
2131             m_s2z_present_apdu = 0;
2132             int max = 0;
2133             if (srw_req->maximumRecords)
2134                 max = *srw_req->maximumRecords;
2135             int start = 1;
2136             if (srw_req->startRecord)
2137                 start = *srw_req->startRecord;
2138             if (max > 0)
2139             {
2140                 // Some backend, such as Voyager doesn't honor piggyback
2141                 // So we use present always (0 &&).
2142                 if (0 && start <= 1)  // Z39.50 piggyback
2143                 {
2144                     *z_searchRequest->smallSetUpperBound = max;
2145                     *z_searchRequest->mediumSetPresentNumber = max;
2146                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2147
2148                     z_searchRequest->preferredRecordSyntax =
2149                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2150                                                VAL_TEXT_XML);
2151                     if (srw_req->recordSchema)
2152                     {
2153                         z_searchRequest->smallSetElementSetNames =
2154                             z_searchRequest->mediumSetElementSetNames =
2155                             mk_esn_from_schema(m_s2z_odr_search,
2156                                                srw_req->recordSchema);
2157                     }
2158                 }
2159                 else   // Z39.50 present
2160                 {
2161                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2162                                                    Z_APDU_presentRequest);
2163                     Z_PresentRequest *z_presentRequest =
2164                         m_s2z_present_apdu->u.presentRequest;
2165                     *z_presentRequest->resultSetStartPoint = start;
2166                     *z_presentRequest->numberOfRecordsRequested = max;
2167                     z_presentRequest->preferredRecordSyntax =
2168                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2169                                                VAL_TEXT_XML);
2170                     if (srw_req->recordSchema)
2171                     {
2172                         z_presentRequest->recordComposition =
2173                             (Z_RecordComposition *)
2174                             odr_malloc(m_s2z_odr_search,
2175                                        sizeof(Z_RecordComposition));
2176                         z_presentRequest->recordComposition->which =
2177                             Z_RecordComp_simple;
2178                         z_presentRequest->recordComposition->u.simple =
2179                             mk_esn_from_schema(m_s2z_odr_search,
2180                                                srw_req->recordSchema);
2181                     }
2182                 }
2183             }
2184             if (!m_client)
2185             {
2186                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2187                                             Z_APDU_initRequest);
2188
2189                 // prevent m_initRequest_apdu memory from being grabbed
2190                 // in Yaz_Proxy::handle_incoming_Z_PDU
2191                 m_initRequest_apdu = m_s2z_init_apdu;
2192                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2193                 return;
2194             }
2195             else
2196             {
2197                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2198                 return;
2199             }
2200         }
2201         else if (srw_pdu->which == Z_SRW_explain_request)
2202         {
2203             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2204
2205             const char *backend_db = srw_req->database;
2206             srw_get_client(srw_req->database, &backend_db);
2207
2208             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2209
2210             // save stylesheet
2211             if (srw_req->stylesheet)
2212                 m_s2z_stylesheet =
2213                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2214
2215             if (srw_req->recordPacking &&
2216                 !strcmp(srw_req->recordPacking, "xml"))
2217                 m_s2z_packing = Z_SRW_recordPacking_XML;
2218             else
2219                 m_s2z_packing = Z_SRW_recordPacking_string;
2220
2221             if (num_diagnostic)
2222             {
2223                 send_srw_explain_response(diagnostic, num_diagnostic);
2224                 return;
2225             }
2226
2227             if (!m_client)
2228             {
2229                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2230                                             Z_APDU_initRequest);
2231
2232                 // prevent m_initRequest_apdu memory from being grabbed
2233                 // in Yaz_Proxy::handle_incoming_Z_PDU
2234                 m_initRequest_apdu = m_s2z_init_apdu;
2235                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2236             }
2237             else
2238                 send_srw_explain_response(0, 0);
2239             return;
2240         }
2241         else if (srw_pdu->which == Z_SRW_scan_request)
2242         {
2243             m_s2z_database = odr_strdup(m_s2z_odr_init,
2244                                         srw_pdu->u.scan_request->database);
2245
2246             yaz_add_srw_diagnostic(odr_decode(),
2247                                    &diagnostic, &num_diagnostic,
2248                                    4, "scan");
2249             Z_SRW_PDU *srw_pdu =
2250                 yaz_srw_get(odr_encode(),
2251                             Z_SRW_scan_response);
2252             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2253
2254             srw_res->diagnostics = diagnostic;
2255             srw_res->num_diagnostics = num_diagnostic;
2256             send_srw_response(srw_pdu);
2257             return;
2258         }
2259         else
2260         {
2261             m_s2z_database = 0;
2262
2263             send_to_srw_client_error(4, 0);
2264         }
2265     }
2266     send_http_response(400);
2267 }
2268
2269 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2270 {
2271     Z_ReferenceId **refid = get_referenceIdP(apdu);
2272     nmem_reset(m_referenceId_mem);
2273     if (refid && *refid)
2274     {
2275         m_referenceId = (Z_ReferenceId *)
2276             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2277         m_referenceId->len = m_referenceId->size = (*refid)->len;
2278         m_referenceId->buf = (unsigned char *)
2279             nmem_malloc(m_referenceId_mem, (*refid)->len);
2280         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2281     }
2282     else
2283         m_referenceId = 0;
2284
2285     if (!m_client && m_invalid_session)
2286     {
2287         m_apdu_invalid_session = apdu;
2288         m_mem_invalid_session = odr_extract_mem(odr_decode());
2289         apdu = m_initRequest_apdu;
2290     }
2291
2292     // Determine our client.
2293     Z_OtherInformation **oi;
2294     get_otherInfoAPDU(apdu, &oi);
2295     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2296     if (!m_client)
2297     {
2298         if (m_http_version)
2299         {
2300             send_http_response(404);
2301             return;
2302         }
2303         else
2304         {
2305             delete this;
2306             return;
2307         }
2308     }
2309     m_client->m_server = this;
2310
2311     if (apdu->which == Z_APDU_initRequest)
2312     {
2313         if (apdu->u.initRequest->implementationId)
2314             yaz_log(YLOG_LOG, "%simplementationId: %s",
2315                     m_session_str, apdu->u.initRequest->implementationId);
2316         if (apdu->u.initRequest->implementationName)
2317             yaz_log(YLOG_LOG, "%simplementationName: %s",
2318                     m_session_str, apdu->u.initRequest->implementationName);
2319         if (apdu->u.initRequest->implementationVersion)
2320             yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2321                     m_session_str, apdu->u.initRequest->implementationVersion);
2322         if (m_initRequest_apdu == 0)
2323         {
2324             if (m_initRequest_mem)
2325                 nmem_destroy(m_initRequest_mem);
2326             m_initRequest_apdu = apdu;
2327             m_initRequest_mem = odr_extract_mem(odr_decode());
2328
2329             m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2330                 preferredMessageSize;
2331             *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2332             m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2333                 maximumRecordSize;
2334             *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2335
2336             // save init options for the response..
2337             m_initRequest_options = apdu->u.initRequest->options;
2338
2339             apdu->u.initRequest->options =
2340                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2341                                            sizeof(Odr_bitmask));
2342             ODR_MASK_ZERO(apdu->u.initRequest->options);
2343             int i;
2344             for (i = 0; i<= 24; i++)
2345                 ODR_MASK_SET(apdu->u.initRequest->options, i);
2346             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2347                            Z_Options_negotiationModel);
2348             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2349                            Z_Options_concurrentOperations);
2350
2351             // make new version
2352             m_initRequest_version = apdu->u.initRequest->protocolVersion;
2353             apdu->u.initRequest->protocolVersion =
2354                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2355                                            sizeof(Odr_bitmask));
2356             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2357
2358             for (i = 0; i<= 8; i++)
2359                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2360         }
2361         if (m_client->m_init_flag)
2362         {
2363             if (handle_init_response_for_invalid_session(apdu))
2364                 return;
2365             if (m_client->m_initResponse)
2366             {
2367                 Z_APDU *apdu2 = m_client->m_initResponse;
2368                 apdu2->u.initResponse->otherInfo = 0;
2369                 if (m_client->m_cookie && *m_client->m_cookie)
2370                     set_otherInformationString(apdu2, VAL_COOKIE, 1,
2371                                                m_client->m_cookie);
2372                 apdu2->u.initResponse->referenceId =
2373                     apdu->u.initRequest->referenceId;
2374                 apdu2->u.initResponse->options = m_client->m_initResponse_options;
2375                 apdu2->u.initResponse->protocolVersion =
2376                     m_client->m_initResponse_version;
2377
2378                 send_to_client(apdu2);
2379                 return;
2380             }
2381         }
2382         m_client->m_init_flag = 1;
2383     }
2384     handle_max_record_retrieve(apdu);
2385
2386     if (apdu)
2387         apdu = handle_syntax_validation(apdu);
2388
2389     if (apdu)
2390         apdu = handle_query_transformation(apdu);
2391
2392     if (apdu)
2393         apdu = handle_query_validation(apdu);
2394
2395     if (apdu)
2396         apdu = result_set_optimize(apdu);
2397     if (!apdu)
2398     {
2399         m_client->timeout(m_target_idletime);  // mark it active even
2400         // though we didn't use it
2401         return;
2402     }
2403
2404     // delete other info construct completely if 0 elements
2405     get_otherInfoAPDU(apdu, &oi);
2406     if (oi && *oi && (*oi)->num_elements == 0)
2407         *oi = 0;
2408
2409     if (apdu->which == Z_APDU_presentRequest &&
2410         m_client->m_resultSetStartPoint == 0)
2411     {
2412         Z_PresentRequest *pr = apdu->u.presentRequest;
2413         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2414         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2415     } else {
2416         m_client->m_resultSetStartPoint = 0;
2417     }
2418     if (m_client->send_to_target(apdu) < 0)
2419     {
2420         delete m_client;
2421         m_client = 0;
2422         delete this;
2423     }
2424     else
2425         m_client->m_waiting = 1;
2426 }
2427
2428 void Yaz_Proxy::connectNotify()
2429 {
2430 }
2431
2432 void Yaz_Proxy::releaseClient()
2433 {
2434     xfree(m_proxyTarget);
2435     m_proxyTarget = 0;
2436     m_invalid_session = 0;
2437     // only keep if keep_alive flag is set...
2438     if (m_client &&
2439         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2440         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2441         m_client->m_waiting == 0)
2442     {
2443         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2444                  m_session_str,
2445                  m_client->get_hostname());
2446         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2447                 m_session_str, m_client->m_pdu_recv,
2448                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2449                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2450         assert (m_client->m_waiting != 2);
2451         // Tell client (if any) that no server connection is there..
2452         m_client->m_server = 0;
2453         m_client = 0;
2454     }
2455     else if (m_client)
2456     {
2457         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
2458                  m_session_str,
2459                  m_client->get_hostname());
2460         assert (m_client->m_waiting != 2);
2461         delete m_client;
2462         m_client = 0;
2463     }
2464     else if (!m_parent)
2465     {
2466         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
2467                  m_session_str);
2468         assert (m_parent);
2469     }
2470     else
2471     {
2472         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
2473                  m_session_str);
2474     }
2475     if (m_parent)
2476         m_parent->pre_init();
2477 }
2478
2479 void Yaz_Proxy::shutdown()
2480 {
2481     releaseClient();
2482     delete this;
2483 }
2484
2485 const char *Yaz_ProxyClient::get_session_str()
2486 {
2487     if (!m_server)
2488         return "0 ";
2489     return m_server->get_session_str();
2490 }
2491
2492 void Yaz_ProxyClient::shutdown()
2493 {
2494     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2495              get_hostname());
2496     delete m_server;
2497     delete this;
2498 }
2499
2500 void Yaz_Proxy::failNotify()
2501 {
2502     inc_request_no();
2503     yaz_log (YLOG_LOG, "%sConnection closed by client",
2504              get_session_str());
2505     shutdown();
2506 }
2507
2508 void Yaz_ProxyClient::failNotify()
2509 {
2510     if (m_server)
2511         m_server->inc_request_no();
2512     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
2513              get_session_str(), get_hostname());
2514     shutdown();
2515 }
2516
2517 void Yaz_ProxyClient::connectNotify()
2518 {
2519     const char *s = get_session_str();
2520     const char *h = get_hostname();
2521     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2522              m_target_idletime);
2523     timeout(m_target_idletime);
2524     if (!m_server)
2525         pre_init_client();
2526 }
2527
2528 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2529                                                   *the_PDU_Observable, int fd)
2530 {
2531     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2532 }
2533
2534 Yaz_ProxyClient::~Yaz_ProxyClient()
2535 {
2536     if (m_prev)
2537         *m_prev = m_next;
2538     if (m_next)
2539         m_next->m_prev = m_prev;
2540     m_waiting = 2;     // for debugging purposes only.
2541     odr_destroy(m_init_odr);
2542     delete m_last_query;
2543     xfree (m_last_resultSetId);
2544     xfree (m_cookie);
2545 }
2546
2547 void Yaz_ProxyClient::pre_init_client()
2548 {
2549     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2550     Z_InitRequest *req = apdu->u.initRequest;
2551
2552     int i;
2553     for (i = 0; i<= 24; i++)
2554         ODR_MASK_SET(req->options, i);
2555     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2556                    Z_Options_negotiationModel);
2557     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2558                    Z_Options_concurrentOperations);
2559     for (i = 0; i<= 10; i++)
2560         ODR_MASK_SET(req->protocolVersion, i);
2561
2562     if (send_to_target(apdu) < 0)
2563     {
2564         delete this;
2565     }
2566     else
2567     {
2568         m_waiting = 1;
2569         m_init_flag = 1;
2570     }
2571 }
2572
2573 void Yaz_Proxy::pre_init()
2574 {
2575     int i;
2576     const char *name = 0;
2577     const char *zurl_in_use[MAX_ZURL_PLEX];
2578     int limit_bw, limit_pdu, limit_req;
2579     int target_idletime, client_idletime;
2580     int max_clients;
2581     int keepalive_limit_bw, keepalive_limit_pdu;
2582     int pre_init;
2583     const char *cql2rpn = 0;
2584     const char *authentication = 0;
2585
2586     Yaz_ProxyConfig *cfg = check_reconfigure();
2587
2588     zurl_in_use[0] = 0;
2589
2590     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2591         set_APDU_yazlog(1);
2592     else
2593         set_APDU_yazlog(0);
2594
2595     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2596                                           &limit_bw, &limit_pdu, &limit_req,
2597                                           &target_idletime, &client_idletime,
2598                                           &max_clients,
2599                                           &keepalive_limit_bw,
2600                                           &keepalive_limit_pdu,
2601                                           &pre_init,
2602                                           &cql2rpn,
2603                                           &authentication) ; i++)
2604     {
2605         if (pre_init)
2606         {
2607             int j;
2608             for (j = 0; zurl_in_use[j]; j++)
2609             {
2610                 Yaz_ProxyClient *c;
2611                 int spare = 0;
2612                 int spare_waiting = 0;
2613                 int in_use = 0;
2614                 int other = 0;
2615                 for (c = m_clientPool; c; c = c->m_next)
2616                 {
2617                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
2618                     {
2619                         if (c->m_cookie == 0)
2620                         {
2621                             if (c->m_server == 0)
2622                                 if (c->m_waiting)
2623                                     spare_waiting++;
2624                                 else
2625                                     spare++;
2626                             else
2627                                 in_use++;
2628                         }
2629                         else
2630                             other++;
2631                     }
2632                 }
2633                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2634                         "sparew=%d preinit=%d",m_session_str,
2635                         name, zurl_in_use[j], in_use, other,
2636                         spare, spare_waiting, pre_init);
2637                 if (spare + spare_waiting < pre_init)
2638                 {
2639                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2640                     c->m_next = m_clientPool;
2641                     if (c->m_next)
2642                         c->m_next->m_prev = &c->m_next;
2643                     m_clientPool = c;
2644                     c->m_prev = &m_clientPool;
2645
2646                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
2647                         c->set_APDU_yazlog(1);
2648                     else
2649                         c->set_APDU_yazlog(0);
2650
2651                     if (c->client(zurl_in_use[j]))
2652                     {
2653                         timeout(60);
2654                         delete c;
2655                         return;
2656                     }
2657                     c->timeout(30);
2658                     c->m_waiting = 1;
2659                     c->m_target_idletime = target_idletime;
2660                     c->m_seqno = m_seqno++;
2661                 }
2662             }
2663         }
2664     }
2665 }
2666
2667 void Yaz_Proxy::timeoutNotify()
2668 {
2669     if (m_parent)
2670     {
2671         if (m_bw_hold_PDU)
2672         {
2673             timeout(m_client_idletime);
2674             Z_GDU *apdu = m_bw_hold_PDU;
2675             m_bw_hold_PDU = 0;
2676
2677             if (apdu->which == Z_GDU_Z3950)
2678                 handle_incoming_Z_PDU(apdu->u.z3950);
2679             else if (apdu->which == Z_GDU_HTTP_Request)
2680                 handle_incoming_HTTP(apdu->u.HTTP_Request);
2681         }
2682         else if (m_stylesheet_nprl)
2683             convert_xsl_delay();
2684         else
2685         {
2686             inc_request_no();
2687
2688             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2689             shutdown();
2690         }
2691     }
2692     else
2693     {
2694         timeout(600);
2695         pre_init();
2696     }
2697 }
2698
2699 void Yaz_Proxy::markInvalid()
2700 {
2701     m_client = 0;
2702     m_invalid_session = 1;
2703 }
2704
2705 void Yaz_ProxyClient::timeoutNotify()
2706 {
2707     if (m_server)
2708         m_server->inc_request_no();
2709
2710     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2711              get_hostname());
2712     m_waiting = 1;
2713     m_root->pre_init();
2714     if (m_server && m_init_flag)
2715     {
2716         // target timed out in a session that was properly initialized
2717         // server object stay alive but we mark it as invalid so it
2718         // gets initialized again
2719         m_server->markInvalid();
2720         m_server = 0;
2721     }
2722     shutdown();
2723 }
2724
2725 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2726                                  Yaz_Proxy *parent) :
2727     Yaz_Z_Assoc (the_PDU_Observable)
2728 {
2729     m_cookie = 0;
2730     m_next = 0;
2731     m_prev = 0;
2732     m_init_flag = 0;
2733     m_last_query = 0;
2734     m_last_resultSetId = 0;
2735     m_last_resultCount = 0;
2736     m_last_ok = 0;
2737     m_sr_transform = 0;
2738     m_waiting = 0;
2739     m_init_odr = odr_createmem (ODR_DECODE);
2740     m_initResponse = 0;
2741     m_initResponse_options = 0;
2742     m_initResponse_version = 0;
2743     m_initResponse_preferredMessageSize = 0;
2744     m_initResponse_maximumRecordSize = 0;
2745     m_resultSetStartPoint = 0;
2746     m_bytes_sent = m_bytes_recv = 0;
2747     m_pdu_recv = 0;
2748     m_server = 0;
2749     m_seqno = 0;
2750     m_target_idletime = 600;
2751     m_root = parent;
2752 }
2753
2754 const char *Yaz_Proxy::option(const char *name, const char *value)
2755 {
2756     if (!strcmp (name, "optimize")) {
2757         if (value) {
2758             xfree (m_optimize);
2759             m_optimize = xstrdup (value);
2760         }
2761         return m_optimize;
2762     }
2763     return 0;
2764 }
2765
2766 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2767 {
2768
2769 }
2770
2771 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2772 {
2773     if (apdu->which == Z_GDU_Z3950)
2774         recv_Z_PDU(apdu->u.z3950, len);
2775     else if (apdu->which == Z_GDU_HTTP_Response)
2776         recv_HTTP_response(apdu->u.HTTP_Response, len);
2777     else
2778         shutdown();
2779 }
2780
2781 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2782 {
2783     if (!m_invalid_session)
2784         return 0;
2785     m_invalid_session = 0;
2786     handle_incoming_Z_PDU(m_apdu_invalid_session);
2787     assert (m_mem_invalid_session);
2788     nmem_destroy(m_mem_invalid_session);
2789     m_mem_invalid_session = 0;
2790     return 1;
2791 }
2792
2793 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2794 {
2795     m_bytes_recv += len;
2796
2797     m_pdu_recv++;
2798     m_waiting = 0;
2799     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2800         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2801                  apdu_name(apdu), get_hostname(), len);
2802     if (apdu->which == Z_APDU_initResponse)
2803     {
2804         if (!m_server)  // if this is a pre init session , check for more
2805             m_root->pre_init();
2806         NMEM nmem = odr_extract_mem (odr_decode());
2807         odr_reset (m_init_odr);
2808         nmem_transfer (m_init_odr->mem, nmem);
2809         m_initResponse = apdu;
2810         m_initResponse_options = apdu->u.initResponse->options;
2811         m_initResponse_version = apdu->u.initResponse->protocolVersion;
2812         m_initResponse_preferredMessageSize =
2813             *apdu->u.initResponse->preferredMessageSize;
2814         m_initResponse_maximumRecordSize =
2815             *apdu->u.initResponse->maximumRecordSize;
2816
2817         Z_InitResponse *ir = apdu->u.initResponse;
2818         char *im0 = ir->implementationName;
2819
2820         char *im1 = (char*)
2821             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2822         *im1 = '\0';
2823         if (im0)
2824         {
2825             strcat(im1, im0);
2826             strcat(im1, " ");
2827         }
2828         strcat(im1, "(YAZ Proxy)");
2829         ir->implementationName = im1;
2830
2831         nmem_destroy (nmem);
2832
2833         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2834             return;
2835     }
2836     if (apdu->which == Z_APDU_searchResponse)
2837     {
2838         Z_SearchResponse *sr = apdu->u.searchResponse;
2839         m_last_resultCount = *sr->resultCount;
2840         int status = *sr->searchStatus;
2841         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2842         {
2843             m_last_ok = 1;
2844
2845             if (sr->records && sr->records->which == Z_Records_DBOSD)
2846             {
2847                 m_cache.add(odr_decode(),
2848                             sr->records->u.databaseOrSurDiagnostics, 1,
2849                             *sr->resultCount);
2850             }
2851         }
2852     }
2853     if (apdu->which == Z_APDU_presentResponse)
2854     {
2855         Z_PresentResponse *pr = apdu->u.presentResponse;
2856         if (m_sr_transform)
2857         {
2858             m_sr_transform = 0;
2859             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2860             Z_SearchResponse *sr = new_apdu->u.searchResponse;
2861             sr->referenceId = pr->referenceId;
2862             *sr->resultCount = m_last_resultCount;
2863             sr->records = pr->records;
2864             sr->nextResultSetPosition = pr->nextResultSetPosition;
2865             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2866             apdu = new_apdu;
2867         }
2868         if (pr->records &&
2869             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2870         {
2871             m_cache.add(odr_decode(),
2872                         pr->records->u.databaseOrSurDiagnostics,
2873                         m_resultSetStartPoint, -1);
2874             m_resultSetStartPoint = 0;
2875         }
2876     }
2877     if (m_cookie)
2878         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2879     if (m_server)
2880     {
2881         m_server->send_to_client(apdu);
2882     }
2883     if (apdu->which == Z_APDU_close)
2884     {
2885         shutdown();
2886     }
2887 }
2888
2889 void Yaz_Proxy::low_socket_close()
2890 {
2891 #if WIN32
2892 #else
2893     int i;
2894     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
2895         if  (m_lo_fd[i] >= 0)
2896             ::close(m_lo_fd[i]);
2897 #endif
2898 }
2899
2900 void Yaz_Proxy::low_socket_open()
2901 {
2902 #if WIN32
2903 #else
2904     int i;
2905     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
2906         m_lo_fd[i] = open("/dev/null", O_RDONLY);
2907 #endif
2908 }
2909
2910 int Yaz_Proxy::server(const char *addr)
2911 {
2912     int r = Yaz_Z_Assoc::server(addr);
2913     if (!r)
2914     {
2915         yaz_log(YLOG_LOG, "%sStarted proxy "
2916 #ifdef VERSION
2917             VERSION
2918 #endif
2919             " on %s", m_session_str, addr);
2920         timeout(1);
2921     }
2922     return r;
2923 }
2924