src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.34 2005-06-25 15:58:33 adam Exp $
   2    Copyright (c) 1998-2005, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <stdlib.h>
  42 #include <time.h>
  43 #include <fcntl.h>
  44
  45 #include <yaz/srw.h>
  46 #include <yaz/marcdisp.h>
  47 #include <yaz/yaz-iconv.h>
  48 #include <yaz/log.h>
  49 #include <yaz/diagbib1.h>
  50 #include "proxyp.h"
  51 #include <yaz/pquery.h>
  52 #include <yaz/otherinfo.h>
  53 #include <yaz/charneg.h>
  54 #include "msg-thread.h"
  55
  56 using namespace yazpp_1;
  57
  58 #define USE_AUTH_MSG 1
  59
  60 #if USE_AUTH_MSG
  61 class Auth_Msg : public IMsg_Thread {
  62 public:
  63     int m_ret;
  64     IMsg_Thread *handle();
  65     void result();
  66     Yaz_Proxy *m_proxy;
  67     NMEM m_nmem;
  68     char *m_apdu_buf;
  69     int m_apdu_len;
  70     Auth_Msg();
  71     virtual ~Auth_Msg();
  72 };
  73
  74 Auth_Msg::Auth_Msg()
  75 {
  76     m_nmem = nmem_create();
  77 }
  78
  79 Auth_Msg::~Auth_Msg()
  80 {
  81     nmem_destroy(m_nmem);
  82 }
  83
  84 IMsg_Thread *Auth_Msg::handle()
  85 {
  86     yaz_log(YLOG_LOG, "Auth_Msg:handle begin");
  87     ODR decode = odr_createmem(ODR_DECODE);
  88     Z_APDU *apdu;
  89
  90     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
  91     int r = z_APDU(decode, &apdu, 0, 0);
  92     if (!r)
  93     {
  94         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
  95     }
  96     else
  97     {
  98         m_ret = m_proxy->handle_authentication(apdu);
  99     }
 100     yaz_log(YLOG_LOG, "Auth_Msg:handle end");
 101     odr_destroy(decode);
 102     return this;
 103 }
 104
 105 void Auth_Msg::result()
 106 {
 107     yaz_log(YLOG_LOG, "Auth_Msg:result proxy ok buf=%p len=%d",
 108             m_apdu_buf, m_apdu_len);
 109     odr_reset(m_proxy->odr_decode());
 110     odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
 111     Z_APDU *apdu = 0;
 112     int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
 113     if (r)
 114         yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU OK");
 115     else
 116         yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
 117     m_proxy->result_authentication(apdu, m_ret);
 118     delete this;
 119 }
 120
 121 #endif
 122
 123 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
 124 {
 125     if (ret == 0)
 126     {
 127         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
 128         *apdu_reject->u.initResponse->result = 0;
 129         send_to_client(apdu_reject);
 130         dec_ref();
 131     }
 132     else
 133         handle_incoming_Z_PDU_2(apdu);
 134 }
 135
 136 static const char *apdu_name(Z_APDU *apdu)
 137 {
 138     switch (apdu->which)
 139     {
 140     case Z_APDU_initRequest:
 141         return "initRequest";
 142     case Z_APDU_initResponse:
 143         return "initResponse";
 144     case Z_APDU_searchRequest:
 145         return "searchRequest";
 146     case Z_APDU_searchResponse:
 147         return "searchResponse";
 148     case Z_APDU_presentRequest:
 149         return "presentRequest";
 150     case Z_APDU_presentResponse:
 151         return "presentResponse";
 152     case Z_APDU_deleteResultSetRequest:
 153         return "deleteResultSetRequest";
 154     case Z_APDU_deleteResultSetResponse:
 155         return "deleteResultSetResponse";
 156     case Z_APDU_scanRequest:
 157         return "scanRequest";
 158     case Z_APDU_scanResponse:
 159         return "scanResponse";
 160     case Z_APDU_sortRequest:
 161         return "sortRequest";
 162     case Z_APDU_sortResponse:
 163         return "sortResponse";
 164     case Z_APDU_extendedServicesRequest:
 165         return "extendedServicesRequest";
 166     case Z_APDU_extendedServicesResponse:
 167         return "extendedServicesResponse";
 168     case Z_APDU_close:
 169         return "close";
 170     }
 171     return "other";
 172 }
 173
 174 static const char *gdu_name(Z_GDU *gdu)
 175 {
 176     switch(gdu->which)
 177     {
 178     case Z_GDU_Z3950:
 179         return apdu_name(gdu->u.z3950);
 180     case Z_GDU_HTTP_Request:
 181         return "HTTP Request";
 182     case Z_GDU_HTTP_Response:
 183         return "HTTP Response";
 184     }
 185     return "Unknown request/response";
 186 }
 187
 188 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
 189                      ISocketObservable *the_socket_observable,
 190                      Yaz_Proxy *parent)
 191     :
 192     Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 193 {
 194     m_PDU_Observable = the_PDU_Observable;
 195     m_socket_observable = the_socket_observable;
 196     m_client = 0;
 197     m_parent = parent;
 198     m_clientPool = 0;
 199     m_seqno = 1;
 200     m_keepalive_limit_bw = 500000;
 201     m_keepalive_limit_pdu = 1000;
 202     m_proxyTarget = 0;
 203     m_default_target = 0;
 204     m_proxy_negotiation_charset = 0;
 205     m_proxy_negotiation_lang = 0;
 206     m_charset_converter = new Yaz_CharsetConverter;
 207     m_max_clients = 150;
 208     m_log_mask = 0;
 209     m_seed = time(0);
 210     m_client_idletime = 600;
 211     m_target_idletime = 600;
 212     m_optimize = xstrdup ("1");
 213     strcpy(m_session_str, "0 ");
 214     m_session_no = 0;
 215     m_bytes_sent = 0;
 216     m_bytes_recv = 0;
 217     m_bw_hold_PDU = 0;
 218     m_bw_max = 0;
 219     m_pdu_max = 0;
 220     m_timeout_mode = timeout_normal;
 221     m_timeout_gdu = 0;
 222     m_max_record_retrieve = 0;
 223     m_reconfig_flag = 0;
 224     m_config_fname = 0;
 225     m_request_no = 0;
 226     m_flag_invalid_session = 0;
 227     m_referenceId = 0;
 228     m_referenceId_mem = nmem_create();
 229     m_config = 0;
 230     m_marcxml_mode = none;
 231     m_stylesheet_xsp = 0;
 232     m_stylesheet_nprl = 0;
 233     m_stylesheet_apdu = 0;
 234     m_s2z_stylesheet = 0;
 235     m_s2z_database = 0;
 236     m_schema = 0;
 237     m_backend_type = 0;
 238     m_backend_charset = 0;
 239     m_frontend_type = 0;
 240     m_initRequest_apdu = 0;
 241     m_initRequest_mem = 0;
 242     m_initRequest_preferredMessageSize = 0;
 243     m_initRequest_maximumRecordSize = 0;
 244     m_initRequest_options = 0;
 245     m_initRequest_version = 0;
 246     m_initRequest_oi_negotiation_charsets = 0;
 247     m_initRequest_oi_negotiation_num_charsets = 0;
 248     m_initRequest_oi_negotiation_langs = 0;
 249     m_initRequest_oi_negotiation_num_langs = 0;
 250     m_initRequest_oi_negotiation_selected = 0;
 251     m_apdu_invalid_session = 0;
 252     m_mem_invalid_session = 0;
 253     m_s2z_odr_init = 0;
 254     m_s2z_odr_search = 0;
 255     m_s2z_init_apdu = 0;
 256     m_s2z_search_apdu = 0;
 257     m_s2z_present_apdu = 0;
 258     m_http_keepalive = 0;
 259     m_http_version = 0;
 260     m_soap_ns = 0;
 261     m_s2z_packing = Z_SRW_recordPacking_string;
 262 #if HAVE_GETTIMEOFDAY
 263     m_time_tv = xmalloc(sizeof(struct timeval));
 264     struct timeval *tv = (struct timeval *) m_time_tv;
 265     tv->tv_sec = 0;
 266     tv->tv_usec = 0;
 267 #else
 268     m_time_tv = 0;
 269 #endif
 270     m_usemarcon_ini_stage1 = 0;
 271     m_usemarcon_ini_stage2 = 0;
 272     m_usemarcon = new Yaz_usemarcon();
 273     if (!m_parent)
 274         low_socket_open();
 275     m_my_thread = 0;
 276     m_ref_count = 1;
 277     m_peername = 0;
 278 }
 279
 280 void Yaz_Proxy::inc_ref()
 281 {
 282 //    m_ref_count++;
 283 }
 284
 285 Yaz_Proxy::~Yaz_Proxy()
 286 {
 287     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 288             m_bytes_sent, m_bytes_recv);
 289     nmem_destroy(m_initRequest_mem);
 290     nmem_destroy(m_mem_invalid_session);
 291     nmem_destroy(m_referenceId_mem);
 292
 293     xfree(m_proxyTarget);
 294     xfree(m_default_target);
 295     xfree(m_proxy_negotiation_charset);
 296     xfree(m_proxy_negotiation_lang);
 297     delete m_charset_converter;
 298     xfree(m_optimize);
 299
 300 #if HAVE_XSLT
 301     if (m_stylesheet_xsp)
 302         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 303 #endif
 304     xfree (m_time_tv);
 305
 306     xfree (m_peername);
 307     xfree (m_schema);
 308     xfree (m_backend_type);
 309     xfree (m_backend_charset);
 310     xfree (m_usemarcon_ini_stage1);
 311     xfree (m_usemarcon_ini_stage2);
 312     delete m_usemarcon;
 313     if (m_s2z_odr_init)
 314         odr_destroy(m_s2z_odr_init);
 315     if (m_s2z_odr_search)
 316         odr_destroy(m_s2z_odr_search);
 317     if (!m_parent)
 318         low_socket_close();
 319     if (!m_parent)
 320         delete m_my_thread;
 321     delete m_config;
 322 }
 323
 324 void Yaz_Proxy::set_debug_mode(int mode)
 325 {
 326     m_debug_mode = mode;
 327 }
 328
 329 int Yaz_Proxy::set_config(const char *config)
 330 {
 331     delete m_config;
 332     m_config = new Yaz_ProxyConfig();
 333     xfree(m_config_fname);
 334     m_config_fname = xstrdup(config);
 335     int r = m_config->read_xml(config);
 336     if (!r)
 337         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 338     return r;
 339 }
 340
 341 void Yaz_Proxy::set_default_target(const char *target)
 342 {
 343     xfree (m_default_target);
 344     m_default_target = 0;
 345     if (target)
 346         m_default_target = (char *) xstrdup (target);
 347 }
 348
 349 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang)
 350 {
 351     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
 352         "language to '%s'", m_session_str, charset?charset:"none",
 353         lang?lang:"none");
 354     xfree (m_proxy_negotiation_charset);
 355     xfree (m_proxy_negotiation_lang);
 356     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 357     if (charset)
 358         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 359     if (lang)
 360         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 361 }
 362
 363 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 364 {
 365     if (m_parent)
 366         return m_parent->check_reconfigure();
 367
 368     Yaz_ProxyConfig *cfg = m_config;
 369     if (m_reconfig_flag)
 370     {
 371         yaz_log(YLOG_LOG, "reconfigure");
 372         yaz_log_reopen();
 373         if (m_config_fname && cfg)
 374         {
 375             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 376             int r = cfg->read_xml(m_config_fname);
 377             if (r)
 378                 yaz_log(YLOG_WARN, "reconfigure failed");
 379             else
 380             {
 381                 m_log_mask = 0;
 382                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 383             }
 384         }
 385         else
 386             yaz_log(YLOG_LOG, "reconfigure");
 387         m_reconfig_flag = 0;
 388     }
 389     return cfg;
 390 }
 391
 392 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 393                                         *the_PDU_Observable, int fd)
 394 {
 395     check_reconfigure();
 396     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 397                                          m_socket_observable, this);
 398     new_proxy->m_config = 0;
 399     new_proxy->m_config_fname = 0;
 400     new_proxy->timeout(m_client_idletime);
 401     new_proxy->m_target_idletime = m_target_idletime;
 402     new_proxy->set_default_target(m_default_target);
 403     new_proxy->m_max_clients = m_max_clients;
 404     new_proxy->m_log_mask = m_log_mask;
 405     new_proxy->set_APDU_log(get_APDU_log());
 406     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 407         new_proxy->set_APDU_yazlog(1);
 408     else
 409         new_proxy->set_APDU_yazlog(0);
 410     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 411     m_session_no++;
 412     new_proxy->m_peername = xstrdup(the_PDU_Observable->getpeername());
 413     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 414              new_proxy->m_peername);
 415     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 416         m_proxy_negotiation_lang);
 417     // create thread object the first time we get an incoming connection
 418     if (!m_my_thread)
 419         m_my_thread = new Msg_Thread(m_socket_observable);
 420     new_proxy->m_my_thread = m_my_thread;
 421     return new_proxy;
 422 }
 423
 424 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 425 {
 426     int oid[OID_SIZE];
 427     Z_OtherInformationUnit *oi;
 428     struct oident ent;
 429     ent.proto = PROTO_Z3950;
 430     ent.oclass = CLASS_USERINFO;
 431     ent.value = (oid_value) VAL_COOKIE;
 432     assert (oid_ent_to_oid (&ent, oid));
 433
 434     if (oid_ent_to_oid (&ent, oid) &&
 435         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 436         oi->which == Z_OtherInfo_characterInfo)
 437         return oi->information.characterInfo;
 438     return 0;
 439 }
 440 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 441 {
 442     int oid[OID_SIZE];
 443     Z_OtherInformationUnit *oi;
 444     struct oident ent;
 445     ent.proto = PROTO_Z3950;
 446     ent.oclass = CLASS_USERINFO;
 447     ent.value = (oid_value) VAL_PROXY;
 448     if (oid_ent_to_oid (&ent, oid) &&
 449         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 450         oi->which == Z_OtherInfo_characterInfo)
 451         return oi->information.characterInfo;
 452     return 0;
 453 }
 454 const char *Yaz_Proxy::load_balance(const char **url)
 455 {
 456     int zurl_in_use[MAX_ZURL_PLEX];
 457     int zurl_in_spare[MAX_ZURL_PLEX];
 458     Yaz_ProxyClient *c;
 459     int i;
 460
 461     for (i = 0; i<MAX_ZURL_PLEX; i++)
 462     {
 463         zurl_in_use[i] = 0;
 464         zurl_in_spare[i] = 0;
 465     }
 466     for (c = m_parent->m_clientPool; c; c = c->m_next)
 467     {
 468         for (i = 0; url[i]; i++)
 469             if (!strcmp(url[i], c->get_hostname()))
 470             {
 471                 zurl_in_use[i]++;
 472                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 473                     zurl_in_spare[i]++;
 474             }
 475     }
 476     int min_use = 100000;
 477     int spare_for_min = 0;
 478     int max_spare = 0;
 479     const char *ret_min = 0;
 480     const char *ret_spare = 0;
 481     for (i = 0; url[i]; i++)
 482     {
 483         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 484                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 485         if (min_use > zurl_in_use[i])
 486         {
 487             ret_min = url[i];
 488             min_use = zurl_in_use[i];
 489             spare_for_min = zurl_in_spare[i];
 490         }
 491         if (max_spare < zurl_in_spare[i])
 492         {
 493             ret_spare = url[i];
 494             max_spare = zurl_in_spare[i];
 495         }
 496     }
 497     return ret_min;
 498 }
 499
 500 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 501                                        const char *proxy_host)
 502 {
 503     assert (m_parent);
 504     Yaz_Proxy *parent = m_parent;
 505     Yaz_ProxyClient *c = m_client;
 506
 507     if (!m_proxyTarget)
 508     {
 509         const char *url[MAX_ZURL_PLEX];
 510         Yaz_ProxyConfig *cfg = check_reconfigure();
 511         if (proxy_host)
 512         {
 513             if (parent && parent->m_debug_mode)
 514             {
 515                 // only to be enabled for debugging...
 516                 if (!strcmp(proxy_host, "stop"))
 517                     exit(0);
 518             }
 519             xfree(m_default_target);
 520             m_default_target = xstrdup(proxy_host);
 521         }
 522         proxy_host = m_default_target;
 523         int client_idletime = -1;
 524         const char *cql2rpn_fname = 0;
 525         const char *negotiation_charset = 0;
 526         const char *negotiation_lang = 0;
 527         const char *query_charset = 0;
 528         url[0] = m_default_target;
 529         url[1] = 0;
 530         if (cfg)
 531         {
 532             int pre_init = 0;
 533             cfg->get_target_info(proxy_host, url, &m_bw_max,
 534                                  &m_pdu_max, &m_max_record_retrieve,
 535                                  &m_target_idletime, &client_idletime,
 536                                  &parent->m_max_clients,
 537                                  &m_keepalive_limit_bw,
 538                                  &m_keepalive_limit_pdu,
 539                                  &pre_init,
 540                                  &cql2rpn_fname,
 541                                  &negotiation_charset,
 542                                  &negotiation_lang,
 543                                  &query_charset);
 544         }
 545         if (client_idletime != -1)
 546         {
 547             m_client_idletime = client_idletime;
 548             timeout(m_client_idletime);
 549         }
 550         if (cql2rpn_fname)
 551             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 552         if (negotiation_charset || negotiation_lang)
 553         {
 554             set_proxy_negotiation(negotiation_charset,
 555                 negotiation_lang);
 556         }
 557         m_charset_converter->set_target_query_charset(query_charset);
 558         if (!url[0])
 559         {
 560             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 561             return 0;
 562         }
 563         // we don't handle multiplexing for cookie session, so we just
 564         // pick the first one in this case (anonymous users will be able
 565         // to use any backend)
 566         if (cookie && *cookie)
 567             m_proxyTarget = (char*) xstrdup(url[0]);
 568         else
 569             m_proxyTarget = (char*) xstrdup(load_balance(url));
 570     }
 571     if (cookie && *cookie)
 572     {   // search in sessions with a cookie
 573         for (c = parent->m_clientPool; c; c = c->m_next)
 574         {
 575             assert (c->m_prev);
 576             assert (*c->m_prev == c);
 577             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 578                 !strcmp(m_proxyTarget, c->get_hostname()))
 579             {
 580                 // Found it in cache
 581                 // The following handles "cancel"
 582                 // If connection is busy (waiting for PDU) and
 583                 // we have an initRequest we can safely do re-open
 584                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 585                 {
 586                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 587                              c->get_hostname());
 588                     c->close();
 589                     c->m_init_flag = 0;
 590
 591                     c->m_last_ok = 0;
 592                     c->m_cache.clear();
 593                     c->m_last_resultCount = 0;
 594                     c->m_sr_transform = 0;
 595                     c->m_waiting = 0;
 596                     c->m_resultSetStartPoint = 0;
 597                     c->m_target_idletime = m_target_idletime;
 598                     if (c->client(m_proxyTarget))
 599                     {
 600                         delete c;
 601                         return 0;
 602                     }
 603                     c->timeout(30);
 604                 }
 605                 c->m_seqno = parent->m_seqno;
 606                 if (c->m_server && c->m_server != this)
 607                     c->m_server->m_client = 0;
 608                 c->m_server = this;
 609                 (parent->m_seqno)++;
 610                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 611                 return c;
 612             }
 613         }
 614     }
 615     else if (!c &&
 616             apdu->which == Z_APDU_initRequest &&
 617             apdu->u.initRequest->idAuthentication == 0 &&
 618             !ODR_MASK_GET(apdu->u.initRequest->options, Z_Options_negotiationModel))
 619     {
 620         // anonymous sessions without cookie.
 621         // if authentication is set it is NOT anonymous se we can't share them.
 622         // If charset and lang negotiation is use it is NOT anonymous session too.
 623         for (c = parent->m_clientPool; c; c = c->m_next)
 624         {
 625             assert(c->m_prev);
 626             assert(*c->m_prev == c);
 627             if (c->m_server == 0 && c->m_cookie == 0 &&
 628                 c->m_waiting == 0 &&
 629                 !strcmp(m_proxyTarget, c->get_hostname()))
 630             {
 631                 // found it in cache
 632                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 633                          m_session_str, parent->m_seqno, c->get_hostname());
 634
 635                 c->m_seqno = parent->m_seqno;
 636                 assert(c->m_server == 0);
 637                 c->m_server = this;
 638
 639                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 640                     c->set_APDU_yazlog(1);
 641                 else
 642                     c->set_APDU_yazlog(0);
 643
 644                 (parent->m_seqno)++;
 645
 646                 parent->pre_init();
 647
 648                 return c;
 649             }
 650         }
 651     }
 652     if (!m_client)
 653     {
 654         if (apdu->which != Z_APDU_initRequest)
 655         {
 656             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 657             return 0;
 658         }
 659         Z_InitRequest *initRequest = apdu->u.initRequest;
 660
 661         if (initRequest->idAuthentication)
 662         {
 663             // the client uses authentication. We set the keepalive PDU
 664             // to 0 so we don't cache it in releaseClient
 665             m_keepalive_limit_pdu = 0;
 666         }
 667         // go through list of clients - and find the lowest/oldest one.
 668         Yaz_ProxyClient *c_min = 0;
 669         int min_seq = -1;
 670         int no_of_clients = 0;
 671         if (parent->m_clientPool)
 672             yaz_log (YLOG_DEBUG, "Existing sessions");
 673         for (c = parent->m_clientPool; c; c = c->m_next)
 674         {
 675             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 676                                c->m_waiting, c->get_hostname(),
 677                                c->m_cookie ? c->m_cookie : "");
 678             no_of_clients++;
 679             if (min_seq < 0 || c->m_seqno < min_seq)
 680             {
 681                 min_seq = c->m_seqno;
 682                 c_min = c;
 683             }
 684         }
 685         if (no_of_clients >= parent->m_max_clients)
 686         {
 687             c = c_min;
 688             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 689             {
 690                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 691                          m_session_str, parent->m_max_clients, c->m_seqno);
 692                 if (c->m_server && c->m_server != this)
 693                     delete c->m_server;
 694                 c->m_server = 0;
 695             }
 696             else
 697             {
 698                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 699                          m_session_str, parent->m_max_clients,
 700                          c->m_seqno, parent->m_seqno, c->get_hostname());
 701                 xfree (c->m_cookie);
 702                 c->m_cookie = 0;
 703                 if (cookie)
 704                     c->m_cookie = xstrdup(cookie);
 705                 c->m_seqno = parent->m_seqno;
 706                 if (c->m_server && c->m_server != this)
 707                 {
 708                     c->m_server->m_client = 0;
 709                     delete c->m_server;
 710                 }
 711                 (parent->m_seqno)++;
 712                 c->m_target_idletime = m_target_idletime;
 713                 c->timeout(m_target_idletime);
 714
 715                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 716                     c->set_APDU_yazlog(1);
 717                 else
 718                     c->set_APDU_yazlog(0);
 719
 720                 return c;
 721             }
 722         }
 723         else
 724         {
 725             yaz_log (YLOG_LOG, "%sNEW %d %s",
 726                      m_session_str, parent->m_seqno, m_proxyTarget);
 727             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 728             c->m_next = parent->m_clientPool;
 729             if (c->m_next)
 730                 c->m_next->m_prev = &c->m_next;
 731             parent->m_clientPool = c;
 732             c->m_prev = &parent->m_clientPool;
 733         }
 734
 735         xfree (c->m_cookie);
 736         c->m_cookie = 0;
 737         if (cookie)
 738             c->m_cookie = xstrdup(cookie);
 739
 740         c->m_seqno = parent->m_seqno;
 741         c->m_init_flag = 0;
 742         c->m_last_resultCount = 0;
 743         c->m_last_ok = 0;
 744         c->m_cache.clear();
 745         c->m_sr_transform = 0;
 746         c->m_waiting = 0;
 747         c->m_resultSetStartPoint = 0;
 748         (parent->m_seqno)++;
 749         if (c->client(m_proxyTarget))
 750         {
 751             delete c;
 752             return 0;
 753         }
 754         c->m_target_idletime = m_target_idletime;
 755         c->timeout(30);
 756
 757         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 758             c->set_APDU_yazlog(1);
 759         else
 760             c->set_APDU_yazlog(0);
 761     }
 762     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 763     return c;
 764 }
 765
 766 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 767 {
 768     int i;
 769     for (i = 0; i<num; i++)
 770     {
 771         oident *ent;
 772         Z_DefaultDiagFormat *r;
 773         Z_DiagRec *p = pp[i];
 774         if (p->which != Z_DiagRec_defaultFormat)
 775         {
 776             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 777             return;
 778         }
 779         else
 780             r = p->u.defaultFormat;
 781         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 782             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 783             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 784         switch (r->which)
 785         {
 786         case Z_DefaultDiagFormat_v2Addinfo:
 787             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 788                     m_session_str,
 789                     *r->condition, diagbib1_str(*r->condition),
 790                     r->u.v2Addinfo);
 791             break;
 792         case Z_DefaultDiagFormat_v3Addinfo:
 793             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 794                     m_session_str,
 795                     *r->condition, diagbib1_str(*r->condition),
 796                     r->u.v3Addinfo);
 797             break;
 798         }
 799     }
 800 }
 801
 802 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 803 {
 804     if (!m_stylesheet_xsp || p->num_records <= 0)
 805         return 0;  /* no XSLT to be done ... */
 806
 807     m_stylesheet_offset = 0;
 808     m_stylesheet_nprl = p;
 809     m_stylesheet_apdu = apdu;
 810     m_timeout_mode = timeout_xsl;
 811
 812     timeout(0);
 813     return 1;
 814 }
 815
 816 void Yaz_Proxy::convert_xsl_delay()
 817 {
 818 #if HAVE_XSLT
 819     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 820     if (npr->which == Z_NamePlusRecord_databaseRecord)
 821     {
 822         Z_External *r = npr->u.databaseRecord;
 823         if (r->which == Z_External_octet)
 824         {
 825 #if 0
 826             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 827 #endif
 828             xmlDocPtr res, doc = xmlParseMemory(
 829                 (char*) r->u.octet_aligned->buf,
 830                 r->u.octet_aligned->len);
 831
 832
 833             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 834                     m_session_str, m_stylesheet_offset);
 835             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 836                                       doc, 0);
 837
 838             if (res)
 839             {
 840                 xmlChar *out_buf;
 841                 int out_len;
 842                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 843
 844                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 845                     u.databaseRecord =
 846                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 847                                  (char*) out_buf, out_len);
 848                 xmlFree(out_buf);
 849                 xmlFreeDoc(res);
 850             }
 851
 852             xmlFreeDoc(doc);
 853         }
 854     }
 855 #endif
 856     m_stylesheet_offset++;
 857     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 858     {
 859         m_timeout_mode = timeout_normal;
 860         m_stylesheet_nprl = 0;
 861 #if HAVE_XSLT
 862         if (m_stylesheet_xsp)
 863             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 864 #endif
 865         m_stylesheet_xsp = 0;
 866         timeout(m_client_idletime);
 867         send_PDU_convert(m_stylesheet_apdu);
 868     }
 869     else
 870         timeout(0);
 871 }
 872
 873 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 874 {
 875     if (m_frontend_type != VAL_NONE)
 876     {
 877         int i;
 878         for (i = 0; i < p->num_records; i++)
 879         {
 880             Z_NamePlusRecord *npr = p->records[i];
 881             if (npr->which == Z_NamePlusRecord_databaseRecord)
 882             {
 883                 Z_External *r = npr->u.databaseRecord;
 884                 if (r->which == Z_External_octet)
 885                 {
 886 #if HAVE_USEMARCON
 887                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 888                     {
 889                         if (!m_usemarcon->m_stage1)
 890                         {
 891                             m_usemarcon->m_stage1 = new CDetails();
 892                         }
 893                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 894                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 895                         int res = m_usemarcon->m_stage1->Start();
 896                         if (res == 0)
 897                         {
 898                             char *converted;
 899                             int convlen;
 900                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 901                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 902                             {
 903                                 if (!m_usemarcon->m_stage2)
 904                                 {
 905                                     m_usemarcon->m_stage2 = new CDetails();
 906                                 }
 907                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 908                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 909                                 res = m_usemarcon->m_stage2->Start();
 910                                 if (res == 0)
 911                                 {
 912                                     free(converted);
 913                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 914                                 }
 915                                 else
 916                                 {
 917                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 918                                 }
 919                             }
 920                             npr->u.databaseRecord =
 921                                 z_ext_record(odr_encode(),
 922                                              m_frontend_type,
 923                                              converted,
 924                                              strlen(converted));
 925                             free(converted);
 926                         }
 927                         else
 928                         {
 929                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 930                         }
 931                         continue;
 932                     }
 933 #endif
 934 /* HAVE_USEMARCON */
 935                     npr->u.databaseRecord =
 936                         z_ext_record(odr_encode(),
 937                                      m_frontend_type,
 938                                      (char*) r->u.octet_aligned->buf,
 939                                      r->u.octet_aligned->len);
 940                 }
 941             }
 942         }
 943     }
 944 }
 945
 946 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
 947                                         const char *backend_charset)
 948 {
 949     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
 950     int sel =   m_charset_converter->get_client_charset_selected();
 951     const char *client_record_charset =
 952         m_charset_converter->get_client_query_charset();
 953     if (sel && backend_charset && client_record_charset &&
 954         strcmp(backend_charset, client_record_charset))
 955     {
 956         int i;
 957         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
 958                                         backend_charset);
 959         yaz_marc_t mt = yaz_marc_create();
 960         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
 961         yaz_marc_iconv(mt, cd);
 962         for (i = 0; i < p->num_records; i++)
 963         {
 964             Z_NamePlusRecord *npr = p->records[i];
 965             if (npr->which == Z_NamePlusRecord_databaseRecord)
 966             {
 967                 Z_External *r = npr->u.databaseRecord;
 968                 oident *ent = oid_getentbyoid(r->direct_reference);
 969                 if (!ent || ent->value == VAL_NONE)
 970                     continue;
 971
 972                 if (ent->value == VAL_SUTRS)
 973                 {
 974                     WRBUF w = wrbuf_alloc();
 975
 976                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
 977                                       r->u.octet_aligned->len);
 978                     npr->u.databaseRecord =
 979                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
 980                                      wrbuf_len(w));
 981                     wrbuf_free(w, 1);
 982                 }
 983                 else if (ent->value == VAL_TEXT_XML)
 984                 {
 985                     ;
 986                 }
 987                 else if (r->which == Z_External_octet)
 988                 {
 989                     int rlen;
 990                     char *result;
 991                     if (yaz_marc_decode_buf(mt,
 992                                             (char*) r->u.octet_aligned->buf,
 993                                             r->u.octet_aligned->len,
 994                                             &result, &rlen))
 995                     {
 996                         npr->u.databaseRecord =
 997                             z_ext_record(odr_encode(), ent->value, result, rlen);
 998                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
 999                                 m_session_str);
1000                     }
1001                 }
1002             }
1003         }
1004         if (cd)
1005             yaz_iconv_close(cd);
1006         yaz_marc_destroy(mt);
1007     }
1008     else
1009     {
1010         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
1011     }
1012 }
1013
1014 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1015                                    const char *backend_charset)
1016 {
1017     int i;
1018     if (!backend_charset)
1019         backend_charset = "MARC-8";
1020     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1021     yaz_marc_t mt = yaz_marc_create();
1022     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1023     yaz_marc_iconv(mt, cd);
1024     for (i = 0; i < p->num_records; i++)
1025     {
1026         Z_NamePlusRecord *npr = p->records[i];
1027         if (npr->which == Z_NamePlusRecord_databaseRecord)
1028         {
1029             Z_External *r = npr->u.databaseRecord;
1030             if (r->which == Z_External_OPAC)
1031             {
1032                 WRBUF w = wrbuf_alloc();
1033
1034                 yaz_display_OPAC(w, r->u.opac, 0);
1035                 npr->u.databaseRecord = z_ext_record(
1036                     odr_encode(), VAL_TEXT_XML,
1037                     wrbuf_buf(w), wrbuf_len(w)
1038                     );
1039                 wrbuf_free(w, 1);
1040             }
1041             else if (r->which == Z_External_octet)
1042             {
1043                 int rlen;
1044                 char *result;
1045                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1046                                         r->u.octet_aligned->len,
1047                                         &result, &rlen))
1048                 {
1049                     npr->u.databaseRecord =
1050                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
1051                 }
1052             }
1053         }
1054     }
1055     if (cd)
1056         yaz_iconv_close(cd);
1057     yaz_marc_destroy(mt);
1058 }
1059
1060 void Yaz_Proxy::logtime()
1061 {
1062 #if HAVE_GETTIMEOFDAY
1063     struct timeval *tv = (struct timeval*) m_time_tv;
1064     if (tv->tv_sec)
1065     {
1066         struct timeval tv1;
1067         gettimeofday(&tv1, 0);
1068         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1069             (tv1.tv_usec - tv->tv_usec);
1070         if (diff >= 0)
1071             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1072                     diff/1000000, (diff/1000)%1000);
1073     }
1074     tv->tv_sec = 0;
1075     tv->tv_usec = 0;
1076 #endif
1077 }
1078
1079 int Yaz_Proxy::send_http_response(int code)
1080 {
1081     ODR o = odr_encode();
1082     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1083     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1084     if (m_http_version)
1085         hres->version = odr_strdup(o, m_http_version);
1086     if (m_http_keepalive)
1087         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1088     else
1089         timeout(0);
1090
1091     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1092     {
1093         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1094                  gdu_name(gdu));
1095     }
1096     int len;
1097     int r = send_GDU(gdu, &len);
1098     m_bytes_sent += len;
1099     m_bw_stat.add_bytes(len);
1100     logtime();
1101     return r;
1102 }
1103
1104 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
1105 {
1106     ODR o = odr_encode();
1107     const char *ctype = "text/xml";
1108     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
1109     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1110     if (m_http_version)
1111         hres->version = odr_strdup(o, m_http_version);
1112     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1113     if (m_http_keepalive)
1114         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1115     else
1116         timeout(0);
1117
1118     static Z_SOAP_Handler soap_handlers[2] = {
1119 #if HAVE_XSLT
1120         {"http://www.loc.gov/zing/srw/", 0,
1121          (Z_SOAP_fun) yaz_srw_codec},
1122 #endif
1123         {0, 0, 0}
1124     };
1125
1126     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1127     soap_package->which = Z_SOAP_generic;
1128     soap_package->u.generic =
1129         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1130     soap_package->u.generic->no = 0;
1131     soap_package->u.generic->ns = soap_handlers[0].ns;
1132     soap_package->u.generic->p = (void *) srw_pdu;
1133     soap_package->ns = m_soap_ns;
1134     z_soap_codec_enc_xsl(o, &soap_package,
1135                          &hres->content_buf, &hres->content_len,
1136                          soap_handlers, 0, m_s2z_stylesheet);
1137     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1138     {
1139         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1140                  gdu_name(gdu));
1141     }
1142     int len;
1143     int r = send_GDU(gdu, &len);
1144     m_bytes_sent += len;
1145     m_bw_stat.add_bytes(len);
1146     logtime();
1147     return r;
1148 }
1149
1150 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1151 {
1152     ODR o = odr_encode();
1153     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1154     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1155
1156     srw_res->num_diagnostics = 1;
1157     srw_res->diagnostics = (Z_SRW_diagnostic *)
1158         odr_malloc(o, sizeof(*srw_res->diagnostics));
1159     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
1160     return send_srw_response(srw_pdu);
1161 }
1162
1163 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1164                              Z_DefaultDiagFormat *ddf)
1165 {
1166     int bib1_code = *ddf->condition;
1167     if (bib1_code == 109)
1168         return 404;
1169     srw_res->num_diagnostics = 1;
1170     srw_res->diagnostics = (Z_SRW_diagnostic *)
1171         odr_malloc(o, sizeof(*srw_res->diagnostics));
1172     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1173                           yaz_diag_bib1_to_srw(*ddf->condition),
1174                           ddf->u.v2Addinfo);
1175     return 0;
1176 }
1177
1178 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1179 {
1180     ODR o = odr_encode();
1181     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1182     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1183
1184     srw_res->numberOfRecords = odr_intdup (o, hits);
1185     if (records && records->which == Z_Records_DBOSD)
1186     {
1187         srw_res->num_records =
1188             records->u.databaseOrSurDiagnostics->num_records;
1189         int i;
1190         srw_res->records = (Z_SRW_record *)
1191             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1192         for (i = 0; i < srw_res->num_records; i++)
1193         {
1194             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1195             if (npr->which != Z_NamePlusRecord_databaseRecord)
1196             {
1197                 srw_res->records[i].recordSchema = "diagnostic";
1198                 srw_res->records[i].recordPacking = m_s2z_packing;
1199                 srw_res->records[i].recordData_buf = "67";
1200                 srw_res->records[i].recordData_len = 2;
1201                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1202                 continue;
1203             }
1204             Z_External *r = npr->u.databaseRecord;
1205             oident *ent = oid_getentbyoid(r->direct_reference);
1206             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1207             {
1208                 srw_res->records[i].recordSchema = m_schema;
1209                 srw_res->records[i].recordPacking = m_s2z_packing;
1210                 srw_res->records[i].recordData_buf = (char*)
1211                     r->u.octet_aligned->buf;
1212                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1213                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1214             }
1215             else
1216             {
1217                 srw_res->records[i].recordSchema = "diagnostic";
1218                 srw_res->records[i].recordPacking = m_s2z_packing;
1219                 srw_res->records[i].recordData_buf = "67";
1220                 srw_res->records[i].recordData_len = 2;
1221                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1222             }
1223         }
1224     }
1225     if (records && records->which == Z_Records_NSD)
1226     {
1227         int http_code;
1228         http_code = z_to_srw_diag(odr_encode(), srw_res,
1229                                    records->u.nonSurrogateDiagnostic);
1230         if (http_code)
1231             return send_http_response(http_code);
1232     }
1233     return send_srw_response(srw_pdu);
1234
1235 }
1236
1237 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1238                                         int num_diagnostics)
1239 {
1240     Yaz_ProxyConfig *cfg = check_reconfigure();
1241     if (cfg)
1242     {
1243         int len;
1244         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1245                                        m_s2z_database, &len);
1246         if (b)
1247         {
1248             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1249             Z_SRW_explainResponse *er = res->u.explain_response;
1250
1251             er->record.recordData_buf = b;
1252             er->record.recordData_len = len;
1253             er->record.recordPacking = m_s2z_packing;
1254             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1255
1256             er->diagnostics = diagnostics;
1257             er->num_diagnostics = num_diagnostics;
1258             return send_srw_response(res);
1259         }
1260     }
1261     return send_http_response(404);
1262 }
1263
1264 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1265 {
1266     if (m_http_version)
1267     {
1268         if (apdu->which == Z_APDU_initResponse)
1269         {
1270             Z_InitResponse *res = apdu->u.initResponse;
1271             if (*res->result == 0)
1272             {
1273                 send_to_srw_client_error(3, 0);
1274             }
1275             else if (!m_s2z_search_apdu)
1276             {
1277                 send_srw_explain_response(0, 0);
1278             }
1279             else
1280             {
1281                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1282             }
1283         }
1284         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1285         {
1286             m_s2z_search_apdu = 0;
1287             Z_SearchResponse *res = apdu->u.searchResponse;
1288             m_s2z_hit_count = *res->resultCount;
1289             if (res->records && res->records->which == Z_Records_NSD)
1290             {
1291                 send_to_srw_client_ok(0, res->records, 1);
1292             }
1293             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1294             {
1295                 // adjust
1296                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1297
1298                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1299                 {
1300                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1301                         > m_s2z_hit_count)
1302                         *pr->numberOfRecordsRequested =
1303                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1304                 }
1305                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1306             }
1307             else
1308             {
1309                 m_s2z_present_apdu = 0;
1310                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1311             }
1312         }
1313         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1314         {
1315             int start =
1316                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1317
1318             m_s2z_present_apdu = 0;
1319             Z_PresentResponse *res = apdu->u.presentResponse;
1320             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1321         }
1322     }
1323     else
1324     {
1325         int len = 0;
1326         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1327             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1328                      apdu_name(apdu));
1329         int r = send_Z_PDU(apdu, &len);
1330         m_bytes_sent += len;
1331         m_bw_stat.add_bytes(len);
1332         logtime();
1333         return r;
1334     }
1335     return 0;
1336 }
1337
1338 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1339 {
1340     int kill_session = 0;
1341     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1342
1343     if (new_id)
1344         *new_id = m_referenceId;
1345
1346     if (apdu->which == Z_APDU_searchResponse)
1347     {
1348         Z_SearchResponse *sr = apdu->u.searchResponse;
1349         Z_Records *p = sr->records;
1350         if (p && p->which == Z_Records_NSD)
1351         {
1352             Z_DiagRec dr, *dr_p = &dr;
1353             dr.which = Z_DiagRec_defaultFormat;
1354             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1355
1356             *sr->searchStatus = 0;
1357             display_diagrecs(&dr_p, 1);
1358         }
1359         else
1360         {
1361             if (p && p->which == Z_Records_DBOSD)
1362             {
1363                 if (m_backend_type
1364 #if HAVE_USEMARCON
1365                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1366 #endif
1367                     )
1368                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1369                 if (m_marcxml_mode == marcxml)
1370                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1371                                        m_backend_charset);
1372                 else
1373                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1374                                             m_backend_charset);
1375                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1376                     return 0;
1377
1378             }
1379             if (sr->resultCount)
1380             {
1381                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1382                         *sr->resultCount);
1383                 if (*sr->resultCount < 0)
1384                 {
1385                     m_flag_invalid_session = 1;
1386                     kill_session = 1;
1387
1388                     *sr->searchStatus = 0;
1389                     sr->records =
1390                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1391                     *sr->resultCount = 0;
1392                 }
1393             }
1394         }
1395     }
1396     else if (apdu->which == Z_APDU_presentResponse)
1397     {
1398         Z_PresentResponse *sr = apdu->u.presentResponse;
1399         Z_Records *p = sr->records;
1400         if (p && p->which == Z_Records_NSD)
1401         {
1402             Z_DiagRec dr, *dr_p = &dr;
1403             dr.which = Z_DiagRec_defaultFormat;
1404             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1405             if (*sr->presentStatus == Z_PresentStatus_success)
1406                 *sr->presentStatus = Z_PresentStatus_failure;
1407             display_diagrecs(&dr_p, 1);
1408         }
1409         if (p && p->which == Z_Records_DBOSD)
1410         {
1411             if (m_backend_type
1412 #if HAVE_USEMARCON
1413                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1414 #endif
1415                 )
1416                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1417             if (m_marcxml_mode == marcxml)
1418                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1419                                    m_backend_charset);
1420             else
1421                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1422                                         m_backend_charset);
1423             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1424                 return 0;
1425         }
1426     }
1427     else if (apdu->which == Z_APDU_initResponse)
1428     {
1429         //Get and check negotiation record
1430         //from init response.
1431         handle_charset_lang_negotiation(apdu);
1432
1433         if (m_initRequest_options)
1434         {
1435             Z_Options *nopt =
1436                 (Odr_bitmask *)odr_malloc(odr_encode(),
1437                                           sizeof(Odr_bitmask));
1438             ODR_MASK_ZERO(nopt);
1439
1440             int i;
1441             for (i = 0; i<24; i++)
1442                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1443                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1444                     ODR_MASK_SET(nopt, i);
1445             apdu->u.initResponse->options = nopt;
1446         }
1447         if (m_initRequest_version)
1448         {
1449             Z_ProtocolVersion *nopt =
1450                 (Odr_bitmask *)odr_malloc(odr_encode(),
1451                                           sizeof(Odr_bitmask));
1452             ODR_MASK_ZERO(nopt);
1453
1454             int i;
1455             for (i = 0; i<8; i++)
1456                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1457                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1458                     ODR_MASK_SET(nopt, i);
1459             apdu->u.initResponse->protocolVersion = nopt;
1460         }
1461         apdu->u.initResponse->preferredMessageSize =
1462             odr_intdup(odr_encode(),
1463                        m_client->m_initResponse_preferredMessageSize >
1464                        m_initRequest_preferredMessageSize ?
1465                        m_initRequest_preferredMessageSize :
1466                        m_client->m_initResponse_preferredMessageSize);
1467         apdu->u.initResponse->maximumRecordSize =
1468             odr_intdup(odr_encode(),
1469                        m_client->m_initResponse_maximumRecordSize >
1470                        m_initRequest_maximumRecordSize ?
1471                        m_initRequest_maximumRecordSize :
1472                        m_client->m_initResponse_maximumRecordSize);
1473     }
1474
1475     int r = send_PDU_convert(apdu);
1476     if (r)
1477         return r;
1478     if (kill_session)
1479     {
1480         delete m_client;
1481         m_client = 0;
1482         m_parent->pre_init();
1483     }
1484     return r;
1485 }
1486
1487 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1488 {
1489     int len = 0;
1490     const char *apdu_name_tmp = apdu_name(apdu);
1491     int r = send_Z_PDU(apdu, &len);
1492     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1493         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1494                  get_session_str(),
1495                  apdu_name_tmp, get_hostname(), len);
1496     m_bytes_sent += len;
1497     return r;
1498 }
1499
1500 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1501 {
1502     if (apdu->which == Z_APDU_presentRequest)
1503     {
1504         Z_PresentRequest *pr = apdu->u.presentRequest;
1505         int toget = *pr->numberOfRecordsRequested;
1506         int start = *pr->resultSetStartPoint;
1507
1508         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1509                 pr->resultSetId, start, toget);
1510
1511         if (*m_parent->m_optimize == '0')
1512             return apdu;
1513
1514         if (!m_client->m_last_resultSetId)
1515         {
1516             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1517             new_apdu->u.presentResponse->records =
1518                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1519                                                pr->resultSetId);
1520             send_to_client(new_apdu);
1521             return 0;
1522         }
1523         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1524         {
1525             if (start+toget-1 > m_client->m_last_resultCount)
1526             {
1527                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1528                 new_apdu->u.presentResponse->records =
1529                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1530                 send_to_client(new_apdu);
1531                 return 0;
1532             }
1533             Z_NamePlusRecordList *npr;
1534 #if 0
1535             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1536                     m_session_str, start, toget, yaz_z3950oid_to_str(
1537                         pr->preferredRecordSyntax, &oclass));
1538 #endif
1539             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1540                                           pr->preferredRecordSyntax,
1541                                           pr->recordComposition))
1542             {
1543                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1544                          m_session_str);
1545                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1546                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1547
1548                 new_apdu->u.presentResponse->numberOfRecordsReturned
1549                     = odr_intdup(odr_encode(), toget);
1550
1551                 new_apdu->u.presentResponse->records = (Z_Records*)
1552                     odr_malloc(odr_encode(), sizeof(Z_Records));
1553                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1554                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1555                 new_apdu->u.presentResponse->nextResultSetPosition =
1556                     odr_intdup(odr_encode(), start+toget);
1557
1558                 send_to_client(new_apdu);
1559                 return 0;
1560             }
1561         }
1562     }
1563
1564     if (apdu->which != Z_APDU_searchRequest)
1565         return apdu;
1566     Z_SearchRequest *sr = apdu->u.searchRequest;
1567     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1568     Yaz_Z_Databases this_databases;
1569
1570     this_databases.set(sr->num_databaseNames, (const char **)
1571                        sr->databaseNames);
1572
1573     this_query->set_Z_Query(sr->query);
1574
1575     char query_str[120];
1576     this_query->print(query_str, sizeof(query_str)-1);
1577     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1578
1579     if (*m_parent->m_optimize != '0' &&
1580         m_client->m_last_ok && m_client->m_last_query &&
1581         m_client->m_last_query->match(this_query) &&
1582         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1583         m_client->m_last_databases.match(this_databases))
1584     {
1585         delete this_query;
1586         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1587             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1588         {
1589             Z_NamePlusRecordList *npr;
1590             int toget = *sr->mediumSetPresentNumber;
1591             Z_RecordComposition *comp = 0;
1592
1593             if (toget > m_client->m_last_resultCount)
1594                 toget = m_client->m_last_resultCount;
1595
1596             if (sr->mediumSetElementSetNames)
1597             {
1598                 comp = (Z_RecordComposition *)
1599                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1600                 comp->which = Z_RecordComp_simple;
1601                 comp->u.simple = sr->mediumSetElementSetNames;
1602             }
1603
1604             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1605                                           sr->preferredRecordSyntax, comp))
1606             {
1607                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1608                          m_session_str);
1609                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1610                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1611                 new_apdu->u.searchResponse->resultCount =
1612                     &m_client->m_last_resultCount;
1613
1614                 new_apdu->u.searchResponse->numberOfRecordsReturned
1615                     = odr_intdup(odr_encode(), toget);
1616
1617                 new_apdu->u.searchResponse->presentStatus =
1618                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1619                 new_apdu->u.searchResponse->records = (Z_Records*)
1620                     odr_malloc(odr_encode(), sizeof(Z_Records));
1621                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1622                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1623                 new_apdu->u.searchResponse->nextResultSetPosition =
1624                     odr_intdup(odr_encode(), toget+1);
1625                 send_to_client(new_apdu);
1626                 return 0;
1627             }
1628             else
1629             {
1630                 // medium Set
1631                 // send present request (medium size)
1632                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1633                          m_session_str);
1634
1635                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1636                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1637                 pr->referenceId = sr->referenceId;
1638                 pr->resultSetId = sr->resultSetName;
1639                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1640                 *pr->numberOfRecordsRequested = toget;
1641                 pr->recordComposition = comp;
1642                 m_client->m_sr_transform = 1;
1643                 return new_apdu;
1644             }
1645         }
1646         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1647             m_client->m_last_resultCount <= 0)
1648         {
1649             // large set. Return pseudo-search response immediately
1650             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1651                      m_session_str);
1652             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1653             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1654             new_apdu->u.searchResponse->resultCount =
1655                 &m_client->m_last_resultCount;
1656             send_to_client(new_apdu);
1657             return 0;
1658         }
1659         else
1660         {
1661             Z_NamePlusRecordList *npr;
1662             int toget = m_client->m_last_resultCount;
1663             Z_RecordComposition *comp = 0;
1664             // small set
1665             // send a present request (small set)
1666
1667             if (sr->smallSetElementSetNames)
1668             {
1669                 comp = (Z_RecordComposition *)
1670                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1671                 comp->which = Z_RecordComp_simple;
1672                 comp->u.simple = sr->smallSetElementSetNames;
1673             }
1674
1675             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1676                                           sr->preferredRecordSyntax, comp))
1677             {
1678                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1679                          m_session_str);
1680                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1681                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1682                 new_apdu->u.searchResponse->resultCount =
1683                     &m_client->m_last_resultCount;
1684
1685                 new_apdu->u.searchResponse->numberOfRecordsReturned
1686                     = odr_intdup(odr_encode(), toget);
1687
1688                 new_apdu->u.searchResponse->presentStatus =
1689                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1690                 new_apdu->u.searchResponse->records = (Z_Records*)
1691                     odr_malloc(odr_encode(), sizeof(Z_Records));
1692                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1693                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1694                 new_apdu->u.searchResponse->nextResultSetPosition =
1695                     odr_intdup(odr_encode(), toget+1);
1696                 send_to_client(new_apdu);
1697                 return 0;
1698             }
1699             else
1700             {
1701                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1702                          m_session_str);
1703                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1704                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1705                 pr->referenceId = sr->referenceId;
1706                 pr->resultSetId = sr->resultSetName;
1707                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1708                 *pr->numberOfRecordsRequested = toget;
1709                 pr->recordComposition = comp;
1710                 m_client->m_sr_transform = 1;
1711                 return new_apdu;
1712             }
1713         }
1714     }
1715     else  // query doesn't match
1716     {
1717         delete m_client->m_last_query;
1718         m_client->m_last_query = this_query;
1719         m_client->m_last_ok = 0;
1720         m_client->m_cache.clear();
1721         m_client->m_resultSetStartPoint = 0;
1722
1723         xfree (m_client->m_last_resultSetId);
1724         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1725
1726         m_client->m_last_databases.set(sr->num_databaseNames,
1727                                        (const char **) sr->databaseNames);
1728     }
1729     return apdu;
1730 }
1731
1732
1733 void Yaz_Proxy::inc_request_no()
1734 {
1735     char *cp = strchr(m_session_str, ' ');
1736     m_request_no++;
1737     if (cp)
1738         sprintf(cp+1, "%d ", m_request_no);
1739 }
1740
1741 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1742 {
1743     inc_request_no();
1744
1745     m_bytes_recv += len;
1746
1747     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1748         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1749                  m_session_str, gdu_name(apdu), len);
1750
1751 #if HAVE_GETTIMEOFDAY
1752     gettimeofday((struct timeval *) m_time_tv, 0);
1753 #endif
1754     m_bw_stat.add_bytes(len);
1755     m_pdu_stat.add_bytes(1);
1756
1757     GDU *gdu = new GDU(apdu);
1758     int qsize = m_in_queue.size();
1759     if (m_timeout_mode != timeout_normal)
1760     {
1761         yaz_log(YLOG_LOG, "%sAdded gdu in queue of size %d", m_session_str,
1762                 qsize);
1763         m_in_queue.enqueue(gdu);
1764     }
1765     else
1766     {
1767         recv_GDU_reduce(gdu);
1768         recv_GDU_more();
1769     }
1770 }
1771
1772 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
1773 {
1774     int bw_total = m_bw_stat.get_total();
1775     int pdu_total = m_pdu_stat.get_total();
1776     int reduce = 0;
1777
1778     assert(m_timeout_mode == timeout_normal);
1779     assert(m_timeout_gdu == 0);
1780
1781     if (m_bw_max)
1782     {
1783         if (bw_total > m_bw_max)
1784         {
1785             reduce = (bw_total/m_bw_max);
1786         }
1787     }
1788     if (m_pdu_max)
1789     {
1790         if (pdu_total > m_pdu_max)
1791         {
1792             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1793             reduce = (reduce > nreduce) ? reduce : nreduce;
1794         }
1795     }
1796     m_http_version = 0;
1797
1798     if (reduce)
1799     {
1800         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1801                 m_session_str, reduce, bw_total, pdu_total,
1802                 m_bw_max, m_pdu_max);
1803
1804         m_timeout_mode = timeout_reduce;
1805         m_timeout_gdu = gdu;
1806         // m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1807         timeout(reduce);       // call us reduce seconds later
1808     }
1809     else
1810         recv_GDU_normal(gdu);
1811 }
1812
1813 void Yaz_Proxy::recv_GDU_more()
1814 {
1815     GDU *g;
1816     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
1817         recv_GDU_reduce(g);
1818 }
1819
1820 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
1821 {
1822     Z_GDU *apdu = gdu->get();
1823     gdu->extract_odr_to(odr_decode());
1824     delete gdu;
1825
1826     if (apdu->which == Z_GDU_Z3950)
1827         handle_incoming_Z_PDU(apdu->u.z3950);
1828     else if (apdu->which == Z_GDU_HTTP_Request)
1829         handle_incoming_HTTP(apdu->u.HTTP_Request);
1830 }
1831
1832 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1833 {
1834     if (m_max_record_retrieve)
1835     {
1836         if (apdu->which == Z_APDU_presentRequest)
1837         {
1838             Z_PresentRequest *pr = apdu->u.presentRequest;
1839             if (pr->numberOfRecordsRequested &&
1840                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1841                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1842         }
1843     }
1844 }
1845
1846 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1847 {
1848     if (apdu->which == Z_APDU_initRequest)
1849     {
1850         yaz_log(YLOG_LOG, "%shandle_charset_lang_negotiation",
1851                 m_session_str);
1852         if (m_initRequest_options &&
1853             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1854             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1855         {
1856             // There is no negotiation proposal from
1857             // client's side. OK. The proxy negotiation
1858             // in use, only.
1859             Z_InitRequest *initRequest = apdu->u.initRequest;
1860             Z_OtherInformation **otherInfo;
1861             Z_OtherInformationUnit *oi;
1862             get_otherInfoAPDU(apdu, &otherInfo);
1863             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1864             if (oi)
1865             {
1866                 ODR_MASK_SET(initRequest->options,
1867                     Z_Options_negotiationModel);
1868                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1869                 oi->information.externallyDefinedInfo =
1870                 yaz_set_proposal_charneg(odr_encode(),
1871                     (const char**)&m_proxy_negotiation_charset,
1872                     m_proxy_negotiation_charset ? 1:0,
1873                     (const char**)&m_proxy_negotiation_lang,
1874                     m_proxy_negotiation_lang ? 1:0,
1875                     1);
1876             }
1877         }
1878         else if (m_initRequest_options &&
1879                  ODR_MASK_GET(m_initRequest_options,
1880                               Z_Options_negotiationModel) &&
1881                  m_charset_converter->get_target_query_charset())
1882         {
1883             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1884                     m_session_str,
1885                     m_charset_converter->get_target_query_charset());
1886             Z_InitRequest *initRequest = apdu->u.initRequest;
1887             Z_CharSetandLanguageNegotiation *negotiation =
1888                 yaz_get_charneg_record (initRequest->otherInfo);
1889             if (negotiation &&
1890                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1891             {
1892                 NMEM nmem = nmem_create();
1893                 char **charsets = 0;
1894                 int num_charsets = 0;
1895                 char **langs = 0;
1896                 int num_langs = 0;
1897                 int selected = 0;
1898                 yaz_get_proposal_charneg (nmem, negotiation,
1899                                           &charsets, &num_charsets,
1900                                           &langs, &num_langs, &selected);
1901                 int i;
1902                 for (i = 0; i<num_charsets; i++)
1903                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
1904                             charsets[i]);
1905                 for (i = 0; i<num_langs; i++)
1906                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
1907                             langs[i]);
1908
1909                 const char *t_charset =
1910                     m_charset_converter->get_target_query_charset();
1911                 // sweep through charsets and pick the first supported
1912                 // conversion
1913                 for (i = 0; i<num_charsets; i++)
1914                 {
1915                     const char *c_charset = charsets[i];
1916                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
1917                         break;
1918                 }
1919                 if (i != num_charsets)
1920                 {
1921                     // got one .. set up ODR for reverse direction
1922                     const char *c_charset = charsets[i];
1923                     odr_set_charset(odr_encode(), c_charset, t_charset);
1924                     m_charset_converter->set_client_query_charset(c_charset);
1925                     m_charset_converter->set_client_charset_selected(selected);
1926                 }
1927                 nmem_destroy(nmem);
1928                 ODR_MASK_CLEAR(m_initRequest_options,
1929                                Z_Options_negotiationModel);
1930                 yaz_del_charneg_record(&initRequest->otherInfo);
1931             }
1932             else
1933             {
1934                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
1935                         m_session_str);
1936             }
1937         }
1938     }
1939     else if (apdu->which == Z_APDU_initResponse)
1940     {
1941         Z_InitResponse *initResponse = apdu->u.initResponse;
1942         Z_OtherInformation **otherInfo;
1943
1944         if (ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel))
1945         {
1946             char *charset = 0;
1947             char *lang = 0;
1948             int selected = 0;
1949
1950             get_otherInfoAPDU(apdu, &otherInfo);
1951
1952             if (!otherInfo && !(*otherInfo))
1953                 return;
1954
1955             Z_CharSetandLanguageNegotiation *charneg =
1956                 yaz_get_charneg_record(*otherInfo);
1957
1958             if (!charneg)
1959                 return;
1960
1961             yaz_get_response_charneg(m_referenceId_mem, charneg,
1962                 &charset, &lang, &selected);
1963
1964             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
1965                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
1966
1967             if (m_initRequest_options &&
1968                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
1969             {
1970                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
1971                     m_session_str);
1972             }
1973             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1974             {
1975                 // negotiation-charset, negotiation-lang
1976                 // elements of config file in use.
1977
1978                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
1979                     m_session_str);
1980
1981                 // clear negotiation option.
1982                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
1983
1984                 // Delete negotiation (charneg-3) entry.
1985                 yaz_del_charneg_record(otherInfo);
1986             }
1987         }
1988         else
1989         {
1990             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1991             {
1992                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
1993                         m_session_str);
1994             }
1995             else if (m_charset_converter->get_client_query_charset())
1996             {
1997                 Z_OtherInformation **otherInfo;
1998                 Z_OtherInformationUnit *oi;
1999                 get_otherInfoAPDU(apdu, &otherInfo);
2000                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2001                 if (oi)
2002                 {
2003                     ODR_MASK_SET(initResponse->options,
2004                                  Z_Options_negotiationModel);
2005                     ODR_MASK_SET(m_initRequest_options,
2006                                  Z_Options_negotiationModel);
2007
2008                     oi->which = Z_OtherInfo_externallyDefinedInfo;
2009                     oi->information.externallyDefinedInfo =
2010                         yaz_set_response_charneg(
2011                             odr_encode(),
2012                             m_charset_converter->get_client_query_charset(),
2013                             0 /* no lang */,
2014                             m_charset_converter->get_client_charset_selected());
2015                 }
2016             }
2017         }
2018     }
2019 }
2020
2021 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2022                                                      int error,
2023                                                      const char *addinfo)
2024 {
2025     Z_Records *rec = (Z_Records *)
2026         odr_malloc (odr, sizeof(*rec));
2027     int *err = (int *)
2028         odr_malloc (odr, sizeof(*err));
2029     Z_DiagRec *drec = (Z_DiagRec *)
2030         odr_malloc (odr, sizeof(*drec));
2031     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2032         odr_malloc (odr, sizeof(*dr));
2033     *err = error;
2034     rec->which = Z_Records_NSD;
2035     rec->u.nonSurrogateDiagnostic = dr;
2036     dr->diagnosticSetId =
2037         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
2038     dr->condition = err;
2039     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2040     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2041     return rec;
2042 }
2043
2044 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2045 {
2046     if (apdu->which == Z_APDU_searchRequest &&
2047         apdu->u.searchRequest->query &&
2048         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2049         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2050     {
2051         Z_RPNQuery *rpnquery = 0;
2052         Z_SearchRequest *sr = apdu->u.searchRequest;
2053         char *addinfo = 0;
2054
2055         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2056                 sr->query->u.type_104->u.cql);
2057
2058         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2059                                           &rpnquery, odr_encode(),
2060                                           &addinfo);
2061         if (r == -3)
2062             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2063         else if (r)
2064         {
2065             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2066             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2067
2068             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2069             new_apdu->u.searchResponse->records =
2070                 create_nonSurrogateDiagnostics(odr_encode(),
2071                                                yaz_diag_srw_to_bib1(r),
2072                                                addinfo);
2073             *new_apdu->u.searchResponse->searchStatus = 0;
2074
2075             send_to_client(new_apdu);
2076
2077             return 0;
2078         }
2079         else
2080         {
2081             sr->query->which = Z_Query_type_1;
2082             sr->query->u.type_1 = rpnquery;
2083         }
2084         return apdu;
2085     }
2086     return apdu;
2087 }
2088
2089 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2090 {
2091     if (apdu->which == Z_APDU_searchRequest &&
2092         apdu->u.searchRequest->query)
2093     {
2094         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2095             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2096         {
2097             if (m_http_version)
2098                 m_charset_converter->set_client_query_charset("UTF-8");
2099             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2100             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2101         }
2102     }
2103     return apdu;
2104 }
2105
2106
2107 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2108 {
2109     if (apdu->which == Z_APDU_searchRequest)
2110     {
2111         Z_SearchRequest *sr = apdu->u.searchRequest;
2112         int err = 0;
2113         char *addinfo = 0;
2114
2115         Yaz_ProxyConfig *cfg = check_reconfigure();
2116         if (cfg)
2117             err = cfg->check_query(odr_encode(), m_default_target,
2118                                    sr->query, &addinfo);
2119         if (err)
2120         {
2121             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2122
2123             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2124             new_apdu->u.searchResponse->records =
2125                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2126             *new_apdu->u.searchResponse->searchStatus = 0;
2127
2128             send_to_client(new_apdu);
2129
2130             return 0;
2131         }
2132     }
2133     return apdu;
2134 }
2135
2136 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2137 {
2138     if (apdu->which != Z_APDU_initRequest)
2139         return 1;  // pass if no init request
2140     Z_InitRequest *req = apdu->u.initRequest;
2141
2142     Yaz_ProxyConfig *cfg = check_reconfigure();
2143     if (!cfg)
2144         return 1;  // pass if no config
2145
2146     int ret;
2147     if (req->idAuthentication == 0)
2148     {
2149         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2150                                          m_peername);
2151     }
2152     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2153     {
2154         ret = cfg->client_authentication(
2155             m_default_target,
2156             req->idAuthentication->u.idPass->userId,
2157             req->idAuthentication->u.idPass->groupId,
2158             req->idAuthentication->u.idPass->password,
2159             m_peername);
2160     }
2161     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2162     {
2163         char user[64], pass[64];
2164         *user = '\0';
2165         *pass = '\0';
2166         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2167         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2168                                          m_peername);
2169     }
2170     else
2171         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2172                                          m_peername);
2173
2174     cfg->target_authentication(m_default_target, odr_encode(), req);
2175
2176     return ret;
2177 }
2178
2179 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2180 {
2181     m_marcxml_mode = none;
2182     if (apdu->which == Z_APDU_searchRequest)
2183     {
2184         Z_SearchRequest *sr = apdu->u.searchRequest;
2185         int err = 0;
2186         char *addinfo = 0;
2187         Yaz_ProxyConfig *cfg = check_reconfigure();
2188
2189         Z_RecordComposition rc_temp, *rc = 0;
2190         if (sr->smallSetElementSetNames)
2191         {
2192             rc_temp.which = Z_RecordComp_simple;
2193             rc_temp.u.simple = sr->smallSetElementSetNames;
2194             rc = &rc_temp;
2195         }
2196
2197         if (sr->preferredRecordSyntax)
2198         {
2199             struct oident *ent;
2200             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2201             m_frontend_type = ent->value;
2202         }
2203         else
2204             m_frontend_type = VAL_NONE;
2205
2206         char *stylesheet_name = 0;
2207         if (cfg)
2208             err = cfg->check_syntax(odr_encode(),
2209                                     m_default_target,
2210                                     sr->preferredRecordSyntax, rc,
2211                                     &addinfo, &stylesheet_name, &m_schema,
2212                                     &m_backend_type, &m_backend_charset,
2213                                     &m_usemarcon_ini_stage1,
2214                                     &m_usemarcon_ini_stage2);
2215         if (stylesheet_name)
2216         {
2217             m_parent->low_socket_close();
2218
2219 #if HAVE_XSLT
2220             if (m_stylesheet_xsp)
2221                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2222             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2223                                                        stylesheet_name);
2224 #endif
2225             m_stylesheet_offset = 0;
2226             xfree(stylesheet_name);
2227
2228             m_parent->low_socket_open();
2229         }
2230         if (err == -1)
2231         {
2232             sr->smallSetElementSetNames = 0;
2233             sr->mediumSetElementSetNames = 0;
2234             m_marcxml_mode = marcxml;
2235             if (m_backend_type)
2236             {
2237
2238                 sr->preferredRecordSyntax =
2239                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2240                                         m_backend_type);
2241             }
2242             else
2243                 sr->preferredRecordSyntax =
2244                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2245                                            VAL_USMARC);
2246         }
2247         else if (err)
2248         {
2249             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2250
2251             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2252             new_apdu->u.searchResponse->records =
2253                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2254             *new_apdu->u.searchResponse->searchStatus = 0;
2255
2256             send_to_client(new_apdu);
2257
2258             return 0;
2259         }
2260         else if (m_backend_type)
2261         {
2262             sr->preferredRecordSyntax =
2263                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2264         }
2265     }
2266     else if (apdu->which == Z_APDU_presentRequest)
2267     {
2268         Z_PresentRequest *pr = apdu->u.presentRequest;
2269         int err = 0;
2270         char *addinfo = 0;
2271         Yaz_ProxyConfig *cfg = check_reconfigure();
2272
2273         if (pr->preferredRecordSyntax)
2274         {
2275             struct oident *ent;
2276             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2277             m_frontend_type = ent->value;
2278         }
2279         else
2280             m_frontend_type = VAL_NONE;
2281
2282         char *stylesheet_name = 0;
2283         if (cfg)
2284             err = cfg->check_syntax(odr_encode(), m_default_target,
2285                                     pr->preferredRecordSyntax,
2286                                     pr->recordComposition,
2287                                     &addinfo, &stylesheet_name, &m_schema,
2288                                     &m_backend_type, &m_backend_charset,
2289                                     &m_usemarcon_ini_stage1,
2290                                     &m_usemarcon_ini_stage2
2291                                     );
2292         if (stylesheet_name)
2293         {
2294             m_parent->low_socket_close();
2295
2296 #if HAVE_XSLT
2297             if (m_stylesheet_xsp)
2298                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2299             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2300                                                        stylesheet_name);
2301 #endif
2302             m_stylesheet_offset = 0;
2303             xfree(stylesheet_name);
2304
2305             m_parent->low_socket_open();
2306         }
2307         if (err == -1)
2308         {
2309             pr->recordComposition = 0;
2310             m_marcxml_mode = marcxml;
2311             if (m_backend_type)
2312             {
2313
2314                 pr->preferredRecordSyntax =
2315                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2316                                         m_backend_type);
2317             }
2318             else
2319                 pr->preferredRecordSyntax =
2320                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2321                                            VAL_USMARC);
2322         }
2323         else if (err)
2324         {
2325             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2326
2327             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2328             new_apdu->u.presentResponse->records =
2329                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2330             *new_apdu->u.presentResponse->presentStatus =
2331                 Z_PresentStatus_failure;
2332
2333             send_to_client(new_apdu);
2334
2335             return 0;
2336         }
2337         else if (m_backend_type)
2338         {
2339             pr->preferredRecordSyntax =
2340                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2341         }
2342     }
2343     return apdu;
2344 }
2345
2346 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2347 {
2348     if (!schema)
2349         return 0;
2350     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2351         odr_malloc(o, sizeof(Z_ElementSetNames));
2352     esn->which = Z_ElementSetNames_generic;
2353     esn->u.generic = odr_strdup(o, schema);
2354     return esn;
2355 }
2356
2357 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2358 {
2359     const char *t = 0;
2360     Yaz_ProxyConfig *cfg = check_reconfigure();
2361     if (cfg)
2362         t = cfg->get_explain_name(db, backend_db);
2363
2364     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2365     {
2366         releaseClient();
2367     }
2368
2369     if (t)
2370     {
2371         xfree(m_default_target);
2372         m_default_target = xstrdup(t);
2373     }
2374 }
2375
2376 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2377 {
2378     struct stat sbuf;
2379     if (strcmp(hreq->method, "GET"))
2380         return 0;
2381     if (hreq->path[0] != '/')
2382     {
2383         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2384         return 0;
2385     }
2386     const char *cp = hreq->path;
2387     while (*cp)
2388     {
2389         if (*cp == '/' && strchr("/.", cp[1]))
2390         {
2391             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2392             return 0;
2393         }
2394         cp++;
2395     }
2396     const char *fname = hreq->path+1;
2397     if (stat(fname, &sbuf))
2398     {
2399         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2400         return 0;
2401     }
2402     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2403     {
2404         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2405         return 0;
2406     }
2407     if (sbuf.st_size > (off_t) 1000000)
2408     {
2409         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2410         return 0;
2411     }
2412
2413     ODR o = odr_encode();
2414     Yaz_ProxyConfig *cfg = check_reconfigure();
2415     const char *ctype = cfg->check_mime_type(fname);
2416     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2417     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2418     if (m_http_version)
2419         hres->version = odr_strdup(o, m_http_version);
2420     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2421     if (m_http_keepalive)
2422         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2423     else
2424         timeout(0);
2425
2426     hres->content_len = sbuf.st_size;
2427     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2428     FILE *f = fopen(fname, "rb");
2429     if (f)
2430     {
2431         fread(hres->content_buf, 1, hres->content_len, f);
2432         fclose(f);
2433     }
2434     else
2435     {
2436         return 0;
2437     }
2438     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2439     {
2440         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2441                  fname);
2442     }
2443     int len;
2444     send_GDU(gdu, &len);
2445     return 1;
2446 }
2447
2448 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2449 {
2450     if (m_s2z_odr_init)
2451     {
2452         odr_destroy(m_s2z_odr_init);
2453         m_s2z_odr_init = 0;
2454     }
2455     if (m_s2z_odr_search)
2456     {
2457         odr_destroy(m_s2z_odr_search);
2458         m_s2z_odr_search = 0;
2459     }
2460
2461     m_http_keepalive = 0;
2462     m_http_version = 0;
2463     if (!strcmp(hreq->version, "1.0"))
2464     {
2465         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2466         if (v && !strcmp(v, "Keep-Alive"))
2467             m_http_keepalive = 1;
2468         else
2469             m_http_keepalive = 0;
2470         m_http_version = "1.0";
2471     }
2472     else
2473     {
2474         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2475         if (v && !strcmp(v, "close"))
2476             m_http_keepalive = 0;
2477         else
2478             m_http_keepalive = 1;
2479         m_http_version = "1.1";
2480     }
2481
2482     Z_SRW_PDU *srw_pdu = 0;
2483     Z_SOAP *soap_package = 0;
2484     char *charset = 0;
2485     Z_SRW_diagnostic *diagnostic = 0;
2486     int num_diagnostic = 0;
2487
2488     if (file_access(hreq))
2489     {
2490         return;
2491     }
2492     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2493                             &charset) == 0
2494              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2495                                &charset, &diagnostic, &num_diagnostic) == 0)
2496     {
2497         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2498         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2499         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2500         m_s2z_init_apdu = 0;
2501         m_s2z_search_apdu = 0;
2502         m_s2z_present_apdu = 0;
2503
2504         m_s2z_stylesheet = 0;
2505
2506         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2507         {
2508             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2509
2510             const char *backend_db = srw_req->database;
2511             srw_get_client(srw_req->database, &backend_db);
2512
2513             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2514             // recordXPath unsupported.
2515             if (srw_req->recordXPath)
2516             {
2517                 yaz_add_srw_diagnostic(odr_decode(),
2518                                        &diagnostic, &num_diagnostic,
2519                                        72, 0);
2520             }
2521             // sort unsupported
2522             if (srw_req->sort_type != Z_SRW_sort_type_none)
2523             {
2524                 yaz_add_srw_diagnostic(odr_decode(),
2525                                        &diagnostic, &num_diagnostic,
2526                                        80, 0);
2527             }
2528             // save stylesheet
2529             if (srw_req->stylesheet)
2530                 m_s2z_stylesheet =
2531                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2532
2533             // set packing for response records ..
2534             if (srw_req->recordPacking &&
2535                 !strcmp(srw_req->recordPacking, "xml"))
2536                 m_s2z_packing = Z_SRW_recordPacking_XML;
2537             else
2538                 m_s2z_packing = Z_SRW_recordPacking_string;
2539
2540             if (num_diagnostic)
2541             {
2542                 Z_SRW_PDU *srw_pdu =
2543                     yaz_srw_get(odr_encode(),
2544                                 Z_SRW_searchRetrieve_response);
2545                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2546
2547                 srw_res->diagnostics = diagnostic;
2548                 srw_res->num_diagnostics = num_diagnostic;
2549                 send_srw_response(srw_pdu);
2550                 return;
2551             }
2552
2553             // prepare search PDU
2554             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2555                                           Z_APDU_searchRequest);
2556             Z_SearchRequest *z_searchRequest =
2557                 m_s2z_search_apdu->u.searchRequest;
2558
2559             z_searchRequest->num_databaseNames = 1;
2560             z_searchRequest->databaseNames = (char**)
2561                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2562             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2563                                                            backend_db);
2564
2565             // query transformation
2566             Z_Query *query = (Z_Query *)
2567                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2568             z_searchRequest->query = query;
2569
2570             if (srw_req->query_type == Z_SRW_query_type_cql)
2571             {
2572                 Z_External *ext = (Z_External *)
2573                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2574                 ext->direct_reference =
2575                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2576                 ext->indirect_reference = 0;
2577                 ext->descriptor = 0;
2578                 ext->which = Z_External_CQL;
2579                 ext->u.cql = srw_req->query.cql;
2580
2581                 query->which = Z_Query_type_104;
2582                 query->u.type_104 =  ext;
2583             }
2584             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2585             {
2586                 Z_RPNQuery *RPNquery;
2587                 YAZ_PQF_Parser pqf_parser;
2588
2589                 pqf_parser = yaz_pqf_create ();
2590
2591                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2592                                           srw_req->query.pqf);
2593                 if (!RPNquery)
2594                 {
2595                     const char *pqf_msg;
2596                     size_t off;
2597                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2598                     yaz_log(YLOG_LOG, "%*s^\n", off+4, "");
2599                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2600
2601                     send_to_srw_client_error(10, 0);
2602                     return;
2603                 }
2604                 query->which = Z_Query_type_1;
2605                 query->u.type_1 =  RPNquery;
2606
2607                 yaz_pqf_destroy (pqf_parser);
2608             }
2609             else
2610             {
2611                 send_to_srw_client_error(7, "query");
2612                 return;
2613             }
2614
2615             // present
2616             m_s2z_present_apdu = 0;
2617             int max = 0;
2618             if (srw_req->maximumRecords)
2619                 max = *srw_req->maximumRecords;
2620             int start = 1;
2621             if (srw_req->startRecord)
2622                 start = *srw_req->startRecord;
2623             if (max > 0)
2624             {
2625                 // Some backend, such as Voyager doesn't honor piggyback
2626                 // So we use present always (0 &&).
2627                 if (0 && start <= 1)  // Z39.50 piggyback
2628                 {
2629                     *z_searchRequest->smallSetUpperBound = max;
2630                     *z_searchRequest->mediumSetPresentNumber = max;
2631                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2632
2633                     z_searchRequest->preferredRecordSyntax =
2634                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2635                                                VAL_TEXT_XML);
2636                     if (srw_req->recordSchema)
2637                     {
2638                         z_searchRequest->smallSetElementSetNames =
2639                             z_searchRequest->mediumSetElementSetNames =
2640                             mk_esn_from_schema(m_s2z_odr_search,
2641                                                srw_req->recordSchema);
2642                     }
2643                 }
2644                 else   // Z39.50 present
2645                 {
2646                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2647                                                    Z_APDU_presentRequest);
2648                     Z_PresentRequest *z_presentRequest =
2649                         m_s2z_present_apdu->u.presentRequest;
2650                     *z_presentRequest->resultSetStartPoint = start;
2651                     *z_presentRequest->numberOfRecordsRequested = max;
2652                     z_presentRequest->preferredRecordSyntax =
2653                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2654                                                VAL_TEXT_XML);
2655                     if (srw_req->recordSchema)
2656                     {
2657                         z_presentRequest->recordComposition =
2658                             (Z_RecordComposition *)
2659                             odr_malloc(m_s2z_odr_search,
2660                                        sizeof(Z_RecordComposition));
2661                         z_presentRequest->recordComposition->which =
2662                             Z_RecordComp_simple;
2663                         z_presentRequest->recordComposition->u.simple =
2664                             mk_esn_from_schema(m_s2z_odr_search,
2665                                                srw_req->recordSchema);
2666                     }
2667                 }
2668             }
2669             if (!m_client)
2670             {
2671                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2672                                             Z_APDU_initRequest);
2673
2674                 // prevent m_initRequest_apdu memory from being grabbed
2675                 // in Yaz_Proxy::handle_incoming_Z_PDU
2676                 m_initRequest_apdu = m_s2z_init_apdu;
2677                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2678                 return;
2679             }
2680             else
2681             {
2682                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2683                 return;
2684             }
2685         }
2686         else if (srw_pdu->which == Z_SRW_explain_request)
2687         {
2688             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2689
2690             const char *backend_db = srw_req->database;
2691             srw_get_client(srw_req->database, &backend_db);
2692
2693             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2694
2695             // save stylesheet
2696             if (srw_req->stylesheet)
2697                 m_s2z_stylesheet =
2698                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2699
2700             if (srw_req->recordPacking &&
2701                 !strcmp(srw_req->recordPacking, "xml"))
2702                 m_s2z_packing = Z_SRW_recordPacking_XML;
2703             else
2704                 m_s2z_packing = Z_SRW_recordPacking_string;
2705
2706             if (num_diagnostic)
2707             {
2708                 send_srw_explain_response(diagnostic, num_diagnostic);
2709                 return;
2710             }
2711
2712             if (!m_client)
2713             {
2714                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2715                                             Z_APDU_initRequest);
2716
2717                 // prevent m_initRequest_apdu memory from being grabbed
2718                 // in Yaz_Proxy::handle_incoming_Z_PDU
2719                 m_initRequest_apdu = m_s2z_init_apdu;
2720                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2721             }
2722             else
2723                 send_srw_explain_response(0, 0);
2724             return;
2725         }
2726         else if (srw_pdu->which == Z_SRW_scan_request)
2727         {
2728             m_s2z_database = odr_strdup(m_s2z_odr_init,
2729                                         srw_pdu->u.scan_request->database);
2730
2731             yaz_add_srw_diagnostic(odr_decode(),
2732                                    &diagnostic, &num_diagnostic,
2733                                    4, "scan");
2734             Z_SRW_PDU *srw_pdu =
2735                 yaz_srw_get(odr_encode(),
2736                             Z_SRW_scan_response);
2737             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2738
2739             srw_res->diagnostics = diagnostic;
2740             srw_res->num_diagnostics = num_diagnostic;
2741             send_srw_response(srw_pdu);
2742             return;
2743         }
2744         else
2745         {
2746             m_s2z_database = 0;
2747
2748             send_to_srw_client_error(4, 0);
2749         }
2750     }
2751     send_http_response(400);
2752 }
2753
2754 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2755 {
2756     Z_OtherInformation **oi;
2757     get_otherInfoAPDU(apdu, &oi);
2758
2759     if (apdu->u.initRequest->implementationId)
2760         yaz_log(YLOG_LOG, "%simplementationId: %s",
2761                 m_session_str, apdu->u.initRequest->implementationId);
2762     if (apdu->u.initRequest->implementationName)
2763         yaz_log(YLOG_LOG, "%simplementationName: %s",
2764                 m_session_str, apdu->u.initRequest->implementationName);
2765     if (apdu->u.initRequest->implementationVersion)
2766         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2767                 m_session_str, apdu->u.initRequest->implementationVersion);
2768     if (m_initRequest_apdu == 0)
2769     {
2770         if (m_initRequest_mem)
2771             nmem_destroy(m_initRequest_mem);
2772
2773         m_initRequest_apdu = apdu;
2774         m_initRequest_mem = odr_extract_mem(odr_decode());
2775
2776         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2777             preferredMessageSize;
2778         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2779         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2780             maximumRecordSize;
2781         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2782
2783         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2784             yaz_get_charneg_record(*oi);
2785
2786         // Save proposal charsets and langs.
2787         if (ODR_MASK_GET(apdu->u.initRequest->options,
2788                          Z_Options_negotiationModel)
2789             && charSetandLangRecord)
2790         {
2791
2792             yaz_get_proposal_charneg(m_referenceId_mem,
2793                                      charSetandLangRecord,
2794                                      &m_initRequest_oi_negotiation_charsets,
2795                                      &m_initRequest_oi_negotiation_num_charsets,
2796                                      &m_initRequest_oi_negotiation_langs,
2797                                      &m_initRequest_oi_negotiation_num_langs,
2798                                      &m_initRequest_oi_negotiation_selected);
2799
2800             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2801             {
2802                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2803                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2804                         m_initRequest_oi_negotiation_charsets[i]:"none");
2805             }
2806             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2807             {
2808                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2809                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2810                         m_initRequest_oi_negotiation_langs[i]:"none");
2811             }
2812             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2813                     m_session_str, m_initRequest_oi_negotiation_selected);
2814         }
2815         // save init options for the response..
2816         m_initRequest_options = apdu->u.initRequest->options;
2817
2818         apdu->u.initRequest->options =
2819             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2820                                        sizeof(Odr_bitmask));
2821         ODR_MASK_ZERO(apdu->u.initRequest->options);
2822         int i;
2823         for (i = 0; i<= 24; i++)
2824             ODR_MASK_SET(apdu->u.initRequest->options, i);
2825         // check negotiation option
2826         if (!ODR_MASK_GET(m_initRequest_options,
2827                           Z_Options_negotiationModel))
2828         {
2829             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2830                            Z_Options_negotiationModel);
2831         }
2832         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2833                        Z_Options_concurrentOperations);
2834         // make new version
2835         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2836         apdu->u.initRequest->protocolVersion =
2837             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2838                                        sizeof(Odr_bitmask));
2839         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2840
2841         for (i = 0; i<= 8; i++)
2842             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2843     }
2844     if (m_client->m_init_flag)
2845     {
2846         if (handle_init_response_for_invalid_session(apdu))
2847             return;
2848         if (m_client->m_initResponse)
2849         {
2850             Z_APDU *apdu2 = m_client->m_initResponse;
2851             apdu2->u.initResponse->otherInfo = 0;
2852             if (m_client->m_cookie && *m_client->m_cookie)
2853                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2854                                            m_client->m_cookie);
2855             apdu2->u.initResponse->referenceId =
2856                 apdu->u.initRequest->referenceId;
2857             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2858             apdu2->u.initResponse->protocolVersion =
2859                 m_client->m_initResponse_version;
2860
2861             send_to_client(apdu2);
2862             return;
2863         }
2864     }
2865     m_client->m_init_flag = 1;
2866
2867 #if USE_AUTH_MSG
2868     Auth_Msg *m = new Auth_Msg;
2869     m->m_proxy = this;
2870     z_APDU(odr_encode(), &apdu, 0, "encode");
2871     char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
2872     m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
2873     memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
2874     odr_reset(odr_encode());
2875
2876     inc_ref();
2877     m_my_thread->put(m);
2878 #else
2879     int ret = handle_authentication(apdu);
2880     result_authentication(apdu, ret);
2881 #endif
2882 }
2883
2884 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2885 {
2886     Z_ReferenceId **refid = get_referenceIdP(apdu);
2887     nmem_reset(m_referenceId_mem);
2888     if (refid && *refid)
2889     {
2890         m_referenceId = (Z_ReferenceId *)
2891             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2892         m_referenceId->len = m_referenceId->size = (*refid)->len;
2893         m_referenceId->buf = (unsigned char *)
2894             nmem_malloc(m_referenceId_mem, (*refid)->len);
2895         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2896     }
2897     else
2898         m_referenceId = 0;
2899
2900     if (!m_client && m_flag_invalid_session)
2901     {
2902         // Got request for a session that is invalid..
2903         m_apdu_invalid_session = apdu; // save package
2904         m_mem_invalid_session = odr_extract_mem(odr_decode());
2905         apdu = m_initRequest_apdu;     // but throw an init to the target
2906     }
2907
2908     // Determine our client.
2909     Z_OtherInformation **oi;
2910     get_otherInfoAPDU(apdu, &oi);
2911     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2912     if (!m_client)
2913     {
2914         if (m_http_version)
2915         {
2916             send_http_response(404);
2917             return;
2918         }
2919         else
2920         {
2921             delete this;
2922             return;
2923         }
2924     }
2925
2926     m_client->m_server = this;
2927
2928     if (apdu->which == Z_APDU_initRequest)
2929         handle_init(apdu);
2930     else
2931         handle_incoming_Z_PDU_2(apdu);
2932 }
2933
2934 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
2935 {
2936     handle_max_record_retrieve(apdu);
2937
2938     if (apdu)
2939         apdu = handle_syntax_validation(apdu);
2940
2941     if (apdu)
2942         apdu = handle_query_transformation(apdu);
2943
2944     if (apdu)
2945         apdu = handle_target_charset_conversion(apdu);
2946
2947     if (apdu)
2948         apdu = handle_query_validation(apdu);
2949
2950     if (apdu)
2951         apdu = result_set_optimize(apdu);
2952
2953     if (!apdu)
2954     {
2955         m_client->timeout(m_target_idletime);  // mark it active even
2956         // though we didn't use it
2957         return;
2958     }
2959     // Add otherInformation entry in APDU if
2960     // negotiation is in use.
2961     if (apdu)
2962         handle_charset_lang_negotiation(apdu);
2963
2964     // delete other info construct completely if 0 elements
2965     Z_OtherInformation **oi;
2966     get_otherInfoAPDU(apdu, &oi);
2967     if (oi && *oi && (*oi)->num_elements == 0)
2968         *oi = 0;
2969
2970     if (apdu->which == Z_APDU_presentRequest &&
2971         m_client->m_resultSetStartPoint == 0)
2972     {
2973         Z_PresentRequest *pr = apdu->u.presentRequest;
2974         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2975         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2976     } else {
2977         m_client->m_resultSetStartPoint = 0;
2978     }
2979     if (m_client->send_to_target(apdu) < 0)
2980     {
2981         delete m_client;
2982         m_client = 0;
2983         delete this;
2984     }
2985     else
2986         m_client->m_waiting = 1;
2987 }
2988
2989 void Yaz_Proxy::connectNotify()
2990 {
2991 }
2992
2993 void Yaz_Proxy::releaseClient()
2994 {
2995     xfree(m_proxyTarget);
2996     m_proxyTarget = 0;
2997     m_flag_invalid_session = 0;
2998     // only keep if keep_alive flag is set...
2999     if (m_client &&
3000         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3001         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3002         m_client->m_waiting == 0)
3003     {
3004         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3005                  m_session_str,
3006                  m_client->get_hostname());
3007         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3008                 m_session_str, m_client->m_pdu_recv,
3009                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3010                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3011         assert (m_client->m_waiting != 2);
3012         // Tell client (if any) that no server connection is there..
3013         m_client->m_server = 0;
3014         m_client = 0;
3015     }
3016     else if (m_client)
3017     {
3018         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
3019                  m_session_str,
3020                  m_client->get_hostname());
3021         assert (m_client->m_waiting != 2);
3022         delete m_client;
3023         m_client = 0;
3024     }
3025     else if (!m_parent)
3026     {
3027         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
3028                  m_session_str);
3029         assert (m_parent);
3030     }
3031     else
3032     {
3033         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
3034                  m_session_str);
3035     }
3036     if (m_parent)
3037         m_parent->pre_init();
3038 }
3039
3040 bool Yaz_Proxy::dec_ref()
3041 {
3042     --m_ref_count;
3043     assert(m_ref_count >= 0);
3044     bool last = (m_ref_count == 0);
3045     if (m_ref_count == 0)
3046     {
3047         releaseClient();
3048         delete this;
3049     }
3050     return last;
3051 }
3052
3053 const char *Yaz_ProxyClient::get_session_str()
3054 {
3055     if (!m_server)
3056         return "0 ";
3057     return m_server->get_session_str();
3058 }
3059
3060 void Yaz_ProxyClient::shutdown()
3061 {
3062     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3063              get_hostname());
3064     delete m_server;
3065     delete this;
3066 }
3067
3068 void Yaz_Proxy::failNotify()
3069 {
3070     inc_request_no();
3071     yaz_log (YLOG_LOG, "%sConnection closed by client",
3072              get_session_str());
3073     dec_ref();
3074 }
3075
3076 void Yaz_ProxyClient::failNotify()
3077 {
3078     if (m_server)
3079         m_server->inc_request_no();
3080     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
3081              get_session_str(), get_hostname());
3082     shutdown();
3083 }
3084
3085 void Yaz_ProxyClient::connectNotify()
3086 {
3087     const char *s = get_session_str();
3088     const char *h = get_hostname();
3089     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3090              m_target_idletime);
3091     timeout(m_target_idletime);
3092     if (!m_server)
3093         pre_init_client();
3094 }
3095
3096 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3097                                               *the_PDU_Observable, int fd)
3098 {
3099     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3100 }
3101
3102 Yaz_ProxyClient::~Yaz_ProxyClient()
3103 {
3104     if (m_prev)
3105         *m_prev = m_next;
3106     if (m_next)
3107         m_next->m_prev = m_prev;
3108     m_waiting = 2;     // for debugging purposes only.
3109     odr_destroy(m_init_odr);
3110     delete m_last_query;
3111     xfree (m_last_resultSetId);
3112     xfree (m_cookie);
3113 }
3114
3115 void Yaz_ProxyClient::pre_init_client()
3116 {
3117     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3118     Z_InitRequest *req = apdu->u.initRequest;
3119
3120     int i;
3121     for (i = 0; i<= 24; i++)
3122         ODR_MASK_SET(req->options, i);
3123     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3124                    Z_Options_negotiationModel);
3125     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3126                    Z_Options_concurrentOperations);
3127     for (i = 0; i<= 10; i++)
3128         ODR_MASK_SET(req->protocolVersion, i);
3129
3130     if (send_to_target(apdu) < 0)
3131     {
3132         delete this;
3133     }
3134     else
3135     {
3136         m_waiting = 1;
3137         m_init_flag = 1;
3138     }
3139 }
3140
3141 void Yaz_Proxy::pre_init()
3142 {
3143     int i;
3144     const char *name = 0;
3145     const char *zurl_in_use[MAX_ZURL_PLEX];
3146     int limit_bw, limit_pdu, limit_req;
3147     int target_idletime, client_idletime;
3148     int max_clients;
3149     int keepalive_limit_bw, keepalive_limit_pdu;
3150     int pre_init;
3151     const char *cql2rpn = 0;
3152     const char *authentication = 0;
3153     const char *negotiation_charset = 0;
3154     const char *negotiation_lang = 0;
3155
3156     Yaz_ProxyConfig *cfg = check_reconfigure();
3157
3158     zurl_in_use[0] = 0;
3159
3160     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3161         set_APDU_yazlog(1);
3162     else
3163         set_APDU_yazlog(0);
3164
3165     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3166                                           &limit_bw, &limit_pdu, &limit_req,
3167                                           &target_idletime, &client_idletime,
3168                                           &max_clients,
3169                                           &keepalive_limit_bw,
3170                                           &keepalive_limit_pdu,
3171                                           &pre_init,
3172                                           &cql2rpn,
3173                                           &authentication,
3174                                           &negotiation_charset,
3175                                           &negotiation_lang,
3176                                           0) ; i++)
3177     {
3178         if (pre_init)
3179         {
3180             int j;
3181             for (j = 0; zurl_in_use[j]; j++)
3182             {
3183                 Yaz_ProxyClient *c;
3184                 int spare = 0;
3185                 int spare_waiting = 0;
3186                 int in_use = 0;
3187                 int other = 0;
3188                 for (c = m_clientPool; c; c = c->m_next)
3189                 {
3190                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3191                     {
3192                         if (c->m_cookie == 0)
3193                         {
3194                             if (c->m_server == 0)
3195                                 if (c->m_waiting)
3196                                     spare_waiting++;
3197                                 else
3198                                     spare++;
3199                             else
3200                                 in_use++;
3201                         }
3202                         else
3203                             other++;
3204                     }
3205                 }
3206                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3207                         "sparew=%d preinit=%d",m_session_str,
3208                         name, zurl_in_use[j], in_use, other,
3209                         spare, spare_waiting, pre_init);
3210                 if (spare + spare_waiting < pre_init)
3211                 {
3212                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3213                     c->m_next = m_clientPool;
3214                     if (c->m_next)
3215                         c->m_next->m_prev = &c->m_next;
3216                     m_clientPool = c;
3217                     c->m_prev = &m_clientPool;
3218
3219                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3220                         c->set_APDU_yazlog(1);
3221                     else
3222                         c->set_APDU_yazlog(0);
3223
3224                     if (c->client(zurl_in_use[j]))
3225                     {
3226                         timeout(60);
3227                         delete c;
3228                         return;
3229                     }
3230                     c->timeout(30);
3231                     c->m_waiting = 1;
3232                     c->m_target_idletime = target_idletime;
3233                     c->m_seqno = m_seqno++;
3234                 }
3235             }
3236         }
3237     }
3238 }
3239
3240 void Yaz_Proxy::timeoutNotify()
3241 {
3242     if (m_parent)
3243     {
3244         GDU *gdu;
3245         switch(m_timeout_mode)
3246         {
3247         case timeout_normal:
3248             inc_request_no();
3249             m_in_queue.clear();
3250             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3251             dec_ref();
3252             break;
3253         case timeout_reduce:
3254             timeout(m_client_idletime);
3255             m_timeout_mode = timeout_normal;
3256             gdu = m_timeout_gdu;
3257             m_timeout_gdu = 0;
3258             recv_GDU_normal(gdu);
3259             break;
3260         case timeout_xsl:
3261             assert(m_stylesheet_nprl);
3262             convert_xsl_delay();
3263             recv_GDU_more();
3264         }
3265     }
3266     else
3267     {
3268         timeout(600);
3269         pre_init();
3270     }
3271 }
3272
3273 void Yaz_Proxy::markInvalid()
3274 {
3275     m_client = 0;
3276     m_flag_invalid_session = 1;
3277 }
3278
3279 void Yaz_ProxyClient::timeoutNotify()
3280 {
3281     if (m_server)
3282         m_server->inc_request_no();
3283
3284     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3285              get_hostname());
3286     m_waiting = 1;
3287     m_root->pre_init();
3288     if (m_server && m_init_flag)
3289     {
3290         // target timed out in a session that was properly initialized
3291         // server object stay alive but we mark it as invalid so it
3292         // gets initialized again
3293         m_server->markInvalid();
3294         m_server = 0;
3295     }
3296     shutdown();
3297 }
3298
3299 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
3300                                  Yaz_Proxy *parent) :
3301     Z_Assoc (the_PDU_Observable)
3302 {
3303     m_cookie = 0;
3304     m_next = 0;
3305     m_prev = 0;
3306     m_init_flag = 0;
3307     m_last_query = 0;
3308     m_last_resultSetId = 0;
3309     m_last_resultCount = 0;
3310     m_last_ok = 0;
3311     m_sr_transform = 0;
3312     m_waiting = 0;
3313     m_init_odr = odr_createmem (ODR_DECODE);
3314     m_initResponse = 0;
3315     m_initResponse_options = 0;
3316     m_initResponse_version = 0;
3317     m_initResponse_preferredMessageSize = 0;
3318     m_initResponse_maximumRecordSize = 0;
3319     m_resultSetStartPoint = 0;
3320     m_bytes_sent = m_bytes_recv = 0;
3321     m_pdu_recv = 0;
3322     m_server = 0;
3323     m_seqno = 0;
3324     m_target_idletime = 600;
3325     m_root = parent;
3326 }
3327
3328 const char *Yaz_Proxy::option(const char *name, const char *value)
3329 {
3330     if (!strcmp (name, "optimize")) {
3331         if (value) {
3332             xfree (m_optimize);
3333             m_optimize = xstrdup (value);
3334         }
3335         return m_optimize;
3336     }
3337     return 0;
3338 }
3339
3340 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
3341 {
3342
3343 }
3344
3345 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
3346 {
3347     if (apdu->which == Z_GDU_Z3950)
3348         recv_Z_PDU(apdu->u.z3950, len);
3349     else if (apdu->which == Z_GDU_HTTP_Response)
3350         recv_HTTP_response(apdu->u.HTTP_Response, len);
3351     else
3352         shutdown();
3353 }
3354
3355 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
3356 {
3357     if (!m_flag_invalid_session)
3358         return 0;
3359     m_flag_invalid_session = 0;
3360     handle_incoming_Z_PDU(m_apdu_invalid_session);
3361     assert (m_mem_invalid_session);
3362     nmem_destroy(m_mem_invalid_session);
3363     m_mem_invalid_session = 0;
3364     return 1;
3365 }
3366
3367 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
3368 {
3369     m_bytes_recv += len;
3370
3371     m_pdu_recv++;
3372     m_waiting = 0;
3373     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
3374         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
3375                  apdu_name(apdu), get_hostname(), len);
3376     if (apdu->which == Z_APDU_initResponse)
3377     {
3378         if (!m_server)  // if this is a pre init session , check for more
3379             m_root->pre_init();
3380         NMEM nmem = odr_extract_mem (odr_decode());
3381         odr_reset (m_init_odr);
3382         nmem_transfer (m_init_odr->mem, nmem);
3383         m_initResponse = apdu;
3384         m_initResponse_options = apdu->u.initResponse->options;
3385         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3386         m_initResponse_preferredMessageSize =
3387             *apdu->u.initResponse->preferredMessageSize;
3388         m_initResponse_maximumRecordSize =
3389             *apdu->u.initResponse->maximumRecordSize;
3390
3391         Z_InitResponse *ir = apdu->u.initResponse;
3392         char *im0 = ir->implementationName;
3393
3394         char *im1 = (char*)
3395             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3396         *im1 = '\0';
3397         if (im0)
3398         {
3399             strcat(im1, im0);
3400             strcat(im1, " ");
3401         }
3402         strcat(im1, "(YAZ Proxy)");
3403         ir->implementationName = im1;
3404
3405         nmem_destroy (nmem);
3406
3407         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3408             return;
3409     }
3410     if (apdu->which == Z_APDU_searchResponse)
3411     {
3412         Z_SearchResponse *sr = apdu->u.searchResponse;
3413         m_last_resultCount = *sr->resultCount;
3414         int status = *sr->searchStatus;
3415         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3416         {
3417             m_last_ok = 1;
3418
3419             if (sr->records && sr->records->which == Z_Records_DBOSD)
3420             {
3421                 m_cache.add(odr_decode(),
3422                             sr->records->u.databaseOrSurDiagnostics, 1,
3423                             *sr->resultCount);
3424             }
3425         }
3426     }
3427     if (apdu->which == Z_APDU_presentResponse)
3428     {
3429         Z_PresentResponse *pr = apdu->u.presentResponse;
3430         if (m_sr_transform)
3431         {
3432             m_sr_transform = 0;
3433             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3434             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3435             sr->referenceId = pr->referenceId;
3436             *sr->resultCount = m_last_resultCount;
3437             sr->records = pr->records;
3438             sr->nextResultSetPosition = pr->nextResultSetPosition;
3439             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3440             apdu = new_apdu;
3441         }
3442         if (pr->records &&
3443             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3444         {
3445             m_cache.add(odr_decode(),
3446                         pr->records->u.databaseOrSurDiagnostics,
3447                         m_resultSetStartPoint, -1);
3448             m_resultSetStartPoint = 0;
3449         }
3450     }
3451     if (m_cookie)
3452         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3453
3454     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
3455
3456     if (server)
3457         server->send_to_client(apdu);
3458     if (apdu->which == Z_APDU_close)
3459         shutdown();
3460     else if (server)
3461         server->recv_GDU_more();
3462
3463 }
3464
3465 void Yaz_Proxy::low_socket_close()
3466 {
3467 #if WIN32
3468 #else
3469     int i;
3470     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3471         if  (m_lo_fd[i] >= 0)
3472             ::close(m_lo_fd[i]);
3473 #endif
3474 }
3475
3476 void Yaz_Proxy::low_socket_open()
3477 {
3478 #if WIN32
3479 #else
3480     int i;
3481     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3482         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3483 #endif
3484 }
3485
3486 int Yaz_Proxy::server(const char *addr)
3487 {
3488     int r = Z_Assoc::server(addr);
3489     if (!r)
3490     {
3491         yaz_log(YLOG_LOG, "%sStarted proxy "
3492 #ifdef VERSION
3493             VERSION
3494 #endif
3495             " on %s", m_session_str, addr);
3496         timeout(1);
3497     }
3498     return r;
3499 }
3500
3501 /*
3502  * Local variables:
3503  * c-basic-offset: 4
3504  * indent-tabs-mode: nil
3505  * End:
3506  * vim: shiftwidth=4 tabstop=8 expandtab
3507  */
3508