src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.43 2006-03-09 14:12:57 adam Exp $
   2    Copyright (c) 1998-2005, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <stdlib.h>
  42 #include <time.h>
  43 #include <fcntl.h>
  44
  45 #include <yaz/srw.h>
  46 #include <yaz/marcdisp.h>
  47 #include <yaz/yaz-iconv.h>
  48 #include <yaz/log.h>
  49 #include <yaz/diagbib1.h>
  50 #include "proxyp.h"
  51 #include <yaz/pquery.h>
  52 #include <yaz/otherinfo.h>
  53 #include <yaz/charneg.h>
  54 #include "msg-thread.h"
  55
  56 using namespace yazpp_1;
  57
  58 #define USE_AUTH_MSG 1
  59
  60 #if USE_AUTH_MSG
  61 class Auth_Msg : public IMsg_Thread {
  62 public:
  63     int m_ret;
  64     IMsg_Thread *handle();
  65     void result();
  66     Yaz_Proxy *m_proxy;
  67     NMEM m_nmem;
  68     char *m_apdu_buf;
  69     int m_apdu_len;
  70     Auth_Msg();
  71     virtual ~Auth_Msg();
  72 };
  73
  74 Auth_Msg::Auth_Msg()
  75 {
  76     m_nmem = nmem_create();
  77 }
  78
  79 Auth_Msg::~Auth_Msg()
  80 {
  81     nmem_destroy(m_nmem);
  82 }
  83
  84 IMsg_Thread *Auth_Msg::handle()
  85 {
  86     yaz_log(YLOG_LOG, "Auth_Msg:handle begin");
  87     ODR decode = odr_createmem(ODR_DECODE);
  88     Z_APDU *apdu;
  89
  90     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
  91     int r = z_APDU(decode, &apdu, 0, 0);
  92     if (!r)
  93     {
  94         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
  95     }
  96     else
  97     {
  98         m_ret = m_proxy->handle_authentication(apdu);
  99     }
 100     yaz_log(YLOG_LOG, "Auth_Msg:handle end");
 101     odr_destroy(decode);
 102     return this;
 103 }
 104
 105 void Auth_Msg::result()
 106 {
 107     yaz_log(YLOG_LOG, "Auth_Msg:result proxy ok buf=%p len=%d",
 108             m_apdu_buf, m_apdu_len);
 109     if (m_proxy->dec_ref(false))
 110         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
 111     else
 112     {
 113         yaz_log(YLOG_LOG, "Auth_Msg::proxy still alive");
 114         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
 115         Z_APDU *apdu = 0;
 116         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
 117         if (r)
 118             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU OK");
 119         else
 120             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
 121         m_proxy->result_authentication(apdu, m_ret);
 122     }
 123     delete this;
 124 }
 125
 126 #endif
 127
 128 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
 129 {
 130     if (apdu == 0 || ret == 0)
 131     {
 132         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
 133         *apdu_reject->u.initResponse->result = 0;
 134         send_to_client(apdu_reject);
 135         dec_ref(false);
 136     }
 137     else
 138     {
 139         if (apdu->which == Z_APDU_initRequest)
 140         {
 141             Yaz_ProxyConfig *cfg = check_reconfigure();
 142             if (cfg)
 143                 cfg->target_authentication(m_default_target, odr_encode(),
 144                                            apdu->u.initRequest);
 145         }
 146         handle_incoming_Z_PDU_2(apdu);
 147     }
 148 }
 149
 150 static const char *apdu_name(Z_APDU *apdu)
 151 {
 152     switch (apdu->which)
 153     {
 154     case Z_APDU_initRequest:
 155         return "initRequest";
 156     case Z_APDU_initResponse:
 157         return "initResponse";
 158     case Z_APDU_searchRequest:
 159         return "searchRequest";
 160     case Z_APDU_searchResponse:
 161         return "searchResponse";
 162     case Z_APDU_presentRequest:
 163         return "presentRequest";
 164     case Z_APDU_presentResponse:
 165         return "presentResponse";
 166     case Z_APDU_deleteResultSetRequest:
 167         return "deleteResultSetRequest";
 168     case Z_APDU_deleteResultSetResponse:
 169         return "deleteResultSetResponse";
 170     case Z_APDU_scanRequest:
 171         return "scanRequest";
 172     case Z_APDU_scanResponse:
 173         return "scanResponse";
 174     case Z_APDU_sortRequest:
 175         return "sortRequest";
 176     case Z_APDU_sortResponse:
 177         return "sortResponse";
 178     case Z_APDU_extendedServicesRequest:
 179         return "extendedServicesRequest";
 180     case Z_APDU_extendedServicesResponse:
 181         return "extendedServicesResponse";
 182     case Z_APDU_close:
 183         return "close";
 184     }
 185     return "other";
 186 }
 187
 188 static const char *gdu_name(Z_GDU *gdu)
 189 {
 190     switch(gdu->which)
 191     {
 192     case Z_GDU_Z3950:
 193         return apdu_name(gdu->u.z3950);
 194     case Z_GDU_HTTP_Request:
 195         return "HTTP Request";
 196     case Z_GDU_HTTP_Response:
 197         return "HTTP Response";
 198     }
 199     return "Unknown request/response";
 200 }
 201
 202 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
 203                      ISocketObservable *the_socket_observable,
 204                      Yaz_Proxy *parent)
 205     :
 206     Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 207 {
 208     m_PDU_Observable = the_PDU_Observable;
 209     m_socket_observable = the_socket_observable;
 210     m_client = 0;
 211     m_parent = parent;
 212     m_clientPool = 0;
 213     m_seqno = 1;
 214     m_keepalive_limit_bw = 500000;
 215     m_keepalive_limit_pdu = 1000;
 216     m_proxyTarget = 0;
 217     m_default_target = 0;
 218     m_proxy_negotiation_charset = 0;
 219     m_proxy_negotiation_lang = 0;
 220     m_charset_converter = new Yaz_CharsetConverter;
 221     m_max_clients = 150;
 222     m_log_mask = 0;
 223     m_seed = time(0);
 224     m_client_idletime = 600;
 225     m_target_idletime = 600;
 226     m_optimize = xstrdup ("1");
 227     strcpy(m_session_str, "0 ");
 228     m_session_no = 0;
 229     m_bytes_sent = 0;
 230     m_bytes_recv = 0;
 231     m_bw_max = 0;
 232     m_pdu_max = 0;
 233     m_search_max = 0;
 234     m_connect_max = 0;
 235     m_timeout_mode = timeout_normal;
 236     m_timeout_gdu = 0;
 237     m_max_record_retrieve = 0;
 238     m_reconfig_flag = 0;
 239     m_config_fname = 0;
 240     m_request_no = 0;
 241     m_flag_invalid_session = 0;
 242     m_referenceId = 0;
 243     m_referenceId_mem = nmem_create();
 244     m_config = 0;
 245     m_marcxml_mode = none;
 246     m_stylesheet_xsp = 0;
 247     m_stylesheet_nprl = 0;
 248     m_stylesheet_apdu = 0;
 249     m_s2z_stylesheet = 0;
 250     m_s2z_database = 0;
 251     m_schema = 0;
 252     m_backend_type = 0;
 253     m_backend_charset = 0;
 254     m_frontend_type = 0;
 255     m_initRequest_apdu = 0;
 256     m_initRequest_mem = 0;
 257     m_initRequest_preferredMessageSize = 0;
 258     m_initRequest_maximumRecordSize = 0;
 259     m_initRequest_options = 0;
 260     m_initRequest_version = 0;
 261     m_initRequest_oi_negotiation_charsets = 0;
 262     m_initRequest_oi_negotiation_num_charsets = 0;
 263     m_initRequest_oi_negotiation_langs = 0;
 264     m_initRequest_oi_negotiation_num_langs = 0;
 265     m_initRequest_oi_negotiation_selected = 0;
 266     m_apdu_invalid_session = 0;
 267     m_mem_invalid_session = 0;
 268     m_s2z_odr_init = 0;
 269     m_s2z_odr_search = 0;
 270     m_s2z_init_apdu = 0;
 271     m_s2z_search_apdu = 0;
 272     m_s2z_present_apdu = 0;
 273     m_http_keepalive = 0;
 274     m_http_version = 0;
 275     m_soap_ns = 0;
 276     m_s2z_packing = Z_SRW_recordPacking_string;
 277 #if HAVE_GETTIMEOFDAY
 278     m_time_tv = xmalloc(sizeof(struct timeval));
 279     struct timeval *tv = (struct timeval *) m_time_tv;
 280     tv->tv_sec = 0;
 281     tv->tv_usec = 0;
 282 #else
 283     m_time_tv = 0;
 284 #endif
 285     m_usemarcon_ini_stage1 = 0;
 286     m_usemarcon_ini_stage2 = 0;
 287     m_usemarcon = new Yaz_usemarcon();
 288     if (!m_parent)
 289         low_socket_open();
 290     m_my_thread = 0;
 291     m_ref_count = 1;
 292     m_main_ptr_dec = false;
 293     m_peername = 0;
 294 }
 295
 296 void Yaz_Proxy::inc_ref()
 297 {
 298     m_ref_count++;
 299 }
 300
 301 Yaz_Proxy::~Yaz_Proxy()
 302 {
 303     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 304             m_bytes_sent, m_bytes_recv);
 305     nmem_destroy(m_initRequest_mem);
 306     nmem_destroy(m_mem_invalid_session);
 307     nmem_destroy(m_referenceId_mem);
 308
 309     xfree(m_proxyTarget);
 310     xfree(m_default_target);
 311     xfree(m_proxy_negotiation_charset);
 312     xfree(m_proxy_negotiation_lang);
 313     delete m_charset_converter;
 314     xfree(m_optimize);
 315
 316 #if HAVE_XSLT
 317     if (m_stylesheet_xsp)
 318         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 319 #endif
 320     xfree (m_time_tv);
 321
 322     xfree (m_peername);
 323     xfree (m_schema);
 324     xfree (m_backend_type);
 325     xfree (m_backend_charset);
 326     xfree (m_usemarcon_ini_stage1);
 327     xfree (m_usemarcon_ini_stage2);
 328     delete m_usemarcon;
 329     if (m_s2z_odr_init)
 330         odr_destroy(m_s2z_odr_init);
 331     if (m_s2z_odr_search)
 332         odr_destroy(m_s2z_odr_search);
 333     if (!m_parent)
 334         low_socket_close();
 335     if (!m_parent)
 336         delete m_my_thread;
 337     delete m_config;
 338 }
 339
 340 void Yaz_Proxy::set_debug_mode(int mode)
 341 {
 342     m_debug_mode = mode;
 343 }
 344
 345 int Yaz_Proxy::set_config(const char *config)
 346 {
 347     delete m_config;
 348     m_config = new Yaz_ProxyConfig();
 349     xfree(m_config_fname);
 350     m_config_fname = xstrdup(config);
 351     int r = m_config->read_xml(config);
 352     if (!r)
 353         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 354     return r;
 355 }
 356
 357 void Yaz_Proxy::set_default_target(const char *target)
 358 {
 359     xfree (m_default_target);
 360     m_default_target = 0;
 361     if (target)
 362         m_default_target = (char *) xstrdup (target);
 363 }
 364
 365 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang)
 366 {
 367     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
 368         "language to '%s'", m_session_str, charset?charset:"none",
 369         lang?lang:"none");
 370     xfree (m_proxy_negotiation_charset);
 371     xfree (m_proxy_negotiation_lang);
 372     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 373     if (charset)
 374         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 375     if (lang)
 376         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 377 }
 378
 379 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 380 {
 381     if (m_parent)
 382         return m_parent->check_reconfigure();
 383
 384     Yaz_ProxyConfig *cfg = m_config;
 385     if (m_reconfig_flag)
 386     {
 387         yaz_log(YLOG_LOG, "reconfigure");
 388         yaz_log_reopen();
 389         if (m_config_fname && cfg)
 390         {
 391             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 392             int r = cfg->read_xml(m_config_fname);
 393             if (r)
 394                 yaz_log(YLOG_WARN, "reconfigure failed");
 395             else
 396             {
 397                 m_log_mask = 0;
 398                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 399             }
 400         }
 401         else
 402             yaz_log(YLOG_LOG, "reconfigure");
 403         m_reconfig_flag = 0;
 404     }
 405     return cfg;
 406 }
 407
 408 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 409                                         *the_PDU_Observable, int fd)
 410 {
 411     check_reconfigure();
 412     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 413                                          m_socket_observable, this);
 414     new_proxy->m_config = 0;
 415     new_proxy->m_config_fname = 0;
 416     new_proxy->timeout(m_client_idletime);
 417     new_proxy->m_target_idletime = m_target_idletime;
 418     new_proxy->set_default_target(m_default_target);
 419     new_proxy->m_max_clients = m_max_clients;
 420     new_proxy->m_log_mask = m_log_mask;
 421     new_proxy->set_APDU_log(get_APDU_log());
 422     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 423         new_proxy->set_APDU_yazlog(1);
 424     else
 425         new_proxy->set_APDU_yazlog(0);
 426     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 427     m_session_no++;
 428     new_proxy->m_peername = xstrdup(the_PDU_Observable->getpeername());
 429     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 430              new_proxy->m_peername);
 431     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 432         m_proxy_negotiation_lang);
 433     // create thread object the first time we get an incoming connection
 434     if (!m_my_thread)
 435         m_my_thread = new Msg_Thread(m_socket_observable, 1);
 436     new_proxy->m_my_thread = m_my_thread;
 437     return new_proxy;
 438 }
 439
 440 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 441 {
 442     int oid[OID_SIZE];
 443     Z_OtherInformationUnit *oi;
 444     struct oident ent;
 445     ent.proto = PROTO_Z3950;
 446     ent.oclass = CLASS_USERINFO;
 447     ent.value = (oid_value) VAL_COOKIE;
 448     assert (oid_ent_to_oid (&ent, oid));
 449
 450     if (oid_ent_to_oid (&ent, oid) &&
 451         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 452         oi->which == Z_OtherInfo_characterInfo)
 453         return oi->information.characterInfo;
 454     return 0;
 455 }
 456 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 457 {
 458     int oid[OID_SIZE];
 459     Z_OtherInformationUnit *oi;
 460     struct oident ent;
 461     ent.proto = PROTO_Z3950;
 462     ent.oclass = CLASS_USERINFO;
 463     ent.value = (oid_value) VAL_PROXY;
 464     if (oid_ent_to_oid (&ent, oid) &&
 465         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 466         oi->which == Z_OtherInfo_characterInfo)
 467         return oi->information.characterInfo;
 468     return 0;
 469 }
 470 const char *Yaz_Proxy::load_balance(const char **url)
 471 {
 472     int zurl_in_use[MAX_ZURL_PLEX];
 473     int zurl_in_spare[MAX_ZURL_PLEX];
 474     Yaz_ProxyClient *c;
 475     int i;
 476
 477     for (i = 0; i<MAX_ZURL_PLEX; i++)
 478     {
 479         zurl_in_use[i] = 0;
 480         zurl_in_spare[i] = 0;
 481     }
 482     for (c = m_parent->m_clientPool; c; c = c->m_next)
 483     {
 484         for (i = 0; url[i]; i++)
 485             if (!strcmp(url[i], c->get_hostname()))
 486             {
 487                 zurl_in_use[i]++;
 488                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 489                     zurl_in_spare[i]++;
 490             }
 491     }
 492     int min_use = 100000;
 493     int spare_for_min = 0;
 494     int max_spare = 0;
 495     const char *ret_min = 0;
 496     const char *ret_spare = 0;
 497     for (i = 0; url[i]; i++)
 498     {
 499         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 500                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 501         if (min_use > zurl_in_use[i])
 502         {
 503             ret_min = url[i];
 504             min_use = zurl_in_use[i];
 505             spare_for_min = zurl_in_spare[i];
 506         }
 507         if (max_spare < zurl_in_spare[i])
 508         {
 509             ret_spare = url[i];
 510             max_spare = zurl_in_spare[i];
 511         }
 512     }
 513     return ret_min;
 514 }
 515
 516 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 517                                        const char *proxy_host)
 518 {
 519     assert (m_parent);
 520     Yaz_Proxy *parent = m_parent;
 521     Yaz_ProxyClient *c = m_client;
 522
 523     if (!m_proxyTarget)
 524     {
 525         const char *url[MAX_ZURL_PLEX];
 526         Yaz_ProxyConfig *cfg = check_reconfigure();
 527         if (proxy_host)
 528         {
 529             if (parent && parent->m_debug_mode)
 530             {
 531                 // only to be enabled for debugging...
 532                 if (!strcmp(proxy_host, "stop"))
 533                     exit(0);
 534             }
 535             xfree(m_default_target);
 536             m_default_target = xstrdup(proxy_host);
 537         }
 538         proxy_host = m_default_target;
 539         int client_idletime = -1;
 540         const char *cql2rpn_fname = 0;
 541         const char *negotiation_charset = 0;
 542         const char *negotiation_lang = 0;
 543         const char *query_charset = 0;
 544         url[0] = m_default_target;
 545         url[1] = 0;
 546         if (cfg)
 547         {
 548             int pre_init = 0;
 549             cfg->get_target_info(proxy_host, url, &m_bw_max,
 550                                  &m_pdu_max, &m_max_record_retrieve,
 551                                  &m_search_max, &m_connect_max,
 552                                  &m_target_idletime, &client_idletime,
 553                                  &parent->m_max_clients,
 554                                  &m_keepalive_limit_bw,
 555                                  &m_keepalive_limit_pdu,
 556                                  &pre_init,
 557                                  &cql2rpn_fname,
 558                                  &negotiation_charset,
 559                                  &negotiation_lang,
 560                                  &query_charset);
 561         }
 562         if (client_idletime != -1)
 563         {
 564             m_client_idletime = client_idletime;
 565             timeout(m_client_idletime);
 566         }
 567         if (cql2rpn_fname)
 568             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 569         if (negotiation_charset || negotiation_lang)
 570         {
 571             set_proxy_negotiation(negotiation_charset,
 572                 negotiation_lang);
 573         }
 574         m_charset_converter->set_target_query_charset(query_charset);
 575         if (!url[0])
 576         {
 577             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 578             return 0;
 579         }
 580         // we don't handle multiplexing for cookie session, so we just
 581         // pick the first one in this case (anonymous users will be able
 582         // to use any backend)
 583         if (cookie && *cookie)
 584             m_proxyTarget = (char*) xstrdup(url[0]);
 585         else
 586             m_proxyTarget = (char*) xstrdup(load_balance(url));
 587     }
 588     if (cookie && *cookie)
 589     {   // search in sessions with a cookie
 590         for (c = parent->m_clientPool; c; c = c->m_next)
 591         {
 592             assert (c->m_prev);
 593             assert (*c->m_prev == c);
 594             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 595                 !strcmp(m_proxyTarget, c->get_hostname()))
 596             {
 597                 // Found it in cache
 598                 // The following handles "cancel"
 599                 // If connection is busy (waiting for PDU) and
 600                 // we have an initRequest we can safely do re-open
 601                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 602                 {
 603                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 604                              c->get_hostname());
 605                     c->close();
 606                     c->m_init_flag = 0;
 607
 608                     c->m_last_ok = 0;
 609                     c->m_cache.clear();
 610                     c->m_last_resultCount = 0;
 611                     c->m_sr_transform = 0;
 612                     c->m_waiting = 0;
 613                     c->m_resultSetStartPoint = 0;
 614                     c->m_target_idletime = m_target_idletime;
 615                     if (c->client(m_proxyTarget))
 616                     {
 617                         delete c;
 618                         return 0;
 619                     }
 620                     c->timeout(30);
 621                 }
 622                 c->m_seqno = parent->m_seqno;
 623                 if (c->m_server && c->m_server != this)
 624                     c->m_server->m_client = 0;
 625                 c->m_server = this;
 626                 (parent->m_seqno)++;
 627                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 628                 return c;
 629             }
 630         }
 631     }
 632     else if (!c
 633              && apdu->which == Z_APDU_initRequest
 634              && !ODR_MASK_GET(apdu->u.initRequest->options,
 635                               Z_Options_negotiationModel))
 636     {
 637         // anonymous sessions without cookie.
 638         // if authentication is set it is NOT anonymous se we can't share them.
 639         // If charset and lang negotiation is use it is NOT anonymous session too.
 640         for (c = parent->m_clientPool; c; c = c->m_next)
 641         {
 642             assert(c->m_prev);
 643             assert(*c->m_prev == c);
 644             if (c->m_server == 0 && c->m_cookie == 0 &&  c->m_waiting == 0
 645                 && c->compare_idAuthentication(apdu)
 646                 && !strcmp(m_proxyTarget, c->get_hostname()))
 647             {
 648                 // found it in cache
 649                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 650                          m_session_str, parent->m_seqno, c->get_hostname());
 651
 652                 c->m_seqno = parent->m_seqno;
 653                 assert(c->m_server == 0);
 654                 c->m_server = this;
 655
 656                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 657                     c->set_APDU_yazlog(1);
 658                 else
 659                     c->set_APDU_yazlog(0);
 660
 661                 (parent->m_seqno)++;
 662
 663                 parent->pre_init();
 664
 665                 return c;
 666             }
 667         }
 668     }
 669     if (!m_client)
 670     {
 671         if (apdu->which != Z_APDU_initRequest)
 672         {
 673             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 674             return 0;
 675         }
 676         // go through list of clients - and find the lowest/oldest one.
 677         Yaz_ProxyClient *c_min = 0;
 678         int min_seq = -1;
 679         int no_of_clients = 0;
 680         if (parent->m_clientPool)
 681             yaz_log (YLOG_DEBUG, "Existing sessions");
 682         for (c = parent->m_clientPool; c; c = c->m_next)
 683         {
 684             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 685                                c->m_waiting, c->get_hostname(),
 686                                c->m_cookie ? c->m_cookie : "");
 687             no_of_clients++;
 688             if (min_seq < 0 || c->m_seqno < min_seq)
 689             {
 690                 min_seq = c->m_seqno;
 691                 c_min = c;
 692             }
 693         }
 694         if (no_of_clients >= parent->m_max_clients)
 695         {
 696             c = c_min;
 697             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 698             {
 699                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 700                          m_session_str, parent->m_max_clients, c->m_seqno);
 701                 if (c->m_server && c->m_server != this)
 702                     delete c->m_server;   // PROBLEM: m_ref_count!
 703                 c->m_server = 0;
 704             }
 705             else
 706             {
 707                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 708                          m_session_str, parent->m_max_clients,
 709                          c->m_seqno, parent->m_seqno, c->get_hostname());
 710                 xfree (c->m_cookie);
 711                 c->m_cookie = 0;
 712                 if (cookie)
 713                     c->m_cookie = xstrdup(cookie);
 714                 c->m_seqno = parent->m_seqno;
 715                 if (c->m_server && c->m_server != this)
 716                 {
 717                     c->m_server->m_client = 0;
 718                     delete c->m_server;   // PROBLEM: m_ref_count!
 719                 }
 720                 (parent->m_seqno)++;
 721                 c->m_target_idletime = m_target_idletime;
 722                 c->timeout(m_target_idletime);
 723
 724                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 725                     c->set_APDU_yazlog(1);
 726                 else
 727                     c->set_APDU_yazlog(0);
 728
 729                 return c;
 730             }
 731         }
 732         else
 733         {
 734             yaz_log (YLOG_LOG, "%sNEW %d %s",
 735                      m_session_str, parent->m_seqno, m_proxyTarget);
 736             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 737             c->m_next = parent->m_clientPool;
 738             if (c->m_next)
 739                 c->m_next->m_prev = &c->m_next;
 740             parent->m_clientPool = c;
 741             c->m_prev = &parent->m_clientPool;
 742         }
 743
 744         xfree (c->m_cookie);
 745         c->m_cookie = 0;
 746         if (cookie)
 747             c->m_cookie = xstrdup(cookie);
 748
 749         c->m_seqno = parent->m_seqno;
 750         c->m_init_flag = 0;
 751         c->m_last_resultCount = 0;
 752         c->m_last_ok = 0;
 753         c->m_cache.clear();
 754         c->m_sr_transform = 0;
 755         c->m_waiting = 0;
 756         c->m_resultSetStartPoint = 0;
 757         (parent->m_seqno)++;
 758         if (c->client(m_proxyTarget))
 759         {
 760             delete c;
 761             return 0;
 762         }
 763         c->m_target_idletime = m_target_idletime;
 764         c->timeout(30);
 765
 766         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 767             c->set_APDU_yazlog(1);
 768         else
 769             c->set_APDU_yazlog(0);
 770
 771         c->set_idAuthentication(apdu);
 772     }
 773     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 774     return c;
 775 }
 776
 777 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 778 {
 779     int i;
 780     for (i = 0; i<num; i++)
 781     {
 782         oident *ent;
 783         Z_DefaultDiagFormat *r;
 784         Z_DiagRec *p = pp[i];
 785         if (p->which != Z_DiagRec_defaultFormat)
 786         {
 787             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 788             return;
 789         }
 790         else
 791             r = p->u.defaultFormat;
 792         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 793             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 794             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 795         switch (r->which)
 796         {
 797         case Z_DefaultDiagFormat_v2Addinfo:
 798             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 799                     m_session_str,
 800                     *r->condition, diagbib1_str(*r->condition),
 801                     r->u.v2Addinfo);
 802             break;
 803         case Z_DefaultDiagFormat_v3Addinfo:
 804             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 805                     m_session_str,
 806                     *r->condition, diagbib1_str(*r->condition),
 807                     r->u.v3Addinfo);
 808             break;
 809         }
 810     }
 811 }
 812
 813 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 814 {
 815     if (!m_stylesheet_xsp || p->num_records <= 0)
 816     {
 817         return 0;  /* no XSLT to be done ... */
 818     }
 819
 820     m_stylesheet_offset = 0;
 821     m_stylesheet_nprl = p;
 822     m_stylesheet_apdu = apdu;
 823     m_timeout_mode = timeout_xsl;
 824
 825     timeout(0);
 826     return 1;
 827 }
 828
 829 void Yaz_Proxy::convert_xsl_delay()
 830 {
 831 #if HAVE_XSLT
 832     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 833     if (npr->which == Z_NamePlusRecord_databaseRecord)
 834     {
 835         Z_External *r = npr->u.databaseRecord;
 836         if (r->which == Z_External_octet)
 837         {
 838 #if 0
 839             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 840 #endif
 841             xmlDocPtr res, doc = xmlParseMemory(
 842                 (char*) r->u.octet_aligned->buf,
 843                 r->u.octet_aligned->len);
 844
 845
 846             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 847                     m_session_str, m_stylesheet_offset);
 848             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 849                                       doc, 0);
 850
 851             if (res)
 852             {
 853                 xmlChar *out_buf;
 854                 int out_len;
 855                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 856
 857                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 858                     u.databaseRecord =
 859                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 860                                  (char*) out_buf, out_len);
 861                 xmlFree(out_buf);
 862                 xmlFreeDoc(res);
 863             }
 864
 865             xmlFreeDoc(doc);
 866         }
 867     }
 868 #endif
 869     m_stylesheet_offset++;
 870     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 871     {
 872         m_timeout_mode = timeout_normal;
 873         m_stylesheet_nprl = 0;
 874 #if HAVE_XSLT
 875         if (m_stylesheet_xsp)
 876             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 877 #endif
 878         m_stylesheet_xsp = 0;
 879         timeout(m_client_idletime);
 880         send_PDU_convert(m_stylesheet_apdu);
 881     }
 882     else
 883         timeout(0);
 884 }
 885
 886 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 887 {
 888     if (m_frontend_type != VAL_NONE)
 889     {
 890         int i;
 891         for (i = 0; i < p->num_records; i++)
 892         {
 893             Z_NamePlusRecord *npr = p->records[i];
 894             if (npr->which == Z_NamePlusRecord_databaseRecord)
 895             {
 896                 Z_External *r = npr->u.databaseRecord;
 897                 if (r->which == Z_External_octet)
 898                 {
 899 #if HAVE_USEMARCON
 900                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 901                     {
 902                         if (!m_usemarcon->m_stage1)
 903                         {
 904                             m_usemarcon->m_stage1 = new CDetails();
 905                         }
 906                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 907                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 908                         int res = m_usemarcon->m_stage1->Start();
 909                         if (res == 0)
 910                         {
 911                             char *converted;
 912                             int convlen;
 913                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 914                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 915                             {
 916                                 if (!m_usemarcon->m_stage2)
 917                                 {
 918                                     m_usemarcon->m_stage2 = new CDetails();
 919                                 }
 920                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 921                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 922                                 res = m_usemarcon->m_stage2->Start();
 923                                 if (res == 0)
 924                                 {
 925                                     free(converted);
 926                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 927                                 }
 928                                 else
 929                                 {
 930                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 931                                 }
 932                             }
 933                             npr->u.databaseRecord =
 934                                 z_ext_record(odr_encode(),
 935                                              m_frontend_type,
 936                                              converted,
 937                                              strlen(converted));
 938                             free(converted);
 939                         }
 940                         else
 941                         {
 942                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 943                         }
 944                         continue;
 945                     }
 946 #endif
 947 /* HAVE_USEMARCON */
 948                     npr->u.databaseRecord =
 949                         z_ext_record(odr_encode(),
 950                                      m_frontend_type,
 951                                      (char*) r->u.octet_aligned->buf,
 952                                      r->u.octet_aligned->len);
 953                 }
 954             }
 955         }
 956     }
 957 }
 958
 959 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
 960                                         const char *backend_charset)
 961 {
 962     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
 963     int sel =   m_charset_converter->get_client_charset_selected();
 964     const char *client_record_charset =
 965         m_charset_converter->get_client_query_charset();
 966     if (sel && backend_charset && client_record_charset &&
 967         strcmp(backend_charset, client_record_charset))
 968     {
 969         int i;
 970         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
 971                                         backend_charset);
 972         yaz_marc_t mt = yaz_marc_create();
 973         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
 974         yaz_marc_iconv(mt, cd);
 975         for (i = 0; i < p->num_records; i++)
 976         {
 977             Z_NamePlusRecord *npr = p->records[i];
 978             if (npr->which == Z_NamePlusRecord_databaseRecord)
 979             {
 980                 Z_External *r = npr->u.databaseRecord;
 981                 oident *ent = oid_getentbyoid(r->direct_reference);
 982                 if (!ent || ent->value == VAL_NONE)
 983                     continue;
 984
 985                 if (ent->value == VAL_SUTRS)
 986                 {
 987                     WRBUF w = wrbuf_alloc();
 988
 989                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
 990                                       r->u.octet_aligned->len);
 991                     npr->u.databaseRecord =
 992                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
 993                                      wrbuf_len(w));
 994                     wrbuf_free(w, 1);
 995                 }
 996                 else if (ent->value == VAL_TEXT_XML)
 997                 {
 998                     ;
 999                 }
1000                 else if (r->which == Z_External_octet)
1001                 {
1002                     int rlen;
1003                     char *result;
1004                     if (yaz_marc_decode_buf(mt,
1005                                             (char*) r->u.octet_aligned->buf,
1006                                             r->u.octet_aligned->len,
1007                                             &result, &rlen))
1008                     {
1009                         npr->u.databaseRecord =
1010                             z_ext_record(odr_encode(), ent->value, result, rlen);
1011                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1012                                 m_session_str);
1013                     }
1014                 }
1015             }
1016         }
1017         if (cd)
1018             yaz_iconv_close(cd);
1019         yaz_marc_destroy(mt);
1020     }
1021     else
1022     {
1023         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
1024     }
1025 }
1026
1027 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1028                                    const char *backend_charset)
1029 {
1030     int i;
1031     if (!backend_charset)
1032         backend_charset = "MARC-8";
1033     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1034     yaz_marc_t mt = yaz_marc_create();
1035     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1036     yaz_marc_iconv(mt, cd);
1037     for (i = 0; i < p->num_records; i++)
1038     {
1039         Z_NamePlusRecord *npr = p->records[i];
1040         if (npr->which == Z_NamePlusRecord_databaseRecord)
1041         {
1042             Z_External *r = npr->u.databaseRecord;
1043             if (r->which == Z_External_OPAC)
1044             {
1045                 WRBUF w = wrbuf_alloc();
1046
1047                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
1048                 npr->u.databaseRecord = z_ext_record(
1049                     odr_encode(), VAL_TEXT_XML,
1050                     wrbuf_buf(w), wrbuf_len(w)
1051                     );
1052                 wrbuf_free(w, 1);
1053             }
1054             else if (r->which == Z_External_octet)
1055             {
1056                 int rlen;
1057                 char *result;
1058                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1059                                         r->u.octet_aligned->len,
1060                                         &result, &rlen))
1061                 {
1062                     npr->u.databaseRecord =
1063                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
1064                 }
1065             }
1066         }
1067     }
1068     if (cd)
1069         yaz_iconv_close(cd);
1070     yaz_marc_destroy(mt);
1071 }
1072
1073 void Yaz_Proxy::logtime()
1074 {
1075 #if HAVE_GETTIMEOFDAY
1076     struct timeval *tv = (struct timeval*) m_time_tv;
1077     if (tv->tv_sec)
1078     {
1079         struct timeval tv1;
1080         gettimeofday(&tv1, 0);
1081         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1082             (tv1.tv_usec - tv->tv_usec);
1083         if (diff >= 0)
1084             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1085                     diff/1000000, (diff/1000)%1000);
1086     }
1087     tv->tv_sec = 0;
1088     tv->tv_usec = 0;
1089 #endif
1090 }
1091
1092 int Yaz_Proxy::send_http_response(int code)
1093 {
1094     ODR o = odr_encode();
1095     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1096     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1097     if (m_http_version)
1098         hres->version = odr_strdup(o, m_http_version);
1099     if (m_http_keepalive)
1100         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1101     else
1102         timeout(0);
1103
1104     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1105     {
1106         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1107                  gdu_name(gdu));
1108     }
1109     int len;
1110     int r = send_GDU(gdu, &len);
1111     m_bytes_sent += len;
1112     m_bw_stat.add_bytes(len);
1113     logtime();
1114
1115     recv_GDU_more(true);
1116
1117     return r;
1118 }
1119
1120 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
1121 {
1122     ODR o = odr_encode();
1123     const char *ctype = "text/xml";
1124     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
1125     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1126     if (m_http_version)
1127         hres->version = odr_strdup(o, m_http_version);
1128     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1129     if (m_http_keepalive)
1130         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1131     else
1132         timeout(0);
1133
1134     static Z_SOAP_Handler soap_handlers[2] = {
1135 #if HAVE_XSLT
1136         {"http://www.loc.gov/zing/srw/", 0,
1137          (Z_SOAP_fun) yaz_srw_codec},
1138 #endif
1139         {0, 0, 0}
1140     };
1141
1142     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1143     soap_package->which = Z_SOAP_generic;
1144     soap_package->u.generic =
1145         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1146     soap_package->u.generic->no = 0;
1147     soap_package->u.generic->ns = soap_handlers[0].ns;
1148     soap_package->u.generic->p = (void *) srw_pdu;
1149     soap_package->ns = m_soap_ns;
1150     z_soap_codec_enc_xsl(o, &soap_package,
1151                          &hres->content_buf, &hres->content_len,
1152                          soap_handlers, 0, m_s2z_stylesheet);
1153     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1154     {
1155         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1156                  gdu_name(gdu));
1157     }
1158     int len;
1159     int r = send_GDU(gdu, &len);
1160     m_bytes_sent += len;
1161     m_bw_stat.add_bytes(len);
1162     logtime();
1163
1164     recv_GDU_more(true);
1165
1166     return r;
1167 }
1168
1169 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1170 {
1171     ODR o = odr_encode();
1172     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1173         odr_malloc(o, sizeof(*diagnostic));
1174     int num_diagnostic = 1;
1175     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1176     return send_srw_search_response(diagnostic, num_diagnostic);
1177 }
1178
1179 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1180                              Z_DefaultDiagFormat *ddf)
1181 {
1182     int bib1_code = *ddf->condition;
1183     if (bib1_code == 109)
1184         return 404;
1185     srw_res->num_diagnostics = 1;
1186     srw_res->diagnostics = (Z_SRW_diagnostic *)
1187         odr_malloc(o, sizeof(*srw_res->diagnostics));
1188     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1189                           yaz_diag_bib1_to_srw(*ddf->condition),
1190                           ddf->u.v2Addinfo);
1191     return 0;
1192 }
1193
1194 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1195 {
1196     ODR o = odr_encode();
1197     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1198     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1199
1200     srw_res->numberOfRecords = odr_intdup (o, hits);
1201     if (records && records->which == Z_Records_DBOSD)
1202     {
1203         srw_res->num_records =
1204             records->u.databaseOrSurDiagnostics->num_records;
1205         int i;
1206         srw_res->records = (Z_SRW_record *)
1207             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1208         for (i = 0; i < srw_res->num_records; i++)
1209         {
1210             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1211             if (npr->which != Z_NamePlusRecord_databaseRecord)
1212             {
1213                 srw_res->records[i].recordSchema = "diagnostic";
1214                 srw_res->records[i].recordPacking = m_s2z_packing;
1215                 srw_res->records[i].recordData_buf = "67";
1216                 srw_res->records[i].recordData_len = 2;
1217                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1218                 continue;
1219             }
1220             Z_External *r = npr->u.databaseRecord;
1221             oident *ent = oid_getentbyoid(r->direct_reference);
1222             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1223             {
1224                 srw_res->records[i].recordSchema = m_schema;
1225                 srw_res->records[i].recordPacking = m_s2z_packing;
1226                 srw_res->records[i].recordData_buf = (char*)
1227                     r->u.octet_aligned->buf;
1228                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1229                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1230             }
1231             else
1232             {
1233                 srw_res->records[i].recordSchema = "diagnostic";
1234                 srw_res->records[i].recordPacking = m_s2z_packing;
1235                 srw_res->records[i].recordData_buf = "67";
1236                 srw_res->records[i].recordData_len = 2;
1237                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1238             }
1239         }
1240     }
1241     if (records && records->which == Z_Records_NSD)
1242     {
1243         int http_code;
1244         http_code = z_to_srw_diag(odr_encode(), srw_res,
1245                                    records->u.nonSurrogateDiagnostic);
1246         if (http_code)
1247             return send_http_response(http_code);
1248     }
1249     return send_srw_response(srw_pdu);
1250
1251 }
1252
1253 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1254                                         int num_diagnostics)
1255 {
1256     ODR o = odr_encode();
1257     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1258     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1259
1260     srw_res->num_diagnostics = num_diagnostics;
1261     srw_res->diagnostics = diagnostics;
1262     return send_srw_response(srw_pdu);
1263 }
1264
1265 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1266                                         int num_diagnostics)
1267 {
1268     Yaz_ProxyConfig *cfg = check_reconfigure();
1269     if (cfg)
1270     {
1271         int len;
1272         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1273                                        m_s2z_database, &len);
1274         if (b)
1275         {
1276             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1277             Z_SRW_explainResponse *er = res->u.explain_response;
1278
1279             er->record.recordData_buf = b;
1280             er->record.recordData_len = len;
1281             er->record.recordPacking = m_s2z_packing;
1282             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1283
1284             er->diagnostics = diagnostics;
1285             er->num_diagnostics = num_diagnostics;
1286             return send_srw_response(res);
1287         }
1288     }
1289     return send_http_response(404);
1290 }
1291
1292 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1293 {
1294     if (m_http_version)
1295     {
1296         if (apdu->which == Z_APDU_initResponse)
1297         {
1298             Z_InitResponse *res = apdu->u.initResponse;
1299             if (*res->result == 0)
1300             {
1301                 send_to_srw_client_error(3, 0);
1302             }
1303             else if (!m_s2z_search_apdu)
1304             {
1305                 send_srw_explain_response(0, 0);
1306             }
1307             else
1308             {
1309                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1310             }
1311         }
1312         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1313         {
1314             m_s2z_search_apdu = 0;
1315             Z_SearchResponse *res = apdu->u.searchResponse;
1316             m_s2z_hit_count = *res->resultCount;
1317             if (res->records && res->records->which == Z_Records_NSD)
1318             {
1319                 send_to_srw_client_ok(0, res->records, 1);
1320             }
1321             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1322             {
1323                 // adjust
1324                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1325
1326                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1327                 {
1328                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1329                         > m_s2z_hit_count)
1330                         *pr->numberOfRecordsRequested =
1331                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1332                 }
1333                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1334             }
1335             else
1336             {
1337                 m_s2z_present_apdu = 0;
1338                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1339             }
1340         }
1341         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1342         {
1343             int start =
1344                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1345
1346             m_s2z_present_apdu = 0;
1347             Z_PresentResponse *res = apdu->u.presentResponse;
1348             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1349         }
1350     }
1351     else
1352     {
1353         int len = 0;
1354         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1355             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1356                      apdu_name(apdu));
1357         int r = send_Z_PDU(apdu, &len);
1358         m_bytes_sent += len;
1359         m_bw_stat.add_bytes(len);
1360         logtime();
1361         return r;
1362     }
1363     return 0;
1364 }
1365
1366 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1367 {
1368     int kill_session = 0;
1369     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1370
1371     if (new_id)
1372         *new_id = m_referenceId;
1373
1374     if (apdu->which == Z_APDU_searchResponse)
1375     {
1376         Z_SearchResponse *sr = apdu->u.searchResponse;
1377         Z_Records *p = sr->records;
1378         if (p && p->which == Z_Records_NSD)
1379         {
1380             Z_DiagRec dr, *dr_p = &dr;
1381             dr.which = Z_DiagRec_defaultFormat;
1382             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1383
1384             *sr->searchStatus = 0;
1385             display_diagrecs(&dr_p, 1);
1386         }
1387         else
1388         {
1389             if (p && p->which == Z_Records_DBOSD)
1390             {
1391                 if (m_backend_type
1392 #if HAVE_USEMARCON
1393                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1394 #endif
1395                     )
1396                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1397                 if (m_marcxml_mode == marcxml)
1398                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1399                                        m_backend_charset);
1400                 else
1401                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1402                                             m_backend_charset);
1403                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1404                     return 0;
1405
1406             }
1407             if (sr->resultCount)
1408             {
1409                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1410                         *sr->resultCount);
1411                 if (*sr->resultCount < 0)
1412                 {
1413                     m_flag_invalid_session = 1;
1414                     kill_session = 1;
1415
1416                     *sr->searchStatus = 0;
1417                     sr->records =
1418                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1419                     *sr->resultCount = 0;
1420                 }
1421             }
1422         }
1423     }
1424     else if (apdu->which == Z_APDU_presentResponse)
1425     {
1426         Z_PresentResponse *sr = apdu->u.presentResponse;
1427         Z_Records *p = sr->records;
1428         if (p && p->which == Z_Records_NSD)
1429         {
1430             Z_DiagRec dr, *dr_p = &dr;
1431             dr.which = Z_DiagRec_defaultFormat;
1432             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1433             if (*sr->presentStatus == Z_PresentStatus_success)
1434                 *sr->presentStatus = Z_PresentStatus_failure;
1435             display_diagrecs(&dr_p, 1);
1436         }
1437         if (p && p->which == Z_Records_DBOSD)
1438         {
1439             if (m_backend_type
1440 #if HAVE_USEMARCON
1441                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1442 #endif
1443                 )
1444                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1445             if (m_marcxml_mode == marcxml)
1446                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1447                                    m_backend_charset);
1448             else
1449                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1450                                         m_backend_charset);
1451             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1452                 return 0;
1453         }
1454     }
1455     else if (apdu->which == Z_APDU_initResponse)
1456     {
1457         //Get and check negotiation record
1458         //from init response.
1459         handle_charset_lang_negotiation(apdu);
1460
1461         if (m_initRequest_options)
1462         {
1463             Z_Options *nopt =
1464                 (Odr_bitmask *)odr_malloc(odr_encode(),
1465                                           sizeof(Odr_bitmask));
1466             ODR_MASK_ZERO(nopt);
1467
1468             int i;
1469             for (i = 0; i<24; i++)
1470                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1471                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1472                     ODR_MASK_SET(nopt, i);
1473             apdu->u.initResponse->options = nopt;
1474         }
1475         if (m_initRequest_version)
1476         {
1477             Z_ProtocolVersion *nopt =
1478                 (Odr_bitmask *)odr_malloc(odr_encode(),
1479                                           sizeof(Odr_bitmask));
1480             ODR_MASK_ZERO(nopt);
1481
1482             int i;
1483             for (i = 0; i<8; i++)
1484                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1485                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1486                     ODR_MASK_SET(nopt, i);
1487             apdu->u.initResponse->protocolVersion = nopt;
1488         }
1489         apdu->u.initResponse->preferredMessageSize =
1490             odr_intdup(odr_encode(),
1491                        m_client->m_initResponse_preferredMessageSize >
1492                        m_initRequest_preferredMessageSize ?
1493                        m_initRequest_preferredMessageSize :
1494                        m_client->m_initResponse_preferredMessageSize);
1495         apdu->u.initResponse->maximumRecordSize =
1496             odr_intdup(odr_encode(),
1497                        m_client->m_initResponse_maximumRecordSize >
1498                        m_initRequest_maximumRecordSize ?
1499                        m_initRequest_maximumRecordSize :
1500                        m_client->m_initResponse_maximumRecordSize);
1501     }
1502
1503     int r = send_PDU_convert(apdu);
1504     if (r)
1505         return r;
1506     if (kill_session)
1507     {
1508         delete m_client;
1509         m_client = 0;
1510         m_parent->pre_init();
1511     }
1512     return r;
1513 }
1514
1515 void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu)
1516 {
1517     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1518
1519     odr_reset(m_idAuthentication_odr);
1520     z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0);
1521     m_idAuthentication_ber_buf =
1522         odr_getbuf(m_idAuthentication_odr,
1523                    &m_idAuthentication_ber_size, 0);
1524 }
1525
1526 bool Yaz_ProxyClient::compare_idAuthentication(Z_APDU *apdu)
1527 {
1528     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1529     ODR odr = odr_createmem(ODR_ENCODE);
1530
1531     z_IdAuthentication(odr, &t, 1, 0);
1532     int sz;
1533     char *buf = odr_getbuf(odr, &sz, 0);
1534     if (buf && m_idAuthentication_ber_buf
1535         && sz == m_idAuthentication_ber_size
1536         && !memcmp(m_idAuthentication_ber_buf, buf, sz))
1537     {
1538         odr_destroy(odr);
1539         return true;
1540     }
1541     odr_destroy(odr);
1542     if (!buf && !m_idAuthentication_ber_buf)
1543         return true;
1544     return false;
1545 }
1546
1547 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1548 {
1549     int len = 0;
1550     const char *apdu_name_tmp = apdu_name(apdu);
1551     int r = send_Z_PDU(apdu, &len);
1552     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1553         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1554                  get_session_str(),
1555                  apdu_name_tmp, get_hostname(), len);
1556     m_bytes_sent += len;
1557     return r;
1558 }
1559
1560 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1561 {
1562     if (apdu->which == Z_APDU_presentRequest)
1563     {
1564         Z_PresentRequest *pr = apdu->u.presentRequest;
1565         int toget = *pr->numberOfRecordsRequested;
1566         int start = *pr->resultSetStartPoint;
1567
1568         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1569                 pr->resultSetId, start, toget);
1570
1571         if (*m_parent->m_optimize == '0')
1572             return apdu;
1573
1574         if (!m_client->m_last_resultSetId)
1575         {
1576             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1577             new_apdu->u.presentResponse->records =
1578                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1579                                                pr->resultSetId);
1580             send_to_client(new_apdu);
1581             return 0;
1582         }
1583         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1584         {
1585             if (start+toget-1 > m_client->m_last_resultCount)
1586             {
1587                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1588                 new_apdu->u.presentResponse->records =
1589                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1590                 send_to_client(new_apdu);
1591                 return 0;
1592             }
1593             Z_NamePlusRecordList *npr;
1594 #if 0
1595             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1596                     m_session_str, start, toget, yaz_z3950oid_to_str(
1597                         pr->preferredRecordSyntax, &oclass));
1598 #endif
1599             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1600                                           pr->preferredRecordSyntax,
1601                                           pr->recordComposition))
1602             {
1603                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1604                          m_session_str);
1605                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1606                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1607
1608                 new_apdu->u.presentResponse->numberOfRecordsReturned
1609                     = odr_intdup(odr_encode(), toget);
1610
1611                 new_apdu->u.presentResponse->records = (Z_Records*)
1612                     odr_malloc(odr_encode(), sizeof(Z_Records));
1613                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1614                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1615                 new_apdu->u.presentResponse->nextResultSetPosition =
1616                     odr_intdup(odr_encode(), start+toget);
1617
1618                 send_to_client(new_apdu);
1619                 return 0;
1620             }
1621         }
1622     }
1623
1624     if (apdu->which != Z_APDU_searchRequest)
1625         return apdu;
1626     Z_SearchRequest *sr = apdu->u.searchRequest;
1627     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1628     Yaz_Z_Databases this_databases;
1629
1630     this_databases.set(sr->num_databaseNames, (const char **)
1631                        sr->databaseNames);
1632
1633     this_query->set_Z_Query(sr->query);
1634
1635     char query_str[120];
1636     this_query->print(query_str, sizeof(query_str)-1);
1637     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1638
1639     if (*m_parent->m_optimize != '0' &&
1640         m_client->m_last_ok && m_client->m_last_query &&
1641         m_client->m_last_query->match(this_query) &&
1642         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1643         m_client->m_last_databases.match(this_databases))
1644     {
1645         delete this_query;
1646         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1647             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1648         {
1649             Z_NamePlusRecordList *npr;
1650             int toget = *sr->mediumSetPresentNumber;
1651             Z_RecordComposition *comp = 0;
1652
1653             if (toget > m_client->m_last_resultCount)
1654                 toget = m_client->m_last_resultCount;
1655
1656             if (sr->mediumSetElementSetNames)
1657             {
1658                 comp = (Z_RecordComposition *)
1659                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1660                 comp->which = Z_RecordComp_simple;
1661                 comp->u.simple = sr->mediumSetElementSetNames;
1662             }
1663
1664             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1665                                           sr->preferredRecordSyntax, comp))
1666             {
1667                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1668                          m_session_str);
1669                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1670                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1671                 new_apdu->u.searchResponse->resultCount =
1672                     &m_client->m_last_resultCount;
1673
1674                 new_apdu->u.searchResponse->numberOfRecordsReturned
1675                     = odr_intdup(odr_encode(), toget);
1676
1677                 new_apdu->u.searchResponse->presentStatus =
1678                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1679                 new_apdu->u.searchResponse->records = (Z_Records*)
1680                     odr_malloc(odr_encode(), sizeof(Z_Records));
1681                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1682                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1683                 new_apdu->u.searchResponse->nextResultSetPosition =
1684                     odr_intdup(odr_encode(), toget+1);
1685                 send_to_client(new_apdu);
1686                 return 0;
1687             }
1688             else
1689             {
1690                 // medium Set
1691                 // send present request (medium size)
1692                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1693                          m_session_str);
1694
1695                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1696                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1697                 pr->referenceId = sr->referenceId;
1698                 pr->resultSetId = sr->resultSetName;
1699                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1700                 *pr->numberOfRecordsRequested = toget;
1701                 pr->recordComposition = comp;
1702                 m_client->m_sr_transform = 1;
1703                 return new_apdu;
1704             }
1705         }
1706         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1707             m_client->m_last_resultCount <= 0)
1708         {
1709             // large set. Return pseudo-search response immediately
1710             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1711                      m_session_str);
1712             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1713             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1714             new_apdu->u.searchResponse->resultCount =
1715                 &m_client->m_last_resultCount;
1716             send_to_client(new_apdu);
1717             return 0;
1718         }
1719         else
1720         {
1721             Z_NamePlusRecordList *npr;
1722             int toget = m_client->m_last_resultCount;
1723             Z_RecordComposition *comp = 0;
1724             // small set
1725             // send a present request (small set)
1726
1727             if (sr->smallSetElementSetNames)
1728             {
1729                 comp = (Z_RecordComposition *)
1730                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1731                 comp->which = Z_RecordComp_simple;
1732                 comp->u.simple = sr->smallSetElementSetNames;
1733             }
1734
1735             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1736                                           sr->preferredRecordSyntax, comp))
1737             {
1738                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1739                          m_session_str);
1740                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1741                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1742                 new_apdu->u.searchResponse->resultCount =
1743                     &m_client->m_last_resultCount;
1744
1745                 new_apdu->u.searchResponse->numberOfRecordsReturned
1746                     = odr_intdup(odr_encode(), toget);
1747
1748                 new_apdu->u.searchResponse->presentStatus =
1749                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1750                 new_apdu->u.searchResponse->records = (Z_Records*)
1751                     odr_malloc(odr_encode(), sizeof(Z_Records));
1752                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1753                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1754                 new_apdu->u.searchResponse->nextResultSetPosition =
1755                     odr_intdup(odr_encode(), toget+1);
1756                 send_to_client(new_apdu);
1757                 return 0;
1758             }
1759             else
1760             {
1761                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1762                          m_session_str);
1763                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1764                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1765                 pr->referenceId = sr->referenceId;
1766                 pr->resultSetId = sr->resultSetName;
1767                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1768                 *pr->numberOfRecordsRequested = toget;
1769                 pr->recordComposition = comp;
1770                 m_client->m_sr_transform = 1;
1771                 return new_apdu;
1772             }
1773         }
1774     }
1775     else  // query doesn't match
1776     {
1777         delete m_client->m_last_query;
1778         m_client->m_last_query = this_query;
1779         m_client->m_last_ok = 0;
1780         m_client->m_cache.clear();
1781         m_client->m_resultSetStartPoint = 0;
1782
1783         xfree (m_client->m_last_resultSetId);
1784         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1785
1786         m_client->m_last_databases.set(sr->num_databaseNames,
1787                                        (const char **) sr->databaseNames);
1788     }
1789     return apdu;
1790 }
1791
1792
1793 void Yaz_Proxy::inc_request_no()
1794 {
1795     char *cp = strchr(m_session_str, ' ');
1796     m_request_no++;
1797     if (cp)
1798         sprintf(cp+1, "%d ", m_request_no);
1799 }
1800
1801 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1802 {
1803     inc_request_no();
1804
1805     m_bytes_recv += len;
1806
1807     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1808         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1809                  m_session_str, gdu_name(apdu), len);
1810
1811 #if HAVE_GETTIMEOFDAY
1812     gettimeofday((struct timeval *) m_time_tv, 0);
1813 #endif
1814     m_bw_stat.add_bytes(len);
1815     m_pdu_stat.add_bytes(1);
1816
1817     GDU *gdu = new GDU(apdu);
1818     m_in_queue.enqueue(gdu);
1819
1820     recv_GDU_more(false);
1821 }
1822
1823 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
1824 {
1825     int bw_total = m_bw_stat.get_total();
1826     int pdu_total = m_pdu_stat.get_total();
1827     int reduce = 0;
1828
1829     assert(m_timeout_mode == timeout_busy);
1830     assert(m_timeout_gdu == 0);
1831
1832     if (m_bw_max)
1833     {
1834         if (bw_total > m_bw_max)
1835         {
1836             reduce = (bw_total/m_bw_max);
1837         }
1838     }
1839     if (m_pdu_max)
1840     {
1841         if (pdu_total > m_pdu_max)
1842         {
1843             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1844             reduce = (reduce > nreduce) ? reduce : nreduce;
1845         }
1846     }
1847     m_http_version = 0;
1848
1849 #if 0
1850     /* uncomment to force a big reduce */
1851     m_timeout_mode = timeout_reduce;
1852     m_timeout_gdu = gdu;
1853     timeout(3);       // call us reduce seconds later
1854     return;
1855 #endif
1856     if (reduce)
1857     {
1858         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1859                 m_session_str, reduce, bw_total, pdu_total,
1860                 m_bw_max, m_pdu_max);
1861
1862         m_timeout_mode = timeout_reduce;
1863         m_timeout_gdu = gdu;
1864         timeout(reduce);       // call us reduce seconds later
1865     }
1866     else
1867         recv_GDU_normal(gdu);
1868 }
1869
1870 void Yaz_Proxy::recv_GDU_more(bool normal)
1871 {
1872     GDU *g;
1873     if (normal && m_timeout_mode == timeout_busy)
1874         m_timeout_mode = timeout_normal;
1875     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
1876     {
1877         m_timeout_mode = timeout_busy;
1878         recv_GDU_reduce(g);
1879     }
1880 }
1881
1882 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
1883 {
1884     Z_GDU *apdu = 0;
1885     gdu->move_away_gdu(odr_decode(), &apdu);
1886     delete gdu;
1887
1888     if (apdu->which == Z_GDU_Z3950)
1889         handle_incoming_Z_PDU(apdu->u.z3950);
1890     else if (apdu->which == Z_GDU_HTTP_Request)
1891         handle_incoming_HTTP(apdu->u.HTTP_Request);
1892 }
1893
1894 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1895 {
1896     if (m_max_record_retrieve)
1897     {
1898         if (apdu->which == Z_APDU_presentRequest)
1899         {
1900             Z_PresentRequest *pr = apdu->u.presentRequest;
1901             if (pr->numberOfRecordsRequested &&
1902                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1903                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1904         }
1905     }
1906 }
1907
1908 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1909 {
1910     if (apdu->which == Z_APDU_initRequest)
1911     {
1912         yaz_log(YLOG_LOG, "%shandle_charset_lang_negotiation",
1913                 m_session_str);
1914         if (m_initRequest_options &&
1915             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1916             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1917         {
1918             // There is no negotiation proposal from
1919             // client's side. OK. The proxy negotiation
1920             // in use, only.
1921             Z_InitRequest *initRequest = apdu->u.initRequest;
1922             Z_OtherInformation **otherInfo;
1923             Z_OtherInformationUnit *oi;
1924             get_otherInfoAPDU(apdu, &otherInfo);
1925             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1926             if (oi)
1927             {
1928                 ODR_MASK_SET(initRequest->options,
1929                     Z_Options_negotiationModel);
1930                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1931                 oi->information.externallyDefinedInfo =
1932                 yaz_set_proposal_charneg(odr_encode(),
1933                     (const char**)&m_proxy_negotiation_charset,
1934                     m_proxy_negotiation_charset ? 1:0,
1935                     (const char**)&m_proxy_negotiation_lang,
1936                     m_proxy_negotiation_lang ? 1:0,
1937                     1);
1938             }
1939         }
1940         else if (m_initRequest_options &&
1941                  ODR_MASK_GET(m_initRequest_options,
1942                               Z_Options_negotiationModel) &&
1943                  m_charset_converter->get_target_query_charset())
1944         {
1945             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1946                     m_session_str,
1947                     m_charset_converter->get_target_query_charset());
1948             Z_InitRequest *initRequest = apdu->u.initRequest;
1949             Z_CharSetandLanguageNegotiation *negotiation =
1950                 yaz_get_charneg_record (initRequest->otherInfo);
1951             if (negotiation &&
1952                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1953             {
1954                 NMEM nmem = nmem_create();
1955                 char **charsets = 0;
1956                 int num_charsets = 0;
1957                 char **langs = 0;
1958                 int num_langs = 0;
1959                 int selected = 0;
1960                 yaz_get_proposal_charneg (nmem, negotiation,
1961                                           &charsets, &num_charsets,
1962                                           &langs, &num_langs, &selected);
1963                 int i;
1964                 for (i = 0; i<num_charsets; i++)
1965                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
1966                             charsets[i]);
1967                 for (i = 0; i<num_langs; i++)
1968                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
1969                             langs[i]);
1970
1971                 const char *t_charset =
1972                     m_charset_converter->get_target_query_charset();
1973                 // sweep through charsets and pick the first supported
1974                 // conversion
1975                 for (i = 0; i<num_charsets; i++)
1976                 {
1977                     const char *c_charset = charsets[i];
1978                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
1979                         break;
1980                 }
1981                 if (i != num_charsets)
1982                 {
1983                     // got one .. set up ODR for reverse direction
1984                     const char *c_charset = charsets[i];
1985                     odr_set_charset(odr_encode(), c_charset, t_charset);
1986                     m_charset_converter->set_client_query_charset(c_charset);
1987                     m_charset_converter->set_client_charset_selected(selected);
1988                 }
1989                 nmem_destroy(nmem);
1990                 ODR_MASK_CLEAR(m_initRequest_options,
1991                                Z_Options_negotiationModel);
1992                 yaz_del_charneg_record(&initRequest->otherInfo);
1993             }
1994             else
1995             {
1996                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
1997                         m_session_str);
1998             }
1999         }
2000     }
2001     else if (apdu->which == Z_APDU_initResponse)
2002     {
2003         Z_InitResponse *initResponse = apdu->u.initResponse;
2004         Z_OtherInformation **otherInfo;
2005
2006         if (ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel))
2007         {
2008             char *charset = 0;
2009             char *lang = 0;
2010             int selected = 0;
2011
2012             get_otherInfoAPDU(apdu, &otherInfo);
2013
2014             if (!otherInfo && !(*otherInfo))
2015                 return;
2016
2017             Z_CharSetandLanguageNegotiation *charneg =
2018                 yaz_get_charneg_record(*otherInfo);
2019
2020             if (!charneg)
2021                 return;
2022
2023             yaz_get_response_charneg(m_referenceId_mem, charneg,
2024                 &charset, &lang, &selected);
2025
2026             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
2027                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
2028
2029             if (m_initRequest_options &&
2030                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
2031             {
2032                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
2033                     m_session_str);
2034             }
2035             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2036             {
2037                 // negotiation-charset, negotiation-lang
2038                 // elements of config file in use.
2039
2040                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2041                     m_session_str);
2042
2043                 // clear negotiation option.
2044                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2045
2046                 // Delete negotiation (charneg-3) entry.
2047                 yaz_del_charneg_record(otherInfo);
2048             }
2049         }
2050         else
2051         {
2052             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2053             {
2054                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2055                         m_session_str);
2056             }
2057             else if (m_charset_converter->get_client_query_charset())
2058             {
2059                 Z_OtherInformation **otherInfo;
2060                 Z_OtherInformationUnit *oi;
2061                 get_otherInfoAPDU(apdu, &otherInfo);
2062                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2063                 if (oi)
2064                 {
2065                     ODR_MASK_SET(initResponse->options,
2066                                  Z_Options_negotiationModel);
2067                     if (m_initRequest_options)
2068                         ODR_MASK_SET(m_initRequest_options,
2069                                      Z_Options_negotiationModel);
2070
2071                     oi->which = Z_OtherInfo_externallyDefinedInfo;
2072                     oi->information.externallyDefinedInfo =
2073                         yaz_set_response_charneg(
2074                             odr_encode(),
2075                             m_charset_converter->get_client_query_charset(),
2076                             0 /* no lang */,
2077                             m_charset_converter->get_client_charset_selected());
2078                 }
2079             }
2080         }
2081     }
2082 }
2083
2084 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2085                                                      int error,
2086                                                      const char *addinfo)
2087 {
2088     Z_Records *rec = (Z_Records *)
2089         odr_malloc (odr, sizeof(*rec));
2090     int *err = (int *)
2091         odr_malloc (odr, sizeof(*err));
2092     Z_DiagRec *drec = (Z_DiagRec *)
2093         odr_malloc (odr, sizeof(*drec));
2094     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2095         odr_malloc (odr, sizeof(*dr));
2096     *err = error;
2097     rec->which = Z_Records_NSD;
2098     rec->u.nonSurrogateDiagnostic = dr;
2099     dr->diagnosticSetId =
2100         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
2101     dr->condition = err;
2102     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2103     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2104     return rec;
2105 }
2106
2107 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2108 {
2109     if (apdu->which == Z_APDU_searchRequest &&
2110         apdu->u.searchRequest->query &&
2111         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2112         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2113     {
2114         Z_RPNQuery *rpnquery = 0;
2115         Z_SearchRequest *sr = apdu->u.searchRequest;
2116         char *addinfo = 0;
2117
2118         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2119                 sr->query->u.type_104->u.cql);
2120
2121         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2122                                           &rpnquery, odr_encode(),
2123                                           &addinfo);
2124         if (r == -3)
2125             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2126         else if (r)
2127         {
2128             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2129             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2130
2131             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2132             new_apdu->u.searchResponse->records =
2133                 create_nonSurrogateDiagnostics(odr_encode(),
2134                                                yaz_diag_srw_to_bib1(r),
2135                                                addinfo);
2136             *new_apdu->u.searchResponse->searchStatus = 0;
2137
2138             send_to_client(new_apdu);
2139
2140             return 0;
2141         }
2142         else
2143         {
2144             sr->query->which = Z_Query_type_1;
2145             sr->query->u.type_1 = rpnquery;
2146         }
2147         return apdu;
2148     }
2149     return apdu;
2150 }
2151
2152 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2153 {
2154     if (apdu->which == Z_APDU_searchRequest &&
2155         apdu->u.searchRequest->query)
2156     {
2157         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2158             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2159         {
2160             if (m_http_version)
2161                 m_charset_converter->set_client_query_charset("UTF-8");
2162             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2163             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2164         }
2165     }
2166     return apdu;
2167 }
2168
2169
2170 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2171 {
2172     if (apdu->which == Z_APDU_searchRequest)
2173     {
2174         Z_SearchRequest *sr = apdu->u.searchRequest;
2175         int err = 0;
2176         char *addinfo = 0;
2177
2178         Yaz_ProxyConfig *cfg = check_reconfigure();
2179         if (cfg)
2180             err = cfg->check_query(odr_encode(), m_default_target,
2181                                    sr->query, &addinfo);
2182         if (err)
2183         {
2184             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2185
2186             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2187             new_apdu->u.searchResponse->records =
2188                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2189             *new_apdu->u.searchResponse->searchStatus = 0;
2190
2191             send_to_client(new_apdu);
2192
2193             return 0;
2194         }
2195     }
2196     return apdu;
2197 }
2198
2199 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2200 {
2201     if (apdu->which != Z_APDU_initRequest)
2202         return 1;  // pass if no init request
2203     Z_InitRequest *req = apdu->u.initRequest;
2204
2205     Yaz_ProxyConfig *cfg = check_reconfigure();
2206     if (!cfg)
2207         return 1;  // pass if no config
2208
2209     int ret;
2210     if (req->idAuthentication == 0)
2211     {
2212         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2213                                          m_peername);
2214     }
2215     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2216     {
2217         ret = cfg->client_authentication(
2218             m_default_target,
2219             req->idAuthentication->u.idPass->userId,
2220             req->idAuthentication->u.idPass->groupId,
2221             req->idAuthentication->u.idPass->password,
2222             m_peername);
2223     }
2224     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2225     {
2226         char user[64], pass[64];
2227         *user = '\0';
2228         *pass = '\0';
2229         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2230         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2231                                          m_peername);
2232     }
2233     else
2234         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2235                                          m_peername);
2236     return ret;
2237 }
2238
2239 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2240 {
2241     m_marcxml_mode = none;
2242     if (apdu->which == Z_APDU_searchRequest)
2243     {
2244         Z_SearchRequest *sr = apdu->u.searchRequest;
2245         int err = 0;
2246         char *addinfo = 0;
2247         Yaz_ProxyConfig *cfg = check_reconfigure();
2248
2249         Z_RecordComposition rc_temp, *rc = 0;
2250         if (sr->smallSetElementSetNames)
2251         {
2252             rc_temp.which = Z_RecordComp_simple;
2253             rc_temp.u.simple = sr->smallSetElementSetNames;
2254             rc = &rc_temp;
2255         }
2256
2257         if (sr->preferredRecordSyntax)
2258         {
2259             struct oident *ent;
2260             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2261             m_frontend_type = ent->value;
2262         }
2263         else
2264             m_frontend_type = VAL_NONE;
2265
2266         char *stylesheet_name = 0;
2267         if (cfg)
2268             err = cfg->check_syntax(odr_encode(),
2269                                     m_default_target,
2270                                     sr->preferredRecordSyntax, rc,
2271                                     &addinfo, &stylesheet_name, &m_schema,
2272                                     &m_backend_type, &m_backend_charset,
2273                                     &m_usemarcon_ini_stage1,
2274                                     &m_usemarcon_ini_stage2);
2275         if (stylesheet_name)
2276         {
2277             m_parent->low_socket_close();
2278
2279 #if HAVE_XSLT
2280             if (m_stylesheet_xsp)
2281                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2282             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2283                                                        stylesheet_name);
2284 #endif
2285             m_stylesheet_offset = 0;
2286             xfree(stylesheet_name);
2287
2288             m_parent->low_socket_open();
2289         }
2290         if (err == -1)
2291         {
2292             sr->smallSetElementSetNames = 0;
2293             sr->mediumSetElementSetNames = 0;
2294             m_marcxml_mode = marcxml;
2295             if (m_backend_type)
2296             {
2297
2298                 sr->preferredRecordSyntax =
2299                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2300                                         m_backend_type);
2301             }
2302             else
2303                 sr->preferredRecordSyntax =
2304                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2305                                            VAL_USMARC);
2306         }
2307         else if (err)
2308         {
2309             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2310
2311             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2312             new_apdu->u.searchResponse->records =
2313                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2314             *new_apdu->u.searchResponse->searchStatus = 0;
2315
2316             send_to_client(new_apdu);
2317
2318             return 0;
2319         }
2320         else if (m_backend_type)
2321         {
2322             sr->preferredRecordSyntax =
2323                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2324         }
2325     }
2326     else if (apdu->which == Z_APDU_presentRequest)
2327     {
2328         Z_PresentRequest *pr = apdu->u.presentRequest;
2329         int err = 0;
2330         char *addinfo = 0;
2331         Yaz_ProxyConfig *cfg = check_reconfigure();
2332
2333         if (pr->preferredRecordSyntax)
2334         {
2335             struct oident *ent;
2336             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2337             m_frontend_type = ent->value;
2338         }
2339         else
2340             m_frontend_type = VAL_NONE;
2341
2342         char *stylesheet_name = 0;
2343         if (cfg)
2344             err = cfg->check_syntax(odr_encode(), m_default_target,
2345                                     pr->preferredRecordSyntax,
2346                                     pr->recordComposition,
2347                                     &addinfo, &stylesheet_name, &m_schema,
2348                                     &m_backend_type, &m_backend_charset,
2349                                     &m_usemarcon_ini_stage1,
2350                                     &m_usemarcon_ini_stage2
2351                                     );
2352         if (stylesheet_name)
2353         {
2354             m_parent->low_socket_close();
2355
2356 #if HAVE_XSLT
2357             if (m_stylesheet_xsp)
2358                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2359             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2360                                                        stylesheet_name);
2361 #endif
2362             m_stylesheet_offset = 0;
2363             xfree(stylesheet_name);
2364
2365             m_parent->low_socket_open();
2366         }
2367         if (err == -1)
2368         {
2369             pr->recordComposition = 0;
2370             m_marcxml_mode = marcxml;
2371             if (m_backend_type)
2372             {
2373
2374                 pr->preferredRecordSyntax =
2375                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2376                                         m_backend_type);
2377             }
2378             else
2379                 pr->preferredRecordSyntax =
2380                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2381                                            VAL_USMARC);
2382         }
2383         else if (err)
2384         {
2385             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2386
2387             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2388             new_apdu->u.presentResponse->records =
2389                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2390             *new_apdu->u.presentResponse->presentStatus =
2391                 Z_PresentStatus_failure;
2392
2393             send_to_client(new_apdu);
2394
2395             return 0;
2396         }
2397         else if (m_backend_type)
2398         {
2399             pr->preferredRecordSyntax =
2400                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2401         }
2402     }
2403     return apdu;
2404 }
2405
2406 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2407 {
2408     if (!schema)
2409         return 0;
2410     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2411         odr_malloc(o, sizeof(Z_ElementSetNames));
2412     esn->which = Z_ElementSetNames_generic;
2413     esn->u.generic = odr_strdup(o, schema);
2414     return esn;
2415 }
2416
2417 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2418 {
2419     const char *t = 0;
2420     Yaz_ProxyConfig *cfg = check_reconfigure();
2421     if (cfg)
2422         t = cfg->get_explain_name(db, backend_db);
2423
2424     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2425     {
2426         releaseClient();
2427     }
2428
2429     if (t)
2430     {
2431         xfree(m_default_target);
2432         m_default_target = xstrdup(t);
2433     }
2434 }
2435
2436 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2437 {
2438     struct stat sbuf;
2439     if (strcmp(hreq->method, "GET"))
2440         return 0;
2441     if (hreq->path[0] != '/')
2442     {
2443         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2444         return 0;
2445     }
2446     const char *cp = hreq->path;
2447     while (*cp)
2448     {
2449         if (*cp == '/' && strchr("/.", cp[1]))
2450         {
2451             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2452             return 0;
2453         }
2454         cp++;
2455     }
2456     const char *fname = hreq->path+1;
2457     if (stat(fname, &sbuf))
2458     {
2459         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2460         return 0;
2461     }
2462     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2463     {
2464         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2465         return 0;
2466     }
2467     if (sbuf.st_size > (off_t) 1000000)
2468     {
2469         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2470         return 0;
2471     }
2472
2473     ODR o = odr_encode();
2474     Yaz_ProxyConfig *cfg = check_reconfigure();
2475     const char *ctype = cfg->check_mime_type(fname);
2476     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2477     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2478     if (m_http_version)
2479         hres->version = odr_strdup(o, m_http_version);
2480     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2481     if (m_http_keepalive)
2482         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2483     else
2484         timeout(0);
2485
2486     hres->content_len = sbuf.st_size;
2487     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2488     FILE *f = fopen(fname, "rb");
2489     if (f)
2490     {
2491         fread(hres->content_buf, 1, hres->content_len, f);
2492         fclose(f);
2493     }
2494     else
2495     {
2496         return 0;
2497     }
2498     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2499     {
2500         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2501                  fname);
2502     }
2503     int len;
2504     send_GDU(gdu, &len);
2505     recv_GDU_more(true);
2506     return 1;
2507 }
2508
2509 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2510 {
2511     if (m_s2z_odr_init)
2512     {
2513         odr_destroy(m_s2z_odr_init);
2514         m_s2z_odr_init = 0;
2515     }
2516     if (m_s2z_odr_search)
2517     {
2518         odr_destroy(m_s2z_odr_search);
2519         m_s2z_odr_search = 0;
2520     }
2521
2522     m_http_keepalive = 0;
2523     m_http_version = 0;
2524     if (!strcmp(hreq->version, "1.0"))
2525     {
2526         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2527         if (v && !strcmp(v, "Keep-Alive"))
2528             m_http_keepalive = 1;
2529         else
2530             m_http_keepalive = 0;
2531         m_http_version = "1.0";
2532     }
2533     else
2534     {
2535         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2536         if (v && !strcmp(v, "close"))
2537             m_http_keepalive = 0;
2538         else
2539             m_http_keepalive = 1;
2540         m_http_version = "1.1";
2541     }
2542
2543     Z_SRW_PDU *srw_pdu = 0;
2544     Z_SOAP *soap_package = 0;
2545     char *charset = 0;
2546     Z_SRW_diagnostic *diagnostic = 0;
2547     int num_diagnostic = 0;
2548
2549     if (file_access(hreq))
2550     {
2551         return;
2552     }
2553     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2554                             &charset) == 0
2555              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2556                                &charset, &diagnostic, &num_diagnostic) == 0)
2557     {
2558         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2559         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2560         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2561         m_s2z_init_apdu = 0;
2562         m_s2z_search_apdu = 0;
2563         m_s2z_present_apdu = 0;
2564
2565         m_s2z_stylesheet = 0;
2566
2567         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2568         {
2569             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2570
2571             const char *backend_db = srw_req->database;
2572             srw_get_client(srw_req->database, &backend_db);
2573
2574             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2575             // recordXPath unsupported.
2576             if (srw_req->recordXPath)
2577             {
2578                 yaz_add_srw_diagnostic(odr_decode(),
2579                                        &diagnostic, &num_diagnostic,
2580                                        72, 0);
2581             }
2582             // sort unsupported
2583             if (srw_req->sort_type != Z_SRW_sort_type_none)
2584             {
2585                 yaz_add_srw_diagnostic(odr_decode(),
2586                                        &diagnostic, &num_diagnostic,
2587                                        80, 0);
2588             }
2589             // save stylesheet
2590             if (srw_req->stylesheet)
2591                 m_s2z_stylesheet =
2592                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2593
2594             // set packing for response records ..
2595             if (srw_req->recordPacking &&
2596                 !strcmp(srw_req->recordPacking, "xml"))
2597                 m_s2z_packing = Z_SRW_recordPacking_XML;
2598             else
2599                 m_s2z_packing = Z_SRW_recordPacking_string;
2600
2601             if (num_diagnostic)
2602             {
2603                 Z_SRW_PDU *srw_pdu =
2604                     yaz_srw_get(odr_encode(),
2605                                 Z_SRW_searchRetrieve_response);
2606                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2607
2608                 srw_res->diagnostics = diagnostic;
2609                 srw_res->num_diagnostics = num_diagnostic;
2610                 send_srw_response(srw_pdu);
2611                 return;
2612             }
2613
2614             // prepare search PDU
2615             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2616                                           Z_APDU_searchRequest);
2617             Z_SearchRequest *z_searchRequest =
2618                 m_s2z_search_apdu->u.searchRequest;
2619
2620             z_searchRequest->num_databaseNames = 1;
2621             z_searchRequest->databaseNames = (char**)
2622                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2623             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2624                                                            backend_db);
2625
2626             // query transformation
2627             Z_Query *query = (Z_Query *)
2628                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2629             z_searchRequest->query = query;
2630
2631             if (srw_req->query_type == Z_SRW_query_type_cql)
2632             {
2633                 Z_External *ext = (Z_External *)
2634                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2635                 ext->direct_reference =
2636                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2637                 ext->indirect_reference = 0;
2638                 ext->descriptor = 0;
2639                 ext->which = Z_External_CQL;
2640                 ext->u.cql = srw_req->query.cql;
2641
2642                 query->which = Z_Query_type_104;
2643                 query->u.type_104 =  ext;
2644             }
2645             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2646             {
2647                 Z_RPNQuery *RPNquery;
2648                 YAZ_PQF_Parser pqf_parser;
2649
2650                 pqf_parser = yaz_pqf_create ();
2651
2652                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2653                                           srw_req->query.pqf);
2654                 if (!RPNquery)
2655                 {
2656                     const char *pqf_msg;
2657                     size_t off;
2658                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2659                     int ioff = off;
2660                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
2661                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2662
2663                     send_to_srw_client_error(10, 0);
2664                     return;
2665                 }
2666                 query->which = Z_Query_type_1;
2667                 query->u.type_1 =  RPNquery;
2668
2669                 yaz_pqf_destroy (pqf_parser);
2670             }
2671             else
2672             {
2673                 send_to_srw_client_error(7, "query");
2674                 return;
2675             }
2676
2677             // present
2678             m_s2z_present_apdu = 0;
2679             int max = 0;
2680             if (srw_req->maximumRecords)
2681                 max = *srw_req->maximumRecords;
2682             int start = 1;
2683             if (srw_req->startRecord)
2684                 start = *srw_req->startRecord;
2685             if (max > 0)
2686             {
2687                 // Some backend, such as Voyager doesn't honor piggyback
2688                 // So we use present always (0 &&).
2689                 if (0 && start <= 1)  // Z39.50 piggyback
2690                 {
2691                     *z_searchRequest->smallSetUpperBound = max;
2692                     *z_searchRequest->mediumSetPresentNumber = max;
2693                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2694
2695                     z_searchRequest->preferredRecordSyntax =
2696                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2697                                                VAL_TEXT_XML);
2698                     if (srw_req->recordSchema)
2699                     {
2700                         z_searchRequest->smallSetElementSetNames =
2701                             z_searchRequest->mediumSetElementSetNames =
2702                             mk_esn_from_schema(m_s2z_odr_search,
2703                                                srw_req->recordSchema);
2704                     }
2705                 }
2706                 else   // Z39.50 present
2707                 {
2708                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2709                                                    Z_APDU_presentRequest);
2710                     Z_PresentRequest *z_presentRequest =
2711                         m_s2z_present_apdu->u.presentRequest;
2712                     *z_presentRequest->resultSetStartPoint = start;
2713                     *z_presentRequest->numberOfRecordsRequested = max;
2714                     z_presentRequest->preferredRecordSyntax =
2715                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2716                                                VAL_TEXT_XML);
2717                     if (srw_req->recordSchema)
2718                     {
2719                         z_presentRequest->recordComposition =
2720                             (Z_RecordComposition *)
2721                             odr_malloc(m_s2z_odr_search,
2722                                        sizeof(Z_RecordComposition));
2723                         z_presentRequest->recordComposition->which =
2724                             Z_RecordComp_simple;
2725                         z_presentRequest->recordComposition->u.simple =
2726                             mk_esn_from_schema(m_s2z_odr_search,
2727                                                srw_req->recordSchema);
2728                     }
2729                 }
2730             }
2731             if (!m_client)
2732             {
2733                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2734                                             Z_APDU_initRequest);
2735
2736                 // prevent m_initRequest_apdu memory from being grabbed
2737                 // in Yaz_Proxy::handle_incoming_Z_PDU
2738                 m_initRequest_apdu = m_s2z_init_apdu;
2739                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2740                 return;
2741             }
2742             else
2743             {
2744                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2745                 return;
2746             }
2747         }
2748         else if (srw_pdu->which == Z_SRW_explain_request)
2749         {
2750             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2751
2752             const char *backend_db = srw_req->database;
2753             srw_get_client(srw_req->database, &backend_db);
2754
2755             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2756
2757             // save stylesheet
2758             if (srw_req->stylesheet)
2759                 m_s2z_stylesheet =
2760                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2761
2762             if (srw_req->recordPacking &&
2763                 !strcmp(srw_req->recordPacking, "xml"))
2764                 m_s2z_packing = Z_SRW_recordPacking_XML;
2765             else
2766                 m_s2z_packing = Z_SRW_recordPacking_string;
2767
2768             if (num_diagnostic)
2769             {
2770                 send_srw_explain_response(diagnostic, num_diagnostic);
2771                 return;
2772             }
2773
2774             if (!m_client)
2775             {
2776                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2777                                             Z_APDU_initRequest);
2778
2779                 // prevent m_initRequest_apdu memory from being grabbed
2780                 // in Yaz_Proxy::handle_incoming_Z_PDU
2781                 m_initRequest_apdu = m_s2z_init_apdu;
2782                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2783             }
2784             else
2785                 send_srw_explain_response(0, 0);
2786             return;
2787         }
2788         else if (srw_pdu->which == Z_SRW_scan_request)
2789         {
2790             m_s2z_database = odr_strdup(m_s2z_odr_init,
2791                                         srw_pdu->u.scan_request->database);
2792
2793             yaz_add_srw_diagnostic(odr_decode(),
2794                                    &diagnostic, &num_diagnostic,
2795                                    4, "scan");
2796             Z_SRW_PDU *srw_pdu =
2797                 yaz_srw_get(odr_encode(),
2798                             Z_SRW_scan_response);
2799             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2800
2801             srw_res->diagnostics = diagnostic;
2802             srw_res->num_diagnostics = num_diagnostic;
2803             send_srw_response(srw_pdu);
2804             return;
2805         }
2806         else
2807         {
2808             m_s2z_database = 0;
2809
2810             send_to_srw_client_error(4, 0);
2811         }
2812     }
2813     send_http_response(400);
2814 }
2815
2816 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2817 {
2818     Z_OtherInformation **oi;
2819     get_otherInfoAPDU(apdu, &oi);
2820
2821     if (apdu->u.initRequest->implementationId)
2822         yaz_log(YLOG_LOG, "%simplementationId: %s",
2823                 m_session_str, apdu->u.initRequest->implementationId);
2824     if (apdu->u.initRequest->implementationName)
2825         yaz_log(YLOG_LOG, "%simplementationName: %s",
2826                 m_session_str, apdu->u.initRequest->implementationName);
2827     if (apdu->u.initRequest->implementationVersion)
2828         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2829                 m_session_str, apdu->u.initRequest->implementationVersion);
2830     if (m_initRequest_apdu == 0)
2831     {
2832         if (m_initRequest_mem)
2833             nmem_destroy(m_initRequest_mem);
2834
2835         m_initRequest_apdu = apdu;
2836         m_initRequest_mem = odr_extract_mem(odr_decode());
2837
2838         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2839             preferredMessageSize;
2840         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2841         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2842             maximumRecordSize;
2843         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2844
2845         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2846             yaz_get_charneg_record(*oi);
2847
2848         // Save proposal charsets and langs.
2849         if (ODR_MASK_GET(apdu->u.initRequest->options,
2850                          Z_Options_negotiationModel)
2851             && charSetandLangRecord)
2852         {
2853
2854             yaz_get_proposal_charneg(m_referenceId_mem,
2855                                      charSetandLangRecord,
2856                                      &m_initRequest_oi_negotiation_charsets,
2857                                      &m_initRequest_oi_negotiation_num_charsets,
2858                                      &m_initRequest_oi_negotiation_langs,
2859                                      &m_initRequest_oi_negotiation_num_langs,
2860                                      &m_initRequest_oi_negotiation_selected);
2861
2862             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2863             {
2864                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2865                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2866                         m_initRequest_oi_negotiation_charsets[i]:"none");
2867             }
2868             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2869             {
2870                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2871                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2872                         m_initRequest_oi_negotiation_langs[i]:"none");
2873             }
2874             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2875                     m_session_str, m_initRequest_oi_negotiation_selected);
2876         }
2877         // save init options for the response..
2878         m_initRequest_options = apdu->u.initRequest->options;
2879
2880         apdu->u.initRequest->options =
2881             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2882                                        sizeof(Odr_bitmask));
2883         ODR_MASK_ZERO(apdu->u.initRequest->options);
2884         int i;
2885         for (i = 0; i<= 24; i++)
2886             ODR_MASK_SET(apdu->u.initRequest->options, i);
2887         // check negotiation option
2888         if (!ODR_MASK_GET(m_initRequest_options,
2889                           Z_Options_negotiationModel))
2890         {
2891             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2892                            Z_Options_negotiationModel);
2893         }
2894         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2895                        Z_Options_concurrentOperations);
2896         // make new version
2897         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2898         apdu->u.initRequest->protocolVersion =
2899             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2900                                        sizeof(Odr_bitmask));
2901         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2902
2903         for (i = 0; i<= 8; i++)
2904             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2905     }
2906     if (m_client->m_init_flag)
2907     {
2908         if (handle_init_response_for_invalid_session(apdu))
2909             return;
2910         if (m_client->m_initResponse)
2911         {
2912             Z_APDU *apdu2 = m_client->m_initResponse;
2913             apdu2->u.initResponse->otherInfo = 0;
2914             if (m_client->m_cookie && *m_client->m_cookie)
2915                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2916                                            m_client->m_cookie);
2917             apdu2->u.initResponse->referenceId =
2918                 apdu->u.initRequest->referenceId;
2919             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2920             apdu2->u.initResponse->protocolVersion =
2921                 m_client->m_initResponse_version;
2922
2923             send_to_client(apdu2);
2924             m_timeout_mode = timeout_normal;
2925             return;
2926         }
2927     }
2928     m_client->m_init_flag = 1;
2929
2930 #if USE_AUTH_MSG
2931     yaz_log(YLOG_LOG, "%suse_auth_msg", m_session_str);
2932     Auth_Msg *m = new Auth_Msg;
2933     m->m_proxy = this;
2934     z_APDU(odr_encode(), &apdu, 0, "encode");
2935     char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
2936     m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
2937     memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
2938     odr_reset(odr_encode());
2939
2940     inc_ref();
2941     m_my_thread->put(m);
2942 #else
2943     int ret = handle_authentication(apdu);
2944     result_authentication(apdu, ret);
2945 #endif
2946 }
2947
2948 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2949 {
2950     Z_ReferenceId **refid = get_referenceIdP(apdu);
2951     nmem_reset(m_referenceId_mem);
2952     if (refid && *refid)
2953     {
2954         m_referenceId = (Z_ReferenceId *)
2955             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2956         m_referenceId->len = m_referenceId->size = (*refid)->len;
2957         m_referenceId->buf = (unsigned char *)
2958             nmem_malloc(m_referenceId_mem, (*refid)->len);
2959         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2960     }
2961     else
2962         m_referenceId = 0;
2963
2964     if (!m_client && m_flag_invalid_session)
2965     {
2966         // Got request for a session that is invalid..
2967         m_apdu_invalid_session = apdu; // save package
2968         m_mem_invalid_session = odr_extract_mem(odr_decode());
2969         apdu = m_initRequest_apdu;     // but throw an init to the target
2970     }
2971
2972     // Determine our client.
2973     Z_OtherInformation **oi;
2974     get_otherInfoAPDU(apdu, &oi);
2975     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2976     if (!m_client)
2977     {
2978         if (m_http_version)
2979         {   // HTTP. Send not found
2980             send_http_response(404);
2981             return;
2982         }
2983         else
2984         {
2985             // Z39.50 just shutdown
2986             delete this;
2987             return;
2988         }
2989     }
2990
2991     m_client->m_server = this;
2992
2993     if (apdu->which == Z_APDU_initRequest)
2994         handle_init(apdu);
2995     else
2996         handle_incoming_Z_PDU_2(apdu);
2997 }
2998
2999 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
3000 {
3001     handle_max_record_retrieve(apdu);
3002
3003     if (apdu)
3004         apdu = handle_syntax_validation(apdu);
3005
3006     if (apdu)
3007         apdu = handle_query_transformation(apdu);
3008
3009     if (apdu)
3010         apdu = handle_target_charset_conversion(apdu);
3011
3012     if (apdu)
3013         apdu = handle_query_validation(apdu);
3014
3015     if (apdu)
3016         apdu = result_set_optimize(apdu);
3017
3018     if (!apdu)
3019     {
3020         m_client->timeout(m_target_idletime);  // mark it active even
3021         recv_GDU_more(true);
3022         // though we didn't use it
3023         return;
3024     }
3025     // Add otherInformation entry in APDU if
3026     // negotiation is in use.
3027     if (apdu)
3028         handle_charset_lang_negotiation(apdu);
3029
3030     // delete other info construct completely if 0 elements
3031     Z_OtherInformation **oi;
3032     get_otherInfoAPDU(apdu, &oi);
3033     if (oi && *oi && (*oi)->num_elements == 0)
3034         *oi = 0;
3035
3036     if (apdu->which == Z_APDU_presentRequest &&
3037         m_client->m_resultSetStartPoint == 0)
3038     {
3039         Z_PresentRequest *pr = apdu->u.presentRequest;
3040         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3041         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3042     } else {
3043         m_client->m_resultSetStartPoint = 0;
3044     }
3045     if (m_client->send_to_target(apdu) < 0)
3046     {
3047         m_client->shutdown();
3048     }
3049     else
3050         m_client->m_waiting = 1;
3051 }
3052
3053 void Yaz_Proxy::connectNotify()
3054 {
3055 }
3056
3057 void Yaz_Proxy::releaseClient()
3058 {
3059     xfree(m_proxyTarget);
3060     m_proxyTarget = 0;
3061     m_flag_invalid_session = 0;
3062     // only keep if keep_alive flag is set...
3063     if (m_client &&
3064         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3065         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3066         m_client->m_waiting == 0)
3067     {
3068         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3069                  m_session_str,
3070                  m_client->get_hostname());
3071         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3072                 m_session_str, m_client->m_pdu_recv,
3073                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3074                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3075         assert (m_client->m_waiting != 2);
3076         // Tell client (if any) that no server connection is there..
3077         m_client->m_server = 0;
3078         m_client = 0;
3079     }
3080     else if (m_client)
3081     {
3082         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
3083                  m_session_str,
3084                  m_client->get_hostname());
3085         assert (m_client->m_waiting != 2);
3086         delete m_client;
3087         m_client = 0;
3088     }
3089     else if (!m_parent)
3090     {
3091         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
3092                  m_session_str);
3093         assert (m_parent);
3094     }
3095     else
3096     {
3097         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
3098                  m_session_str);
3099     }
3100     if (m_parent)
3101         m_parent->pre_init();
3102 }
3103
3104 bool Yaz_Proxy::dec_ref(bool main_ptr)
3105 {
3106     yaz_log(YLOG_LOG, "%sdec_ref count=%d", m_session_str, m_ref_count);
3107
3108     assert(m_ref_count > 0);
3109     if (main_ptr)
3110     {
3111         if (m_main_ptr_dec)
3112             return false;
3113         m_main_ptr_dec = true;
3114     }
3115
3116     m_http_keepalive = 0;
3117
3118     --m_ref_count;
3119     if (m_ref_count > 0)
3120         return false;
3121
3122     releaseClient();
3123
3124     delete this;
3125     return true;
3126 }
3127
3128 const char *Yaz_ProxyClient::get_session_str()
3129 {
3130     if (!m_server)
3131         return "0 ";
3132     return m_server->get_session_str();
3133 }
3134
3135 void Yaz_ProxyClient::shutdown()
3136 {
3137     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3138              get_hostname());
3139
3140     if (m_server)
3141     {
3142         m_waiting = 1;      // ensure it's released from Proxy in releaseClient
3143         m_server->dec_ref(true);
3144     }
3145     else
3146         delete this;
3147 }
3148
3149 void Yaz_Proxy::failNotify()
3150 {
3151     inc_request_no();
3152     yaz_log (YLOG_LOG, "%sConnection closed by client",
3153              get_session_str());
3154     dec_ref(true);
3155 }
3156
3157 void Yaz_Proxy::send_response_fail_client(const char *addr)
3158 {
3159     yaz_log(YLOG_LOG, "%ssend_close_response", get_session_str());
3160     if (m_http_version)
3161     {
3162         Z_SRW_diagnostic *diagnostic = 0;
3163         int num_diagnostic = 0;
3164
3165         yaz_add_srw_diagnostic(odr_encode(),
3166                                &diagnostic, &num_diagnostic,
3167                                YAZ_SRW_SYSTEM_TEMPORARILY_UNAVAILABLE, addr);
3168         if (m_s2z_search_apdu)
3169             send_srw_search_response(diagnostic, num_diagnostic);
3170         else
3171             send_srw_explain_response(diagnostic, num_diagnostic);
3172     }
3173 }
3174 void Yaz_ProxyClient::failNotify()
3175 {
3176     if (m_server)
3177         m_server->inc_request_no();
3178     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
3179              get_session_str(), get_hostname());
3180
3181     if (m_server)
3182         m_server->send_response_fail_client(get_hostname());
3183     shutdown();
3184 }
3185
3186 void Yaz_ProxyClient::connectNotify()
3187 {
3188     const char *s = get_session_str();
3189     const char *h = get_hostname();
3190     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3191              m_target_idletime);
3192     timeout(m_target_idletime);
3193     if (!m_server)
3194         pre_init_client();
3195 }
3196
3197 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3198                                               *the_PDU_Observable, int fd)
3199 {
3200     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3201 }
3202
3203 Yaz_ProxyClient::~Yaz_ProxyClient()
3204 {
3205     if (m_prev)
3206         *m_prev = m_next;
3207     if (m_next)
3208         m_next->m_prev = m_prev;
3209     m_waiting = 2;     // for debugging purposes only.
3210     odr_destroy(m_init_odr);
3211     odr_destroy(m_idAuthentication_odr);
3212     delete m_last_query;
3213     xfree (m_last_resultSetId);
3214     xfree (m_cookie);
3215 }
3216
3217 void Yaz_ProxyClient::pre_init_client()
3218 {
3219     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3220     Z_InitRequest *req = apdu->u.initRequest;
3221
3222     int i;
3223     for (i = 0; i<= 24; i++)
3224         ODR_MASK_SET(req->options, i);
3225     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3226                    Z_Options_negotiationModel);
3227     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3228                    Z_Options_concurrentOperations);
3229     for (i = 0; i<= 10; i++)
3230         ODR_MASK_SET(req->protocolVersion, i);
3231
3232     if (send_to_target(apdu) < 0)
3233     {
3234         delete this;
3235     }
3236     else
3237     {
3238         m_waiting = 1;
3239         m_init_flag = 1;
3240     }
3241 }
3242
3243 void Yaz_Proxy::pre_init()
3244 {
3245     int i;
3246     const char *name = 0;
3247     const char *zurl_in_use[MAX_ZURL_PLEX];
3248     int limit_bw, limit_pdu, limit_req, limit_search, limit_connect;
3249     int target_idletime, client_idletime;
3250     int max_clients;
3251     int keepalive_limit_bw, keepalive_limit_pdu;
3252     int pre_init;
3253     const char *cql2rpn = 0;
3254     const char *authentication = 0;
3255     const char *negotiation_charset = 0;
3256     const char *negotiation_lang = 0;
3257
3258     Yaz_ProxyConfig *cfg = check_reconfigure();
3259
3260     zurl_in_use[0] = 0;
3261
3262     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3263         set_APDU_yazlog(1);
3264     else
3265         set_APDU_yazlog(0);
3266
3267     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3268                                           &limit_bw, &limit_pdu, &limit_req,
3269                                           &limit_search, &limit_connect,
3270                                           &target_idletime, &client_idletime,
3271                                           &max_clients,
3272                                           &keepalive_limit_bw,
3273                                           &keepalive_limit_pdu,
3274                                           &pre_init,
3275                                           &cql2rpn,
3276                                           &authentication,
3277                                           &negotiation_charset,
3278                                           &negotiation_lang,
3279                                           0) ; i++)
3280     {
3281         if (pre_init)
3282         {
3283             int j;
3284             for (j = 0; zurl_in_use[j]; j++)
3285             {
3286                 Yaz_ProxyClient *c;
3287                 int spare = 0;
3288                 int spare_waiting = 0;
3289                 int in_use = 0;
3290                 int other = 0;
3291                 for (c = m_clientPool; c; c = c->m_next)
3292                 {
3293                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3294                     {
3295                         if (c->m_cookie == 0)
3296                         {
3297                             if (c->m_server == 0)
3298                                 if (c->m_waiting)
3299                                     spare_waiting++;
3300                                 else
3301                                     spare++;
3302                             else
3303                                 in_use++;
3304                         }
3305                         else
3306                             other++;
3307                     }
3308                 }
3309                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3310                         "sparew=%d preinit=%d",m_session_str,
3311                         name, zurl_in_use[j], in_use, other,
3312                         spare, spare_waiting, pre_init);
3313                 if (spare + spare_waiting < pre_init)
3314                 {
3315                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3316                     c->m_next = m_clientPool;
3317                     if (c->m_next)
3318                         c->m_next->m_prev = &c->m_next;
3319                     m_clientPool = c;
3320                     c->m_prev = &m_clientPool;
3321
3322                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3323                         c->set_APDU_yazlog(1);
3324                     else
3325                         c->set_APDU_yazlog(0);
3326
3327                     if (c->client(zurl_in_use[j]))
3328                     {
3329                         timeout(60);
3330                         delete c;
3331                         return;
3332                     }
3333                     c->timeout(30);
3334                     c->m_waiting = 1;
3335                     c->m_target_idletime = target_idletime;
3336                     c->m_seqno = m_seqno++;
3337                 }
3338             }
3339         }
3340     }
3341 }
3342
3343 void Yaz_Proxy::timeoutNotify()
3344 {
3345     if (m_parent)
3346     {
3347         GDU *gdu;
3348         switch(m_timeout_mode)
3349         {
3350         case timeout_normal:
3351         case timeout_busy:
3352             inc_request_no();
3353             m_in_queue.clear();
3354             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3355             dec_ref(true);
3356             break;
3357         case timeout_reduce:
3358             timeout(m_client_idletime);
3359             m_timeout_mode = timeout_busy;
3360             gdu = m_timeout_gdu;
3361             m_timeout_gdu = 0;
3362             recv_GDU_normal(gdu);
3363             break;
3364         case timeout_xsl:
3365             assert(m_stylesheet_nprl);
3366             convert_xsl_delay();
3367             recv_GDU_more(true);
3368         }
3369     }
3370     else
3371     {
3372         timeout(600);
3373         pre_init();
3374     }
3375 }
3376
3377 void Yaz_Proxy::markInvalid()
3378 {
3379     m_client = 0;
3380     m_flag_invalid_session = 1;
3381 }
3382
3383 void Yaz_ProxyClient::timeoutNotify()
3384 {
3385     if (m_server)
3386         m_server->inc_request_no();
3387
3388     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3389              get_hostname());
3390
3391     if (m_server)
3392         m_server->send_response_fail_client(get_hostname());
3393     shutdown();
3394
3395     m_root->pre_init();
3396 }
3397
3398 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
3399                                  Yaz_Proxy *parent) :
3400     Z_Assoc (the_PDU_Observable)
3401 {
3402     m_cookie = 0;
3403     m_next = 0;
3404     m_prev = 0;
3405     m_init_flag = 0;
3406     m_last_query = 0;
3407     m_last_resultSetId = 0;
3408     m_last_resultCount = 0;
3409     m_last_ok = 0;
3410     m_sr_transform = 0;
3411     m_waiting = 0;
3412     m_init_odr = odr_createmem (ODR_DECODE);
3413     m_initResponse = 0;
3414     m_initResponse_options = 0;
3415     m_initResponse_version = 0;
3416     m_initResponse_preferredMessageSize = 0;
3417     m_initResponse_maximumRecordSize = 0;
3418     m_resultSetStartPoint = 0;
3419     m_bytes_sent = m_bytes_recv = 0;
3420     m_pdu_recv = 0;
3421     m_server = 0;
3422     m_seqno = 0;
3423     m_target_idletime = 600;
3424     m_root = parent;
3425     m_idAuthentication_odr = odr_createmem(ODR_ENCODE);
3426     m_idAuthentication_ber_buf = 0;
3427     m_idAuthentication_ber_size = 0;
3428 }
3429
3430 const char *Yaz_Proxy::option(const char *name, const char *value)
3431 {
3432     if (!strcmp (name, "optimize")) {
3433         if (value) {
3434             xfree (m_optimize);
3435             m_optimize = xstrdup (value);
3436         }
3437         return m_optimize;
3438     }
3439     return 0;
3440 }
3441
3442 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
3443 {
3444
3445 }
3446
3447 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
3448 {
3449     if (apdu->which == Z_GDU_Z3950)
3450         recv_Z_PDU(apdu->u.z3950, len);
3451     else if (apdu->which == Z_GDU_HTTP_Response)
3452         recv_HTTP_response(apdu->u.HTTP_Response, len);
3453     else
3454         shutdown();
3455 }
3456
3457 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
3458 {
3459     if (!m_flag_invalid_session)
3460         return 0;
3461     m_flag_invalid_session = 0;
3462     handle_incoming_Z_PDU(m_apdu_invalid_session);
3463     assert (m_mem_invalid_session);
3464     nmem_destroy(m_mem_invalid_session);
3465     m_mem_invalid_session = 0;
3466     return 1;
3467 }
3468
3469 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
3470 {
3471     m_bytes_recv += len;
3472
3473     m_pdu_recv++;
3474     m_waiting = 0;
3475     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
3476         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
3477                  apdu_name(apdu), get_hostname(), len);
3478     if (apdu->which == Z_APDU_initResponse)
3479     {
3480         if (!m_server)  // if this is a pre init session , check for more
3481             m_root->pre_init();
3482         NMEM nmem = odr_extract_mem (odr_decode());
3483         odr_reset (m_init_odr);
3484         nmem_transfer (m_init_odr->mem, nmem);
3485         m_initResponse = apdu;
3486         m_initResponse_options = apdu->u.initResponse->options;
3487         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3488         m_initResponse_preferredMessageSize =
3489             *apdu->u.initResponse->preferredMessageSize;
3490         m_initResponse_maximumRecordSize =
3491             *apdu->u.initResponse->maximumRecordSize;
3492
3493         Z_InitResponse *ir = apdu->u.initResponse;
3494         char *im0 = ir->implementationName;
3495
3496         char *im1 = (char*)
3497             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3498         *im1 = '\0';
3499         if (im0)
3500         {
3501             strcat(im1, im0);
3502             strcat(im1, " ");
3503         }
3504         strcat(im1, "(YAZ Proxy)");
3505         ir->implementationName = im1;
3506
3507         nmem_destroy (nmem);
3508
3509         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3510             return;
3511     }
3512     if (apdu->which == Z_APDU_searchResponse)
3513     {
3514         Z_SearchResponse *sr = apdu->u.searchResponse;
3515         m_last_resultCount = *sr->resultCount;
3516         int status = *sr->searchStatus;
3517         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3518         {
3519             m_last_ok = 1;
3520
3521             if (sr->records && sr->records->which == Z_Records_DBOSD)
3522             {
3523                 m_cache.add(odr_decode(),
3524                             sr->records->u.databaseOrSurDiagnostics, 1,
3525                             *sr->resultCount);
3526             }
3527         }
3528     }
3529     if (apdu->which == Z_APDU_presentResponse)
3530     {
3531         Z_PresentResponse *pr = apdu->u.presentResponse;
3532         if (m_sr_transform)
3533         {
3534             m_sr_transform = 0;
3535             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3536             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3537             sr->referenceId = pr->referenceId;
3538             *sr->resultCount = m_last_resultCount;
3539             sr->records = pr->records;
3540             sr->nextResultSetPosition = pr->nextResultSetPosition;
3541             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3542             apdu = new_apdu;
3543         }
3544         if (pr->records &&
3545             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3546         {
3547             m_cache.add(odr_decode(),
3548                         pr->records->u.databaseOrSurDiagnostics,
3549                         m_resultSetStartPoint, -1);
3550             m_resultSetStartPoint = 0;
3551         }
3552     }
3553     if (m_cookie)
3554         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3555
3556     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
3557
3558     if (server)
3559         server->send_to_client(apdu);
3560     if (apdu->which == Z_APDU_close)
3561         shutdown();
3562     else if (server)
3563         server->recv_GDU_more(true);
3564 }
3565
3566 void Yaz_Proxy::low_socket_close()
3567 {
3568 #if WIN32
3569 #else
3570     int i;
3571     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3572         if  (m_lo_fd[i] >= 0)
3573             ::close(m_lo_fd[i]);
3574 #endif
3575 }
3576
3577 void Yaz_Proxy::low_socket_open()
3578 {
3579 #if WIN32
3580 #else
3581     int i;
3582     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3583         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3584 #endif
3585 }
3586
3587 int Yaz_Proxy::server(const char *addr)
3588 {
3589     int r = Z_Assoc::server(addr);
3590     if (!r)
3591     {
3592         yaz_log(YLOG_LOG, "%sStarted proxy "
3593 #ifdef VERSION
3594             VERSION
3595 #endif
3596             " on %s", m_session_str, addr);
3597         timeout(1);
3598     }
3599     return r;
3600 }
3601
3602 /*
3603  * Local variables:
3604  * c-basic-offset: 4
3605  * indent-tabs-mode: nil
3606  * End:
3607  * vim: shiftwidth=4 tabstop=8 expandtab
3608  */
3609