src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.41 2005-11-30 11:38:46 adam Exp $
   2    Copyright (c) 1998-2005, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <stdlib.h>
  42 #include <time.h>
  43 #include <fcntl.h>
  44
  45 #include <yaz/srw.h>
  46 #include <yaz/marcdisp.h>
  47 #include <yaz/yaz-iconv.h>
  48 #include <yaz/log.h>
  49 #include <yaz/diagbib1.h>
  50 #include "proxyp.h"
  51 #include <yaz/pquery.h>
  52 #include <yaz/otherinfo.h>
  53 #include <yaz/charneg.h>
  54 #include "msg-thread.h"
  55
  56 using namespace yazpp_1;
  57
  58 #define USE_AUTH_MSG 1
  59
  60 #if USE_AUTH_MSG
  61 class Auth_Msg : public IMsg_Thread {
  62 public:
  63     int m_ret;
  64     IMsg_Thread *handle();
  65     void result();
  66     Yaz_Proxy *m_proxy;
  67     NMEM m_nmem;
  68     char *m_apdu_buf;
  69     int m_apdu_len;
  70     Auth_Msg();
  71     virtual ~Auth_Msg();
  72 };
  73
  74 Auth_Msg::Auth_Msg()
  75 {
  76     m_nmem = nmem_create();
  77 }
  78
  79 Auth_Msg::~Auth_Msg()
  80 {
  81     nmem_destroy(m_nmem);
  82 }
  83
  84 IMsg_Thread *Auth_Msg::handle()
  85 {
  86     yaz_log(YLOG_LOG, "Auth_Msg:handle begin");
  87     ODR decode = odr_createmem(ODR_DECODE);
  88     Z_APDU *apdu;
  89
  90     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
  91     int r = z_APDU(decode, &apdu, 0, 0);
  92     if (!r)
  93     {
  94         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
  95     }
  96     else
  97     {
  98         m_ret = m_proxy->handle_authentication(apdu);
  99     }
 100     yaz_log(YLOG_LOG, "Auth_Msg:handle end");
 101     odr_destroy(decode);
 102     return this;
 103 }
 104
 105 void Auth_Msg::result()
 106 {
 107     yaz_log(YLOG_LOG, "Auth_Msg:result proxy ok buf=%p len=%d",
 108             m_apdu_buf, m_apdu_len);
 109     if (m_proxy->dec_ref(false))
 110         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
 111     else
 112     {
 113         yaz_log(YLOG_LOG, "Auth_Msg::proxy still alive");
 114         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
 115         Z_APDU *apdu = 0;
 116         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
 117         if (r)
 118             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU OK");
 119         else
 120             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
 121         m_proxy->result_authentication(apdu, m_ret);
 122     }
 123     delete this;
 124 }
 125
 126 #endif
 127
 128 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
 129 {
 130     if (apdu == 0 || ret == 0)
 131     {
 132         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
 133         *apdu_reject->u.initResponse->result = 0;
 134         send_to_client(apdu_reject);
 135         dec_ref(false);
 136     }
 137     else
 138         handle_incoming_Z_PDU_2(apdu);
 139 }
 140
 141 static const char *apdu_name(Z_APDU *apdu)
 142 {
 143     switch (apdu->which)
 144     {
 145     case Z_APDU_initRequest:
 146         return "initRequest";
 147     case Z_APDU_initResponse:
 148         return "initResponse";
 149     case Z_APDU_searchRequest:
 150         return "searchRequest";
 151     case Z_APDU_searchResponse:
 152         return "searchResponse";
 153     case Z_APDU_presentRequest:
 154         return "presentRequest";
 155     case Z_APDU_presentResponse:
 156         return "presentResponse";
 157     case Z_APDU_deleteResultSetRequest:
 158         return "deleteResultSetRequest";
 159     case Z_APDU_deleteResultSetResponse:
 160         return "deleteResultSetResponse";
 161     case Z_APDU_scanRequest:
 162         return "scanRequest";
 163     case Z_APDU_scanResponse:
 164         return "scanResponse";
 165     case Z_APDU_sortRequest:
 166         return "sortRequest";
 167     case Z_APDU_sortResponse:
 168         return "sortResponse";
 169     case Z_APDU_extendedServicesRequest:
 170         return "extendedServicesRequest";
 171     case Z_APDU_extendedServicesResponse:
 172         return "extendedServicesResponse";
 173     case Z_APDU_close:
 174         return "close";
 175     }
 176     return "other";
 177 }
 178
 179 static const char *gdu_name(Z_GDU *gdu)
 180 {
 181     switch(gdu->which)
 182     {
 183     case Z_GDU_Z3950:
 184         return apdu_name(gdu->u.z3950);
 185     case Z_GDU_HTTP_Request:
 186         return "HTTP Request";
 187     case Z_GDU_HTTP_Response:
 188         return "HTTP Response";
 189     }
 190     return "Unknown request/response";
 191 }
 192
 193 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
 194                      ISocketObservable *the_socket_observable,
 195                      Yaz_Proxy *parent)
 196     :
 197     Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 198 {
 199     m_PDU_Observable = the_PDU_Observable;
 200     m_socket_observable = the_socket_observable;
 201     m_client = 0;
 202     m_parent = parent;
 203     m_clientPool = 0;
 204     m_seqno = 1;
 205     m_keepalive_limit_bw = 500000;
 206     m_keepalive_limit_pdu = 1000;
 207     m_proxyTarget = 0;
 208     m_default_target = 0;
 209     m_proxy_negotiation_charset = 0;
 210     m_proxy_negotiation_lang = 0;
 211     m_charset_converter = new Yaz_CharsetConverter;
 212     m_max_clients = 150;
 213     m_log_mask = 0;
 214     m_seed = time(0);
 215     m_client_idletime = 600;
 216     m_target_idletime = 600;
 217     m_optimize = xstrdup ("1");
 218     strcpy(m_session_str, "0 ");
 219     m_session_no = 0;
 220     m_bytes_sent = 0;
 221     m_bytes_recv = 0;
 222     m_bw_max = 0;
 223     m_pdu_max = 0;
 224     m_search_max = 0;
 225     m_connect_max = 0;
 226     m_timeout_mode = timeout_normal;
 227     m_timeout_gdu = 0;
 228     m_max_record_retrieve = 0;
 229     m_reconfig_flag = 0;
 230     m_config_fname = 0;
 231     m_request_no = 0;
 232     m_flag_invalid_session = 0;
 233     m_referenceId = 0;
 234     m_referenceId_mem = nmem_create();
 235     m_config = 0;
 236     m_marcxml_mode = none;
 237     m_stylesheet_xsp = 0;
 238     m_stylesheet_nprl = 0;
 239     m_stylesheet_apdu = 0;
 240     m_s2z_stylesheet = 0;
 241     m_s2z_database = 0;
 242     m_schema = 0;
 243     m_backend_type = 0;
 244     m_backend_charset = 0;
 245     m_frontend_type = 0;
 246     m_initRequest_apdu = 0;
 247     m_initRequest_mem = 0;
 248     m_initRequest_preferredMessageSize = 0;
 249     m_initRequest_maximumRecordSize = 0;
 250     m_initRequest_options = 0;
 251     m_initRequest_version = 0;
 252     m_initRequest_oi_negotiation_charsets = 0;
 253     m_initRequest_oi_negotiation_num_charsets = 0;
 254     m_initRequest_oi_negotiation_langs = 0;
 255     m_initRequest_oi_negotiation_num_langs = 0;
 256     m_initRequest_oi_negotiation_selected = 0;
 257     m_apdu_invalid_session = 0;
 258     m_mem_invalid_session = 0;
 259     m_s2z_odr_init = 0;
 260     m_s2z_odr_search = 0;
 261     m_s2z_init_apdu = 0;
 262     m_s2z_search_apdu = 0;
 263     m_s2z_present_apdu = 0;
 264     m_http_keepalive = 0;
 265     m_http_version = 0;
 266     m_soap_ns = 0;
 267     m_s2z_packing = Z_SRW_recordPacking_string;
 268 #if HAVE_GETTIMEOFDAY
 269     m_time_tv = xmalloc(sizeof(struct timeval));
 270     struct timeval *tv = (struct timeval *) m_time_tv;
 271     tv->tv_sec = 0;
 272     tv->tv_usec = 0;
 273 #else
 274     m_time_tv = 0;
 275 #endif
 276     m_usemarcon_ini_stage1 = 0;
 277     m_usemarcon_ini_stage2 = 0;
 278     m_usemarcon = new Yaz_usemarcon();
 279     if (!m_parent)
 280         low_socket_open();
 281     m_my_thread = 0;
 282     m_ref_count = 1;
 283     m_main_ptr_dec = false;
 284     m_peername = 0;
 285 }
 286
 287 void Yaz_Proxy::inc_ref()
 288 {
 289     m_ref_count++;
 290 }
 291
 292 Yaz_Proxy::~Yaz_Proxy()
 293 {
 294     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 295             m_bytes_sent, m_bytes_recv);
 296     nmem_destroy(m_initRequest_mem);
 297     nmem_destroy(m_mem_invalid_session);
 298     nmem_destroy(m_referenceId_mem);
 299
 300     xfree(m_proxyTarget);
 301     xfree(m_default_target);
 302     xfree(m_proxy_negotiation_charset);
 303     xfree(m_proxy_negotiation_lang);
 304     delete m_charset_converter;
 305     xfree(m_optimize);
 306
 307 #if HAVE_XSLT
 308     if (m_stylesheet_xsp)
 309         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 310 #endif
 311     xfree (m_time_tv);
 312
 313     xfree (m_peername);
 314     xfree (m_schema);
 315     xfree (m_backend_type);
 316     xfree (m_backend_charset);
 317     xfree (m_usemarcon_ini_stage1);
 318     xfree (m_usemarcon_ini_stage2);
 319     delete m_usemarcon;
 320     if (m_s2z_odr_init)
 321         odr_destroy(m_s2z_odr_init);
 322     if (m_s2z_odr_search)
 323         odr_destroy(m_s2z_odr_search);
 324     if (!m_parent)
 325         low_socket_close();
 326     if (!m_parent)
 327         delete m_my_thread;
 328     delete m_config;
 329 }
 330
 331 void Yaz_Proxy::set_debug_mode(int mode)
 332 {
 333     m_debug_mode = mode;
 334 }
 335
 336 int Yaz_Proxy::set_config(const char *config)
 337 {
 338     delete m_config;
 339     m_config = new Yaz_ProxyConfig();
 340     xfree(m_config_fname);
 341     m_config_fname = xstrdup(config);
 342     int r = m_config->read_xml(config);
 343     if (!r)
 344         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 345     return r;
 346 }
 347
 348 void Yaz_Proxy::set_default_target(const char *target)
 349 {
 350     xfree (m_default_target);
 351     m_default_target = 0;
 352     if (target)
 353         m_default_target = (char *) xstrdup (target);
 354 }
 355
 356 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang)
 357 {
 358     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
 359         "language to '%s'", m_session_str, charset?charset:"none",
 360         lang?lang:"none");
 361     xfree (m_proxy_negotiation_charset);
 362     xfree (m_proxy_negotiation_lang);
 363     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 364     if (charset)
 365         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 366     if (lang)
 367         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 368 }
 369
 370 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 371 {
 372     if (m_parent)
 373         return m_parent->check_reconfigure();
 374
 375     Yaz_ProxyConfig *cfg = m_config;
 376     if (m_reconfig_flag)
 377     {
 378         yaz_log(YLOG_LOG, "reconfigure");
 379         yaz_log_reopen();
 380         if (m_config_fname && cfg)
 381         {
 382             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 383             int r = cfg->read_xml(m_config_fname);
 384             if (r)
 385                 yaz_log(YLOG_WARN, "reconfigure failed");
 386             else
 387             {
 388                 m_log_mask = 0;
 389                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 390             }
 391         }
 392         else
 393             yaz_log(YLOG_LOG, "reconfigure");
 394         m_reconfig_flag = 0;
 395     }
 396     return cfg;
 397 }
 398
 399 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 400                                         *the_PDU_Observable, int fd)
 401 {
 402     check_reconfigure();
 403     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 404                                          m_socket_observable, this);
 405     new_proxy->m_config = 0;
 406     new_proxy->m_config_fname = 0;
 407     new_proxy->timeout(m_client_idletime);
 408     new_proxy->m_target_idletime = m_target_idletime;
 409     new_proxy->set_default_target(m_default_target);
 410     new_proxy->m_max_clients = m_max_clients;
 411     new_proxy->m_log_mask = m_log_mask;
 412     new_proxy->set_APDU_log(get_APDU_log());
 413     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 414         new_proxy->set_APDU_yazlog(1);
 415     else
 416         new_proxy->set_APDU_yazlog(0);
 417     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 418     m_session_no++;
 419     new_proxy->m_peername = xstrdup(the_PDU_Observable->getpeername());
 420     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 421              new_proxy->m_peername);
 422     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 423         m_proxy_negotiation_lang);
 424     // create thread object the first time we get an incoming connection
 425     if (!m_my_thread)
 426         m_my_thread = new Msg_Thread(m_socket_observable, 1);
 427     new_proxy->m_my_thread = m_my_thread;
 428     return new_proxy;
 429 }
 430
 431 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 432 {
 433     int oid[OID_SIZE];
 434     Z_OtherInformationUnit *oi;
 435     struct oident ent;
 436     ent.proto = PROTO_Z3950;
 437     ent.oclass = CLASS_USERINFO;
 438     ent.value = (oid_value) VAL_COOKIE;
 439     assert (oid_ent_to_oid (&ent, oid));
 440
 441     if (oid_ent_to_oid (&ent, oid) &&
 442         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 443         oi->which == Z_OtherInfo_characterInfo)
 444         return oi->information.characterInfo;
 445     return 0;
 446 }
 447 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 448 {
 449     int oid[OID_SIZE];
 450     Z_OtherInformationUnit *oi;
 451     struct oident ent;
 452     ent.proto = PROTO_Z3950;
 453     ent.oclass = CLASS_USERINFO;
 454     ent.value = (oid_value) VAL_PROXY;
 455     if (oid_ent_to_oid (&ent, oid) &&
 456         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 457         oi->which == Z_OtherInfo_characterInfo)
 458         return oi->information.characterInfo;
 459     return 0;
 460 }
 461 const char *Yaz_Proxy::load_balance(const char **url)
 462 {
 463     int zurl_in_use[MAX_ZURL_PLEX];
 464     int zurl_in_spare[MAX_ZURL_PLEX];
 465     Yaz_ProxyClient *c;
 466     int i;
 467
 468     for (i = 0; i<MAX_ZURL_PLEX; i++)
 469     {
 470         zurl_in_use[i] = 0;
 471         zurl_in_spare[i] = 0;
 472     }
 473     for (c = m_parent->m_clientPool; c; c = c->m_next)
 474     {
 475         for (i = 0; url[i]; i++)
 476             if (!strcmp(url[i], c->get_hostname()))
 477             {
 478                 zurl_in_use[i]++;
 479                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 480                     zurl_in_spare[i]++;
 481             }
 482     }
 483     int min_use = 100000;
 484     int spare_for_min = 0;
 485     int max_spare = 0;
 486     const char *ret_min = 0;
 487     const char *ret_spare = 0;
 488     for (i = 0; url[i]; i++)
 489     {
 490         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 491                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 492         if (min_use > zurl_in_use[i])
 493         {
 494             ret_min = url[i];
 495             min_use = zurl_in_use[i];
 496             spare_for_min = zurl_in_spare[i];
 497         }
 498         if (max_spare < zurl_in_spare[i])
 499         {
 500             ret_spare = url[i];
 501             max_spare = zurl_in_spare[i];
 502         }
 503     }
 504     return ret_min;
 505 }
 506
 507 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 508                                        const char *proxy_host)
 509 {
 510     assert (m_parent);
 511     Yaz_Proxy *parent = m_parent;
 512     Yaz_ProxyClient *c = m_client;
 513
 514     if (!m_proxyTarget)
 515     {
 516         const char *url[MAX_ZURL_PLEX];
 517         Yaz_ProxyConfig *cfg = check_reconfigure();
 518         if (proxy_host)
 519         {
 520             if (parent && parent->m_debug_mode)
 521             {
 522                 // only to be enabled for debugging...
 523                 if (!strcmp(proxy_host, "stop"))
 524                     exit(0);
 525             }
 526             xfree(m_default_target);
 527             m_default_target = xstrdup(proxy_host);
 528         }
 529         proxy_host = m_default_target;
 530         int client_idletime = -1;
 531         const char *cql2rpn_fname = 0;
 532         const char *negotiation_charset = 0;
 533         const char *negotiation_lang = 0;
 534         const char *query_charset = 0;
 535         url[0] = m_default_target;
 536         url[1] = 0;
 537         if (cfg)
 538         {
 539             int pre_init = 0;
 540             cfg->get_target_info(proxy_host, url, &m_bw_max,
 541                                  &m_pdu_max, &m_max_record_retrieve,
 542                                  &m_search_max, &m_connect_max,
 543                                  &m_target_idletime, &client_idletime,
 544                                  &parent->m_max_clients,
 545                                  &m_keepalive_limit_bw,
 546                                  &m_keepalive_limit_pdu,
 547                                  &pre_init,
 548                                  &cql2rpn_fname,
 549                                  &negotiation_charset,
 550                                  &negotiation_lang,
 551                                  &query_charset);
 552         }
 553         if (client_idletime != -1)
 554         {
 555             m_client_idletime = client_idletime;
 556             timeout(m_client_idletime);
 557         }
 558         if (cql2rpn_fname)
 559             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 560         if (negotiation_charset || negotiation_lang)
 561         {
 562             set_proxy_negotiation(negotiation_charset,
 563                 negotiation_lang);
 564         }
 565         m_charset_converter->set_target_query_charset(query_charset);
 566         if (!url[0])
 567         {
 568             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 569             return 0;
 570         }
 571         // we don't handle multiplexing for cookie session, so we just
 572         // pick the first one in this case (anonymous users will be able
 573         // to use any backend)
 574         if (cookie && *cookie)
 575             m_proxyTarget = (char*) xstrdup(url[0]);
 576         else
 577             m_proxyTarget = (char*) xstrdup(load_balance(url));
 578     }
 579     if (cookie && *cookie)
 580     {   // search in sessions with a cookie
 581         for (c = parent->m_clientPool; c; c = c->m_next)
 582         {
 583             assert (c->m_prev);
 584             assert (*c->m_prev == c);
 585             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 586                 !strcmp(m_proxyTarget, c->get_hostname()))
 587             {
 588                 // Found it in cache
 589                 // The following handles "cancel"
 590                 // If connection is busy (waiting for PDU) and
 591                 // we have an initRequest we can safely do re-open
 592                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 593                 {
 594                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 595                              c->get_hostname());
 596                     c->close();
 597                     c->m_init_flag = 0;
 598
 599                     c->m_last_ok = 0;
 600                     c->m_cache.clear();
 601                     c->m_last_resultCount = 0;
 602                     c->m_sr_transform = 0;
 603                     c->m_waiting = 0;
 604                     c->m_resultSetStartPoint = 0;
 605                     c->m_target_idletime = m_target_idletime;
 606                     if (c->client(m_proxyTarget))
 607                     {
 608                         delete c;
 609                         return 0;
 610                     }
 611                     c->timeout(30);
 612                 }
 613                 c->m_seqno = parent->m_seqno;
 614                 if (c->m_server && c->m_server != this)
 615                     c->m_server->m_client = 0;
 616                 c->m_server = this;
 617                 (parent->m_seqno)++;
 618                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 619                 return c;
 620             }
 621         }
 622     }
 623     else if (!c &&
 624             apdu->which == Z_APDU_initRequest &&
 625             apdu->u.initRequest->idAuthentication == 0 &&
 626             !ODR_MASK_GET(apdu->u.initRequest->options, Z_Options_negotiationModel))
 627     {
 628         // anonymous sessions without cookie.
 629         // if authentication is set it is NOT anonymous se we can't share them.
 630         // If charset and lang negotiation is use it is NOT anonymous session too.
 631         for (c = parent->m_clientPool; c; c = c->m_next)
 632         {
 633             assert(c->m_prev);
 634             assert(*c->m_prev == c);
 635             if (c->m_server == 0 && c->m_cookie == 0 &&
 636                 c->m_waiting == 0 &&
 637                 !strcmp(m_proxyTarget, c->get_hostname()))
 638             {
 639                 // found it in cache
 640                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 641                          m_session_str, parent->m_seqno, c->get_hostname());
 642
 643                 c->m_seqno = parent->m_seqno;
 644                 assert(c->m_server == 0);
 645                 c->m_server = this;
 646
 647                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 648                     c->set_APDU_yazlog(1);
 649                 else
 650                     c->set_APDU_yazlog(0);
 651
 652                 (parent->m_seqno)++;
 653
 654                 parent->pre_init();
 655
 656                 return c;
 657             }
 658         }
 659     }
 660     if (!m_client)
 661     {
 662         if (apdu->which != Z_APDU_initRequest)
 663         {
 664             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 665             return 0;
 666         }
 667         Z_InitRequest *initRequest = apdu->u.initRequest;
 668
 669         if (initRequest->idAuthentication)
 670         {
 671             // the client uses authentication. We set the keepalive PDU
 672             // to 0 so we don't cache it in releaseClient
 673             m_keepalive_limit_pdu = 0;
 674         }
 675         // go through list of clients - and find the lowest/oldest one.
 676         Yaz_ProxyClient *c_min = 0;
 677         int min_seq = -1;
 678         int no_of_clients = 0;
 679         if (parent->m_clientPool)
 680             yaz_log (YLOG_DEBUG, "Existing sessions");
 681         for (c = parent->m_clientPool; c; c = c->m_next)
 682         {
 683             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 684                                c->m_waiting, c->get_hostname(),
 685                                c->m_cookie ? c->m_cookie : "");
 686             no_of_clients++;
 687             if (min_seq < 0 || c->m_seqno < min_seq)
 688             {
 689                 min_seq = c->m_seqno;
 690                 c_min = c;
 691             }
 692         }
 693         if (no_of_clients >= parent->m_max_clients)
 694         {
 695             c = c_min;
 696             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 697             {
 698                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 699                          m_session_str, parent->m_max_clients, c->m_seqno);
 700                 if (c->m_server && c->m_server != this)
 701                     delete c->m_server;   // PROBLEM: m_ref_count!
 702                 c->m_server = 0;
 703             }
 704             else
 705             {
 706                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 707                          m_session_str, parent->m_max_clients,
 708                          c->m_seqno, parent->m_seqno, c->get_hostname());
 709                 xfree (c->m_cookie);
 710                 c->m_cookie = 0;
 711                 if (cookie)
 712                     c->m_cookie = xstrdup(cookie);
 713                 c->m_seqno = parent->m_seqno;
 714                 if (c->m_server && c->m_server != this)
 715                 {
 716                     c->m_server->m_client = 0;
 717                     delete c->m_server;   // PROBLEM: m_ref_count!
 718                 }
 719                 (parent->m_seqno)++;
 720                 c->m_target_idletime = m_target_idletime;
 721                 c->timeout(m_target_idletime);
 722
 723                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 724                     c->set_APDU_yazlog(1);
 725                 else
 726                     c->set_APDU_yazlog(0);
 727
 728                 return c;
 729             }
 730         }
 731         else
 732         {
 733             yaz_log (YLOG_LOG, "%sNEW %d %s",
 734                      m_session_str, parent->m_seqno, m_proxyTarget);
 735             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 736             c->m_next = parent->m_clientPool;
 737             if (c->m_next)
 738                 c->m_next->m_prev = &c->m_next;
 739             parent->m_clientPool = c;
 740             c->m_prev = &parent->m_clientPool;
 741         }
 742
 743         xfree (c->m_cookie);
 744         c->m_cookie = 0;
 745         if (cookie)
 746             c->m_cookie = xstrdup(cookie);
 747
 748         c->m_seqno = parent->m_seqno;
 749         c->m_init_flag = 0;
 750         c->m_last_resultCount = 0;
 751         c->m_last_ok = 0;
 752         c->m_cache.clear();
 753         c->m_sr_transform = 0;
 754         c->m_waiting = 0;
 755         c->m_resultSetStartPoint = 0;
 756         (parent->m_seqno)++;
 757         if (c->client(m_proxyTarget))
 758         {
 759             delete c;
 760             return 0;
 761         }
 762         c->m_target_idletime = m_target_idletime;
 763         c->timeout(30);
 764
 765         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 766             c->set_APDU_yazlog(1);
 767         else
 768             c->set_APDU_yazlog(0);
 769     }
 770     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 771     return c;
 772 }
 773
 774 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 775 {
 776     int i;
 777     for (i = 0; i<num; i++)
 778     {
 779         oident *ent;
 780         Z_DefaultDiagFormat *r;
 781         Z_DiagRec *p = pp[i];
 782         if (p->which != Z_DiagRec_defaultFormat)
 783         {
 784             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 785             return;
 786         }
 787         else
 788             r = p->u.defaultFormat;
 789         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 790             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 791             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 792         switch (r->which)
 793         {
 794         case Z_DefaultDiagFormat_v2Addinfo:
 795             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 796                     m_session_str,
 797                     *r->condition, diagbib1_str(*r->condition),
 798                     r->u.v2Addinfo);
 799             break;
 800         case Z_DefaultDiagFormat_v3Addinfo:
 801             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 802                     m_session_str,
 803                     *r->condition, diagbib1_str(*r->condition),
 804                     r->u.v3Addinfo);
 805             break;
 806         }
 807     }
 808 }
 809
 810 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 811 {
 812     if (!m_stylesheet_xsp || p->num_records <= 0)
 813     {
 814         return 0;  /* no XSLT to be done ... */
 815     }
 816
 817     m_stylesheet_offset = 0;
 818     m_stylesheet_nprl = p;
 819     m_stylesheet_apdu = apdu;
 820     m_timeout_mode = timeout_xsl;
 821
 822     timeout(0);
 823     return 1;
 824 }
 825
 826 void Yaz_Proxy::convert_xsl_delay()
 827 {
 828 #if HAVE_XSLT
 829     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 830     if (npr->which == Z_NamePlusRecord_databaseRecord)
 831     {
 832         Z_External *r = npr->u.databaseRecord;
 833         if (r->which == Z_External_octet)
 834         {
 835 #if 0
 836             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 837 #endif
 838             xmlDocPtr res, doc = xmlParseMemory(
 839                 (char*) r->u.octet_aligned->buf,
 840                 r->u.octet_aligned->len);
 841
 842
 843             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 844                     m_session_str, m_stylesheet_offset);
 845             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 846                                       doc, 0);
 847
 848             if (res)
 849             {
 850                 xmlChar *out_buf;
 851                 int out_len;
 852                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 853
 854                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 855                     u.databaseRecord =
 856                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 857                                  (char*) out_buf, out_len);
 858                 xmlFree(out_buf);
 859                 xmlFreeDoc(res);
 860             }
 861
 862             xmlFreeDoc(doc);
 863         }
 864     }
 865 #endif
 866     m_stylesheet_offset++;
 867     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 868     {
 869         m_timeout_mode = timeout_normal;
 870         m_stylesheet_nprl = 0;
 871 #if HAVE_XSLT
 872         if (m_stylesheet_xsp)
 873             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 874 #endif
 875         m_stylesheet_xsp = 0;
 876         timeout(m_client_idletime);
 877         send_PDU_convert(m_stylesheet_apdu);
 878     }
 879     else
 880         timeout(0);
 881 }
 882
 883 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 884 {
 885     if (m_frontend_type != VAL_NONE)
 886     {
 887         int i;
 888         for (i = 0; i < p->num_records; i++)
 889         {
 890             Z_NamePlusRecord *npr = p->records[i];
 891             if (npr->which == Z_NamePlusRecord_databaseRecord)
 892             {
 893                 Z_External *r = npr->u.databaseRecord;
 894                 if (r->which == Z_External_octet)
 895                 {
 896 #if HAVE_USEMARCON
 897                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 898                     {
 899                         if (!m_usemarcon->m_stage1)
 900                         {
 901                             m_usemarcon->m_stage1 = new CDetails();
 902                         }
 903                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 904                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 905                         int res = m_usemarcon->m_stage1->Start();
 906                         if (res == 0)
 907                         {
 908                             char *converted;
 909                             int convlen;
 910                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 911                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 912                             {
 913                                 if (!m_usemarcon->m_stage2)
 914                                 {
 915                                     m_usemarcon->m_stage2 = new CDetails();
 916                                 }
 917                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 918                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 919                                 res = m_usemarcon->m_stage2->Start();
 920                                 if (res == 0)
 921                                 {
 922                                     free(converted);
 923                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 924                                 }
 925                                 else
 926                                 {
 927                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 928                                 }
 929                             }
 930                             npr->u.databaseRecord =
 931                                 z_ext_record(odr_encode(),
 932                                              m_frontend_type,
 933                                              converted,
 934                                              strlen(converted));
 935                             free(converted);
 936                         }
 937                         else
 938                         {
 939                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 940                         }
 941                         continue;
 942                     }
 943 #endif
 944 /* HAVE_USEMARCON */
 945                     npr->u.databaseRecord =
 946                         z_ext_record(odr_encode(),
 947                                      m_frontend_type,
 948                                      (char*) r->u.octet_aligned->buf,
 949                                      r->u.octet_aligned->len);
 950                 }
 951             }
 952         }
 953     }
 954 }
 955
 956 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
 957                                         const char *backend_charset)
 958 {
 959     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
 960     int sel =   m_charset_converter->get_client_charset_selected();
 961     const char *client_record_charset =
 962         m_charset_converter->get_client_query_charset();
 963     if (sel && backend_charset && client_record_charset &&
 964         strcmp(backend_charset, client_record_charset))
 965     {
 966         int i;
 967         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
 968                                         backend_charset);
 969         yaz_marc_t mt = yaz_marc_create();
 970         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
 971         yaz_marc_iconv(mt, cd);
 972         for (i = 0; i < p->num_records; i++)
 973         {
 974             Z_NamePlusRecord *npr = p->records[i];
 975             if (npr->which == Z_NamePlusRecord_databaseRecord)
 976             {
 977                 Z_External *r = npr->u.databaseRecord;
 978                 oident *ent = oid_getentbyoid(r->direct_reference);
 979                 if (!ent || ent->value == VAL_NONE)
 980                     continue;
 981
 982                 if (ent->value == VAL_SUTRS)
 983                 {
 984                     WRBUF w = wrbuf_alloc();
 985
 986                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
 987                                       r->u.octet_aligned->len);
 988                     npr->u.databaseRecord =
 989                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
 990                                      wrbuf_len(w));
 991                     wrbuf_free(w, 1);
 992                 }
 993                 else if (ent->value == VAL_TEXT_XML)
 994                 {
 995                     ;
 996                 }
 997                 else if (r->which == Z_External_octet)
 998                 {
 999                     int rlen;
1000                     char *result;
1001                     if (yaz_marc_decode_buf(mt,
1002                                             (char*) r->u.octet_aligned->buf,
1003                                             r->u.octet_aligned->len,
1004                                             &result, &rlen))
1005                     {
1006                         npr->u.databaseRecord =
1007                             z_ext_record(odr_encode(), ent->value, result, rlen);
1008                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1009                                 m_session_str);
1010                     }
1011                 }
1012             }
1013         }
1014         if (cd)
1015             yaz_iconv_close(cd);
1016         yaz_marc_destroy(mt);
1017     }
1018     else
1019     {
1020         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
1021     }
1022 }
1023
1024 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1025                                    const char *backend_charset)
1026 {
1027     int i;
1028     if (!backend_charset)
1029         backend_charset = "MARC-8";
1030     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1031     yaz_marc_t mt = yaz_marc_create();
1032     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1033     yaz_marc_iconv(mt, cd);
1034     for (i = 0; i < p->num_records; i++)
1035     {
1036         Z_NamePlusRecord *npr = p->records[i];
1037         if (npr->which == Z_NamePlusRecord_databaseRecord)
1038         {
1039             Z_External *r = npr->u.databaseRecord;
1040             if (r->which == Z_External_OPAC)
1041             {
1042                 WRBUF w = wrbuf_alloc();
1043
1044                 yaz_display_OPAC(w, r->u.opac, 0);
1045                 npr->u.databaseRecord = z_ext_record(
1046                     odr_encode(), VAL_TEXT_XML,
1047                     wrbuf_buf(w), wrbuf_len(w)
1048                     );
1049                 wrbuf_free(w, 1);
1050             }
1051             else if (r->which == Z_External_octet)
1052             {
1053                 int rlen;
1054                 char *result;
1055                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1056                                         r->u.octet_aligned->len,
1057                                         &result, &rlen))
1058                 {
1059                     npr->u.databaseRecord =
1060                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
1061                 }
1062             }
1063         }
1064     }
1065     if (cd)
1066         yaz_iconv_close(cd);
1067     yaz_marc_destroy(mt);
1068 }
1069
1070 void Yaz_Proxy::logtime()
1071 {
1072 #if HAVE_GETTIMEOFDAY
1073     struct timeval *tv = (struct timeval*) m_time_tv;
1074     if (tv->tv_sec)
1075     {
1076         struct timeval tv1;
1077         gettimeofday(&tv1, 0);
1078         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1079             (tv1.tv_usec - tv->tv_usec);
1080         if (diff >= 0)
1081             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1082                     diff/1000000, (diff/1000)%1000);
1083     }
1084     tv->tv_sec = 0;
1085     tv->tv_usec = 0;
1086 #endif
1087 }
1088
1089 int Yaz_Proxy::send_http_response(int code)
1090 {
1091     ODR o = odr_encode();
1092     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1093     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1094     if (m_http_version)
1095         hres->version = odr_strdup(o, m_http_version);
1096     if (m_http_keepalive)
1097         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1098     else
1099         timeout(0);
1100
1101     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1102     {
1103         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1104                  gdu_name(gdu));
1105     }
1106     int len;
1107     int r = send_GDU(gdu, &len);
1108     m_bytes_sent += len;
1109     m_bw_stat.add_bytes(len);
1110     logtime();
1111
1112     recv_GDU_more(true);
1113
1114     return r;
1115 }
1116
1117 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
1118 {
1119     ODR o = odr_encode();
1120     const char *ctype = "text/xml";
1121     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
1122     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1123     if (m_http_version)
1124         hres->version = odr_strdup(o, m_http_version);
1125     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1126     if (m_http_keepalive)
1127         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1128     else
1129         timeout(0);
1130
1131     static Z_SOAP_Handler soap_handlers[2] = {
1132 #if HAVE_XSLT
1133         {"http://www.loc.gov/zing/srw/", 0,
1134          (Z_SOAP_fun) yaz_srw_codec},
1135 #endif
1136         {0, 0, 0}
1137     };
1138
1139     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1140     soap_package->which = Z_SOAP_generic;
1141     soap_package->u.generic =
1142         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1143     soap_package->u.generic->no = 0;
1144     soap_package->u.generic->ns = soap_handlers[0].ns;
1145     soap_package->u.generic->p = (void *) srw_pdu;
1146     soap_package->ns = m_soap_ns;
1147     z_soap_codec_enc_xsl(o, &soap_package,
1148                          &hres->content_buf, &hres->content_len,
1149                          soap_handlers, 0, m_s2z_stylesheet);
1150     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1151     {
1152         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1153                  gdu_name(gdu));
1154     }
1155     int len;
1156     int r = send_GDU(gdu, &len);
1157     m_bytes_sent += len;
1158     m_bw_stat.add_bytes(len);
1159     logtime();
1160
1161     recv_GDU_more(true);
1162
1163     return r;
1164 }
1165
1166 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1167 {
1168     ODR o = odr_encode();
1169     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1170         odr_malloc(o, sizeof(*diagnostic));
1171     int num_diagnostic = 1;
1172     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1173     return send_srw_search_response(diagnostic, num_diagnostic);
1174 }
1175
1176 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1177                              Z_DefaultDiagFormat *ddf)
1178 {
1179     int bib1_code = *ddf->condition;
1180     if (bib1_code == 109)
1181         return 404;
1182     srw_res->num_diagnostics = 1;
1183     srw_res->diagnostics = (Z_SRW_diagnostic *)
1184         odr_malloc(o, sizeof(*srw_res->diagnostics));
1185     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1186                           yaz_diag_bib1_to_srw(*ddf->condition),
1187                           ddf->u.v2Addinfo);
1188     return 0;
1189 }
1190
1191 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1192 {
1193     ODR o = odr_encode();
1194     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1195     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1196
1197     srw_res->numberOfRecords = odr_intdup (o, hits);
1198     if (records && records->which == Z_Records_DBOSD)
1199     {
1200         srw_res->num_records =
1201             records->u.databaseOrSurDiagnostics->num_records;
1202         int i;
1203         srw_res->records = (Z_SRW_record *)
1204             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1205         for (i = 0; i < srw_res->num_records; i++)
1206         {
1207             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1208             if (npr->which != Z_NamePlusRecord_databaseRecord)
1209             {
1210                 srw_res->records[i].recordSchema = "diagnostic";
1211                 srw_res->records[i].recordPacking = m_s2z_packing;
1212                 srw_res->records[i].recordData_buf = "67";
1213                 srw_res->records[i].recordData_len = 2;
1214                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1215                 continue;
1216             }
1217             Z_External *r = npr->u.databaseRecord;
1218             oident *ent = oid_getentbyoid(r->direct_reference);
1219             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1220             {
1221                 srw_res->records[i].recordSchema = m_schema;
1222                 srw_res->records[i].recordPacking = m_s2z_packing;
1223                 srw_res->records[i].recordData_buf = (char*)
1224                     r->u.octet_aligned->buf;
1225                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1226                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1227             }
1228             else
1229             {
1230                 srw_res->records[i].recordSchema = "diagnostic";
1231                 srw_res->records[i].recordPacking = m_s2z_packing;
1232                 srw_res->records[i].recordData_buf = "67";
1233                 srw_res->records[i].recordData_len = 2;
1234                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1235             }
1236         }
1237     }
1238     if (records && records->which == Z_Records_NSD)
1239     {
1240         int http_code;
1241         http_code = z_to_srw_diag(odr_encode(), srw_res,
1242                                    records->u.nonSurrogateDiagnostic);
1243         if (http_code)
1244             return send_http_response(http_code);
1245     }
1246     return send_srw_response(srw_pdu);
1247
1248 }
1249
1250 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1251                                         int num_diagnostics)
1252 {
1253     ODR o = odr_encode();
1254     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1255     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1256
1257     srw_res->num_diagnostics = num_diagnostics;
1258     srw_res->diagnostics = diagnostics;
1259     return send_srw_response(srw_pdu);
1260 }
1261
1262 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1263                                         int num_diagnostics)
1264 {
1265     Yaz_ProxyConfig *cfg = check_reconfigure();
1266     if (cfg)
1267     {
1268         int len;
1269         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1270                                        m_s2z_database, &len);
1271         if (b)
1272         {
1273             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1274             Z_SRW_explainResponse *er = res->u.explain_response;
1275
1276             er->record.recordData_buf = b;
1277             er->record.recordData_len = len;
1278             er->record.recordPacking = m_s2z_packing;
1279             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1280
1281             er->diagnostics = diagnostics;
1282             er->num_diagnostics = num_diagnostics;
1283             return send_srw_response(res);
1284         }
1285     }
1286     return send_http_response(404);
1287 }
1288
1289 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1290 {
1291     if (m_http_version)
1292     {
1293         if (apdu->which == Z_APDU_initResponse)
1294         {
1295             Z_InitResponse *res = apdu->u.initResponse;
1296             if (*res->result == 0)
1297             {
1298                 send_to_srw_client_error(3, 0);
1299             }
1300             else if (!m_s2z_search_apdu)
1301             {
1302                 send_srw_explain_response(0, 0);
1303             }
1304             else
1305             {
1306                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1307             }
1308         }
1309         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1310         {
1311             m_s2z_search_apdu = 0;
1312             Z_SearchResponse *res = apdu->u.searchResponse;
1313             m_s2z_hit_count = *res->resultCount;
1314             if (res->records && res->records->which == Z_Records_NSD)
1315             {
1316                 send_to_srw_client_ok(0, res->records, 1);
1317             }
1318             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1319             {
1320                 // adjust
1321                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1322
1323                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1324                 {
1325                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1326                         > m_s2z_hit_count)
1327                         *pr->numberOfRecordsRequested =
1328                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1329                 }
1330                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1331             }
1332             else
1333             {
1334                 m_s2z_present_apdu = 0;
1335                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1336             }
1337         }
1338         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1339         {
1340             int start =
1341                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1342
1343             m_s2z_present_apdu = 0;
1344             Z_PresentResponse *res = apdu->u.presentResponse;
1345             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1346         }
1347     }
1348     else
1349     {
1350         int len = 0;
1351         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1352             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1353                      apdu_name(apdu));
1354         int r = send_Z_PDU(apdu, &len);
1355         m_bytes_sent += len;
1356         m_bw_stat.add_bytes(len);
1357         logtime();
1358         return r;
1359     }
1360     return 0;
1361 }
1362
1363 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1364 {
1365     int kill_session = 0;
1366     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1367
1368     if (new_id)
1369         *new_id = m_referenceId;
1370
1371     if (apdu->which == Z_APDU_searchResponse)
1372     {
1373         Z_SearchResponse *sr = apdu->u.searchResponse;
1374         Z_Records *p = sr->records;
1375         if (p && p->which == Z_Records_NSD)
1376         {
1377             Z_DiagRec dr, *dr_p = &dr;
1378             dr.which = Z_DiagRec_defaultFormat;
1379             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1380
1381             *sr->searchStatus = 0;
1382             display_diagrecs(&dr_p, 1);
1383         }
1384         else
1385         {
1386             if (p && p->which == Z_Records_DBOSD)
1387             {
1388                 if (m_backend_type
1389 #if HAVE_USEMARCON
1390                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1391 #endif
1392                     )
1393                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1394                 if (m_marcxml_mode == marcxml)
1395                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1396                                        m_backend_charset);
1397                 else
1398                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1399                                             m_backend_charset);
1400                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1401                     return 0;
1402
1403             }
1404             if (sr->resultCount)
1405             {
1406                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1407                         *sr->resultCount);
1408                 if (*sr->resultCount < 0)
1409                 {
1410                     m_flag_invalid_session = 1;
1411                     kill_session = 1;
1412
1413                     *sr->searchStatus = 0;
1414                     sr->records =
1415                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1416                     *sr->resultCount = 0;
1417                 }
1418             }
1419         }
1420     }
1421     else if (apdu->which == Z_APDU_presentResponse)
1422     {
1423         Z_PresentResponse *sr = apdu->u.presentResponse;
1424         Z_Records *p = sr->records;
1425         if (p && p->which == Z_Records_NSD)
1426         {
1427             Z_DiagRec dr, *dr_p = &dr;
1428             dr.which = Z_DiagRec_defaultFormat;
1429             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1430             if (*sr->presentStatus == Z_PresentStatus_success)
1431                 *sr->presentStatus = Z_PresentStatus_failure;
1432             display_diagrecs(&dr_p, 1);
1433         }
1434         if (p && p->which == Z_Records_DBOSD)
1435         {
1436             if (m_backend_type
1437 #if HAVE_USEMARCON
1438                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1439 #endif
1440                 )
1441                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1442             if (m_marcxml_mode == marcxml)
1443                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1444                                    m_backend_charset);
1445             else
1446                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1447                                         m_backend_charset);
1448             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1449                 return 0;
1450         }
1451     }
1452     else if (apdu->which == Z_APDU_initResponse)
1453     {
1454         //Get and check negotiation record
1455         //from init response.
1456         handle_charset_lang_negotiation(apdu);
1457
1458         if (m_initRequest_options)
1459         {
1460             Z_Options *nopt =
1461                 (Odr_bitmask *)odr_malloc(odr_encode(),
1462                                           sizeof(Odr_bitmask));
1463             ODR_MASK_ZERO(nopt);
1464
1465             int i;
1466             for (i = 0; i<24; i++)
1467                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1468                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1469                     ODR_MASK_SET(nopt, i);
1470             apdu->u.initResponse->options = nopt;
1471         }
1472         if (m_initRequest_version)
1473         {
1474             Z_ProtocolVersion *nopt =
1475                 (Odr_bitmask *)odr_malloc(odr_encode(),
1476                                           sizeof(Odr_bitmask));
1477             ODR_MASK_ZERO(nopt);
1478
1479             int i;
1480             for (i = 0; i<8; i++)
1481                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1482                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1483                     ODR_MASK_SET(nopt, i);
1484             apdu->u.initResponse->protocolVersion = nopt;
1485         }
1486         apdu->u.initResponse->preferredMessageSize =
1487             odr_intdup(odr_encode(),
1488                        m_client->m_initResponse_preferredMessageSize >
1489                        m_initRequest_preferredMessageSize ?
1490                        m_initRequest_preferredMessageSize :
1491                        m_client->m_initResponse_preferredMessageSize);
1492         apdu->u.initResponse->maximumRecordSize =
1493             odr_intdup(odr_encode(),
1494                        m_client->m_initResponse_maximumRecordSize >
1495                        m_initRequest_maximumRecordSize ?
1496                        m_initRequest_maximumRecordSize :
1497                        m_client->m_initResponse_maximumRecordSize);
1498     }
1499
1500     int r = send_PDU_convert(apdu);
1501     if (r)
1502         return r;
1503     if (kill_session)
1504     {
1505         delete m_client;
1506         m_client = 0;
1507         m_parent->pre_init();
1508     }
1509     return r;
1510 }
1511
1512 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1513 {
1514     int len = 0;
1515     const char *apdu_name_tmp = apdu_name(apdu);
1516     int r = send_Z_PDU(apdu, &len);
1517     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1518         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1519                  get_session_str(),
1520                  apdu_name_tmp, get_hostname(), len);
1521     m_bytes_sent += len;
1522     return r;
1523 }
1524
1525 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1526 {
1527     if (apdu->which == Z_APDU_presentRequest)
1528     {
1529         Z_PresentRequest *pr = apdu->u.presentRequest;
1530         int toget = *pr->numberOfRecordsRequested;
1531         int start = *pr->resultSetStartPoint;
1532
1533         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1534                 pr->resultSetId, start, toget);
1535
1536         if (*m_parent->m_optimize == '0')
1537             return apdu;
1538
1539         if (!m_client->m_last_resultSetId)
1540         {
1541             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1542             new_apdu->u.presentResponse->records =
1543                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1544                                                pr->resultSetId);
1545             send_to_client(new_apdu);
1546             return 0;
1547         }
1548         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1549         {
1550             if (start+toget-1 > m_client->m_last_resultCount)
1551             {
1552                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1553                 new_apdu->u.presentResponse->records =
1554                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1555                 send_to_client(new_apdu);
1556                 return 0;
1557             }
1558             Z_NamePlusRecordList *npr;
1559 #if 0
1560             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1561                     m_session_str, start, toget, yaz_z3950oid_to_str(
1562                         pr->preferredRecordSyntax, &oclass));
1563 #endif
1564             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1565                                           pr->preferredRecordSyntax,
1566                                           pr->recordComposition))
1567             {
1568                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1569                          m_session_str);
1570                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1571                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1572
1573                 new_apdu->u.presentResponse->numberOfRecordsReturned
1574                     = odr_intdup(odr_encode(), toget);
1575
1576                 new_apdu->u.presentResponse->records = (Z_Records*)
1577                     odr_malloc(odr_encode(), sizeof(Z_Records));
1578                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1579                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1580                 new_apdu->u.presentResponse->nextResultSetPosition =
1581                     odr_intdup(odr_encode(), start+toget);
1582
1583                 send_to_client(new_apdu);
1584                 return 0;
1585             }
1586         }
1587     }
1588
1589     if (apdu->which != Z_APDU_searchRequest)
1590         return apdu;
1591     Z_SearchRequest *sr = apdu->u.searchRequest;
1592     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1593     Yaz_Z_Databases this_databases;
1594
1595     this_databases.set(sr->num_databaseNames, (const char **)
1596                        sr->databaseNames);
1597
1598     this_query->set_Z_Query(sr->query);
1599
1600     char query_str[120];
1601     this_query->print(query_str, sizeof(query_str)-1);
1602     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1603
1604     if (*m_parent->m_optimize != '0' &&
1605         m_client->m_last_ok && m_client->m_last_query &&
1606         m_client->m_last_query->match(this_query) &&
1607         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1608         m_client->m_last_databases.match(this_databases))
1609     {
1610         delete this_query;
1611         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1612             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1613         {
1614             Z_NamePlusRecordList *npr;
1615             int toget = *sr->mediumSetPresentNumber;
1616             Z_RecordComposition *comp = 0;
1617
1618             if (toget > m_client->m_last_resultCount)
1619                 toget = m_client->m_last_resultCount;
1620
1621             if (sr->mediumSetElementSetNames)
1622             {
1623                 comp = (Z_RecordComposition *)
1624                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1625                 comp->which = Z_RecordComp_simple;
1626                 comp->u.simple = sr->mediumSetElementSetNames;
1627             }
1628
1629             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1630                                           sr->preferredRecordSyntax, comp))
1631             {
1632                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1633                          m_session_str);
1634                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1635                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1636                 new_apdu->u.searchResponse->resultCount =
1637                     &m_client->m_last_resultCount;
1638
1639                 new_apdu->u.searchResponse->numberOfRecordsReturned
1640                     = odr_intdup(odr_encode(), toget);
1641
1642                 new_apdu->u.searchResponse->presentStatus =
1643                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1644                 new_apdu->u.searchResponse->records = (Z_Records*)
1645                     odr_malloc(odr_encode(), sizeof(Z_Records));
1646                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1647                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1648                 new_apdu->u.searchResponse->nextResultSetPosition =
1649                     odr_intdup(odr_encode(), toget+1);
1650                 send_to_client(new_apdu);
1651                 return 0;
1652             }
1653             else
1654             {
1655                 // medium Set
1656                 // send present request (medium size)
1657                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1658                          m_session_str);
1659
1660                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1661                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1662                 pr->referenceId = sr->referenceId;
1663                 pr->resultSetId = sr->resultSetName;
1664                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1665                 *pr->numberOfRecordsRequested = toget;
1666                 pr->recordComposition = comp;
1667                 m_client->m_sr_transform = 1;
1668                 return new_apdu;
1669             }
1670         }
1671         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1672             m_client->m_last_resultCount <= 0)
1673         {
1674             // large set. Return pseudo-search response immediately
1675             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1676                      m_session_str);
1677             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1678             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1679             new_apdu->u.searchResponse->resultCount =
1680                 &m_client->m_last_resultCount;
1681             send_to_client(new_apdu);
1682             return 0;
1683         }
1684         else
1685         {
1686             Z_NamePlusRecordList *npr;
1687             int toget = m_client->m_last_resultCount;
1688             Z_RecordComposition *comp = 0;
1689             // small set
1690             // send a present request (small set)
1691
1692             if (sr->smallSetElementSetNames)
1693             {
1694                 comp = (Z_RecordComposition *)
1695                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1696                 comp->which = Z_RecordComp_simple;
1697                 comp->u.simple = sr->smallSetElementSetNames;
1698             }
1699
1700             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1701                                           sr->preferredRecordSyntax, comp))
1702             {
1703                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1704                          m_session_str);
1705                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1706                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1707                 new_apdu->u.searchResponse->resultCount =
1708                     &m_client->m_last_resultCount;
1709
1710                 new_apdu->u.searchResponse->numberOfRecordsReturned
1711                     = odr_intdup(odr_encode(), toget);
1712
1713                 new_apdu->u.searchResponse->presentStatus =
1714                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1715                 new_apdu->u.searchResponse->records = (Z_Records*)
1716                     odr_malloc(odr_encode(), sizeof(Z_Records));
1717                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1718                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1719                 new_apdu->u.searchResponse->nextResultSetPosition =
1720                     odr_intdup(odr_encode(), toget+1);
1721                 send_to_client(new_apdu);
1722                 return 0;
1723             }
1724             else
1725             {
1726                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1727                          m_session_str);
1728                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1729                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1730                 pr->referenceId = sr->referenceId;
1731                 pr->resultSetId = sr->resultSetName;
1732                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1733                 *pr->numberOfRecordsRequested = toget;
1734                 pr->recordComposition = comp;
1735                 m_client->m_sr_transform = 1;
1736                 return new_apdu;
1737             }
1738         }
1739     }
1740     else  // query doesn't match
1741     {
1742         delete m_client->m_last_query;
1743         m_client->m_last_query = this_query;
1744         m_client->m_last_ok = 0;
1745         m_client->m_cache.clear();
1746         m_client->m_resultSetStartPoint = 0;
1747
1748         xfree (m_client->m_last_resultSetId);
1749         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1750
1751         m_client->m_last_databases.set(sr->num_databaseNames,
1752                                        (const char **) sr->databaseNames);
1753     }
1754     return apdu;
1755 }
1756
1757
1758 void Yaz_Proxy::inc_request_no()
1759 {
1760     char *cp = strchr(m_session_str, ' ');
1761     m_request_no++;
1762     if (cp)
1763         sprintf(cp+1, "%d ", m_request_no);
1764 }
1765
1766 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1767 {
1768     inc_request_no();
1769
1770     m_bytes_recv += len;
1771
1772     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1773         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1774                  m_session_str, gdu_name(apdu), len);
1775
1776 #if HAVE_GETTIMEOFDAY
1777     gettimeofday((struct timeval *) m_time_tv, 0);
1778 #endif
1779     m_bw_stat.add_bytes(len);
1780     m_pdu_stat.add_bytes(1);
1781
1782     GDU *gdu = new GDU(apdu);
1783     m_in_queue.enqueue(gdu);
1784
1785     recv_GDU_more(false);
1786 }
1787
1788 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
1789 {
1790     int bw_total = m_bw_stat.get_total();
1791     int pdu_total = m_pdu_stat.get_total();
1792     int reduce = 0;
1793
1794     assert(m_timeout_mode == timeout_busy);
1795     assert(m_timeout_gdu == 0);
1796
1797     if (m_bw_max)
1798     {
1799         if (bw_total > m_bw_max)
1800         {
1801             reduce = (bw_total/m_bw_max);
1802         }
1803     }
1804     if (m_pdu_max)
1805     {
1806         if (pdu_total > m_pdu_max)
1807         {
1808             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1809             reduce = (reduce > nreduce) ? reduce : nreduce;
1810         }
1811     }
1812     m_http_version = 0;
1813
1814 #if 0
1815     /* uncomment to force a big reduce */
1816     m_timeout_mode = timeout_reduce;
1817     m_timeout_gdu = gdu;
1818     timeout(3);       // call us reduce seconds later
1819     return;
1820 #endif
1821     if (reduce)
1822     {
1823         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1824                 m_session_str, reduce, bw_total, pdu_total,
1825                 m_bw_max, m_pdu_max);
1826
1827         m_timeout_mode = timeout_reduce;
1828         m_timeout_gdu = gdu;
1829         timeout(reduce);       // call us reduce seconds later
1830     }
1831     else
1832         recv_GDU_normal(gdu);
1833 }
1834
1835 void Yaz_Proxy::recv_GDU_more(bool normal)
1836 {
1837     GDU *g;
1838     if (normal && m_timeout_mode == timeout_busy)
1839         m_timeout_mode = timeout_normal;
1840     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
1841     {
1842         m_timeout_mode = timeout_busy;
1843         recv_GDU_reduce(g);
1844     }
1845 }
1846
1847 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
1848 {
1849     Z_GDU *apdu = 0;
1850     gdu->move_away_gdu(odr_decode(), &apdu);
1851     delete gdu;
1852
1853     if (apdu->which == Z_GDU_Z3950)
1854         handle_incoming_Z_PDU(apdu->u.z3950);
1855     else if (apdu->which == Z_GDU_HTTP_Request)
1856         handle_incoming_HTTP(apdu->u.HTTP_Request);
1857 }
1858
1859 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1860 {
1861     if (m_max_record_retrieve)
1862     {
1863         if (apdu->which == Z_APDU_presentRequest)
1864         {
1865             Z_PresentRequest *pr = apdu->u.presentRequest;
1866             if (pr->numberOfRecordsRequested &&
1867                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1868                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1869         }
1870     }
1871 }
1872
1873 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1874 {
1875     if (apdu->which == Z_APDU_initRequest)
1876     {
1877         yaz_log(YLOG_LOG, "%shandle_charset_lang_negotiation",
1878                 m_session_str);
1879         if (m_initRequest_options &&
1880             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1881             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1882         {
1883             // There is no negotiation proposal from
1884             // client's side. OK. The proxy negotiation
1885             // in use, only.
1886             Z_InitRequest *initRequest = apdu->u.initRequest;
1887             Z_OtherInformation **otherInfo;
1888             Z_OtherInformationUnit *oi;
1889             get_otherInfoAPDU(apdu, &otherInfo);
1890             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1891             if (oi)
1892             {
1893                 ODR_MASK_SET(initRequest->options,
1894                     Z_Options_negotiationModel);
1895                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1896                 oi->information.externallyDefinedInfo =
1897                 yaz_set_proposal_charneg(odr_encode(),
1898                     (const char**)&m_proxy_negotiation_charset,
1899                     m_proxy_negotiation_charset ? 1:0,
1900                     (const char**)&m_proxy_negotiation_lang,
1901                     m_proxy_negotiation_lang ? 1:0,
1902                     1);
1903             }
1904         }
1905         else if (m_initRequest_options &&
1906                  ODR_MASK_GET(m_initRequest_options,
1907                               Z_Options_negotiationModel) &&
1908                  m_charset_converter->get_target_query_charset())
1909         {
1910             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1911                     m_session_str,
1912                     m_charset_converter->get_target_query_charset());
1913             Z_InitRequest *initRequest = apdu->u.initRequest;
1914             Z_CharSetandLanguageNegotiation *negotiation =
1915                 yaz_get_charneg_record (initRequest->otherInfo);
1916             if (negotiation &&
1917                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1918             {
1919                 NMEM nmem = nmem_create();
1920                 char **charsets = 0;
1921                 int num_charsets = 0;
1922                 char **langs = 0;
1923                 int num_langs = 0;
1924                 int selected = 0;
1925                 yaz_get_proposal_charneg (nmem, negotiation,
1926                                           &charsets, &num_charsets,
1927                                           &langs, &num_langs, &selected);
1928                 int i;
1929                 for (i = 0; i<num_charsets; i++)
1930                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
1931                             charsets[i]);
1932                 for (i = 0; i<num_langs; i++)
1933                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
1934                             langs[i]);
1935
1936                 const char *t_charset =
1937                     m_charset_converter->get_target_query_charset();
1938                 // sweep through charsets and pick the first supported
1939                 // conversion
1940                 for (i = 0; i<num_charsets; i++)
1941                 {
1942                     const char *c_charset = charsets[i];
1943                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
1944                         break;
1945                 }
1946                 if (i != num_charsets)
1947                 {
1948                     // got one .. set up ODR for reverse direction
1949                     const char *c_charset = charsets[i];
1950                     odr_set_charset(odr_encode(), c_charset, t_charset);
1951                     m_charset_converter->set_client_query_charset(c_charset);
1952                     m_charset_converter->set_client_charset_selected(selected);
1953                 }
1954                 nmem_destroy(nmem);
1955                 ODR_MASK_CLEAR(m_initRequest_options,
1956                                Z_Options_negotiationModel);
1957                 yaz_del_charneg_record(&initRequest->otherInfo);
1958             }
1959             else
1960             {
1961                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
1962                         m_session_str);
1963             }
1964         }
1965     }
1966     else if (apdu->which == Z_APDU_initResponse)
1967     {
1968         Z_InitResponse *initResponse = apdu->u.initResponse;
1969         Z_OtherInformation **otherInfo;
1970
1971         if (ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel))
1972         {
1973             char *charset = 0;
1974             char *lang = 0;
1975             int selected = 0;
1976
1977             get_otherInfoAPDU(apdu, &otherInfo);
1978
1979             if (!otherInfo && !(*otherInfo))
1980                 return;
1981
1982             Z_CharSetandLanguageNegotiation *charneg =
1983                 yaz_get_charneg_record(*otherInfo);
1984
1985             if (!charneg)
1986                 return;
1987
1988             yaz_get_response_charneg(m_referenceId_mem, charneg,
1989                 &charset, &lang, &selected);
1990
1991             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
1992                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
1993
1994             if (m_initRequest_options &&
1995                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
1996             {
1997                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
1998                     m_session_str);
1999             }
2000             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2001             {
2002                 // negotiation-charset, negotiation-lang
2003                 // elements of config file in use.
2004
2005                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2006                     m_session_str);
2007
2008                 // clear negotiation option.
2009                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2010
2011                 // Delete negotiation (charneg-3) entry.
2012                 yaz_del_charneg_record(otherInfo);
2013             }
2014         }
2015         else
2016         {
2017             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2018             {
2019                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2020                         m_session_str);
2021             }
2022             else if (m_charset_converter->get_client_query_charset())
2023             {
2024                 Z_OtherInformation **otherInfo;
2025                 Z_OtherInformationUnit *oi;
2026                 get_otherInfoAPDU(apdu, &otherInfo);
2027                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2028                 if (oi)
2029                 {
2030                     ODR_MASK_SET(initResponse->options,
2031                                  Z_Options_negotiationModel);
2032                     if (m_initRequest_options)
2033                         ODR_MASK_SET(m_initRequest_options,
2034                                      Z_Options_negotiationModel);
2035
2036                     oi->which = Z_OtherInfo_externallyDefinedInfo;
2037                     oi->information.externallyDefinedInfo =
2038                         yaz_set_response_charneg(
2039                             odr_encode(),
2040                             m_charset_converter->get_client_query_charset(),
2041                             0 /* no lang */,
2042                             m_charset_converter->get_client_charset_selected());
2043                 }
2044             }
2045         }
2046     }
2047 }
2048
2049 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2050                                                      int error,
2051                                                      const char *addinfo)
2052 {
2053     Z_Records *rec = (Z_Records *)
2054         odr_malloc (odr, sizeof(*rec));
2055     int *err = (int *)
2056         odr_malloc (odr, sizeof(*err));
2057     Z_DiagRec *drec = (Z_DiagRec *)
2058         odr_malloc (odr, sizeof(*drec));
2059     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2060         odr_malloc (odr, sizeof(*dr));
2061     *err = error;
2062     rec->which = Z_Records_NSD;
2063     rec->u.nonSurrogateDiagnostic = dr;
2064     dr->diagnosticSetId =
2065         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
2066     dr->condition = err;
2067     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2068     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2069     return rec;
2070 }
2071
2072 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2073 {
2074     if (apdu->which == Z_APDU_searchRequest &&
2075         apdu->u.searchRequest->query &&
2076         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2077         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2078     {
2079         Z_RPNQuery *rpnquery = 0;
2080         Z_SearchRequest *sr = apdu->u.searchRequest;
2081         char *addinfo = 0;
2082
2083         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2084                 sr->query->u.type_104->u.cql);
2085
2086         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2087                                           &rpnquery, odr_encode(),
2088                                           &addinfo);
2089         if (r == -3)
2090             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2091         else if (r)
2092         {
2093             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2094             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2095
2096             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2097             new_apdu->u.searchResponse->records =
2098                 create_nonSurrogateDiagnostics(odr_encode(),
2099                                                yaz_diag_srw_to_bib1(r),
2100                                                addinfo);
2101             *new_apdu->u.searchResponse->searchStatus = 0;
2102
2103             send_to_client(new_apdu);
2104
2105             return 0;
2106         }
2107         else
2108         {
2109             sr->query->which = Z_Query_type_1;
2110             sr->query->u.type_1 = rpnquery;
2111         }
2112         return apdu;
2113     }
2114     return apdu;
2115 }
2116
2117 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2118 {
2119     if (apdu->which == Z_APDU_searchRequest &&
2120         apdu->u.searchRequest->query)
2121     {
2122         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2123             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2124         {
2125             if (m_http_version)
2126                 m_charset_converter->set_client_query_charset("UTF-8");
2127             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2128             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2129         }
2130     }
2131     return apdu;
2132 }
2133
2134
2135 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2136 {
2137     if (apdu->which == Z_APDU_searchRequest)
2138     {
2139         Z_SearchRequest *sr = apdu->u.searchRequest;
2140         int err = 0;
2141         char *addinfo = 0;
2142
2143         Yaz_ProxyConfig *cfg = check_reconfigure();
2144         if (cfg)
2145             err = cfg->check_query(odr_encode(), m_default_target,
2146                                    sr->query, &addinfo);
2147         if (err)
2148         {
2149             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2150
2151             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2152             new_apdu->u.searchResponse->records =
2153                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2154             *new_apdu->u.searchResponse->searchStatus = 0;
2155
2156             send_to_client(new_apdu);
2157
2158             return 0;
2159         }
2160     }
2161     return apdu;
2162 }
2163
2164 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2165 {
2166     if (apdu->which != Z_APDU_initRequest)
2167         return 1;  // pass if no init request
2168     Z_InitRequest *req = apdu->u.initRequest;
2169
2170     Yaz_ProxyConfig *cfg = check_reconfigure();
2171     if (!cfg)
2172         return 1;  // pass if no config
2173
2174     int ret;
2175     if (req->idAuthentication == 0)
2176     {
2177         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2178                                          m_peername);
2179     }
2180     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2181     {
2182         ret = cfg->client_authentication(
2183             m_default_target,
2184             req->idAuthentication->u.idPass->userId,
2185             req->idAuthentication->u.idPass->groupId,
2186             req->idAuthentication->u.idPass->password,
2187             m_peername);
2188     }
2189     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2190     {
2191         char user[64], pass[64];
2192         *user = '\0';
2193         *pass = '\0';
2194         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2195         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2196                                          m_peername);
2197     }
2198     else
2199         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2200                                          m_peername);
2201
2202     cfg->target_authentication(m_default_target, odr_encode(), req);
2203
2204     return ret;
2205 }
2206
2207 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2208 {
2209     m_marcxml_mode = none;
2210     if (apdu->which == Z_APDU_searchRequest)
2211     {
2212         Z_SearchRequest *sr = apdu->u.searchRequest;
2213         int err = 0;
2214         char *addinfo = 0;
2215         Yaz_ProxyConfig *cfg = check_reconfigure();
2216
2217         Z_RecordComposition rc_temp, *rc = 0;
2218         if (sr->smallSetElementSetNames)
2219         {
2220             rc_temp.which = Z_RecordComp_simple;
2221             rc_temp.u.simple = sr->smallSetElementSetNames;
2222             rc = &rc_temp;
2223         }
2224
2225         if (sr->preferredRecordSyntax)
2226         {
2227             struct oident *ent;
2228             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2229             m_frontend_type = ent->value;
2230         }
2231         else
2232             m_frontend_type = VAL_NONE;
2233
2234         char *stylesheet_name = 0;
2235         if (cfg)
2236             err = cfg->check_syntax(odr_encode(),
2237                                     m_default_target,
2238                                     sr->preferredRecordSyntax, rc,
2239                                     &addinfo, &stylesheet_name, &m_schema,
2240                                     &m_backend_type, &m_backend_charset,
2241                                     &m_usemarcon_ini_stage1,
2242                                     &m_usemarcon_ini_stage2);
2243         if (stylesheet_name)
2244         {
2245             m_parent->low_socket_close();
2246
2247 #if HAVE_XSLT
2248             if (m_stylesheet_xsp)
2249                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2250             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2251                                                        stylesheet_name);
2252 #endif
2253             m_stylesheet_offset = 0;
2254             xfree(stylesheet_name);
2255
2256             m_parent->low_socket_open();
2257         }
2258         if (err == -1)
2259         {
2260             sr->smallSetElementSetNames = 0;
2261             sr->mediumSetElementSetNames = 0;
2262             m_marcxml_mode = marcxml;
2263             if (m_backend_type)
2264             {
2265
2266                 sr->preferredRecordSyntax =
2267                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2268                                         m_backend_type);
2269             }
2270             else
2271                 sr->preferredRecordSyntax =
2272                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2273                                            VAL_USMARC);
2274         }
2275         else if (err)
2276         {
2277             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2278
2279             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2280             new_apdu->u.searchResponse->records =
2281                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2282             *new_apdu->u.searchResponse->searchStatus = 0;
2283
2284             send_to_client(new_apdu);
2285
2286             return 0;
2287         }
2288         else if (m_backend_type)
2289         {
2290             sr->preferredRecordSyntax =
2291                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2292         }
2293     }
2294     else if (apdu->which == Z_APDU_presentRequest)
2295     {
2296         Z_PresentRequest *pr = apdu->u.presentRequest;
2297         int err = 0;
2298         char *addinfo = 0;
2299         Yaz_ProxyConfig *cfg = check_reconfigure();
2300
2301         if (pr->preferredRecordSyntax)
2302         {
2303             struct oident *ent;
2304             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2305             m_frontend_type = ent->value;
2306         }
2307         else
2308             m_frontend_type = VAL_NONE;
2309
2310         char *stylesheet_name = 0;
2311         if (cfg)
2312             err = cfg->check_syntax(odr_encode(), m_default_target,
2313                                     pr->preferredRecordSyntax,
2314                                     pr->recordComposition,
2315                                     &addinfo, &stylesheet_name, &m_schema,
2316                                     &m_backend_type, &m_backend_charset,
2317                                     &m_usemarcon_ini_stage1,
2318                                     &m_usemarcon_ini_stage2
2319                                     );
2320         if (stylesheet_name)
2321         {
2322             m_parent->low_socket_close();
2323
2324 #if HAVE_XSLT
2325             if (m_stylesheet_xsp)
2326                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2327             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2328                                                        stylesheet_name);
2329 #endif
2330             m_stylesheet_offset = 0;
2331             xfree(stylesheet_name);
2332
2333             m_parent->low_socket_open();
2334         }
2335         if (err == -1)
2336         {
2337             pr->recordComposition = 0;
2338             m_marcxml_mode = marcxml;
2339             if (m_backend_type)
2340             {
2341
2342                 pr->preferredRecordSyntax =
2343                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2344                                         m_backend_type);
2345             }
2346             else
2347                 pr->preferredRecordSyntax =
2348                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2349                                            VAL_USMARC);
2350         }
2351         else if (err)
2352         {
2353             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2354
2355             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2356             new_apdu->u.presentResponse->records =
2357                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2358             *new_apdu->u.presentResponse->presentStatus =
2359                 Z_PresentStatus_failure;
2360
2361             send_to_client(new_apdu);
2362
2363             return 0;
2364         }
2365         else if (m_backend_type)
2366         {
2367             pr->preferredRecordSyntax =
2368                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2369         }
2370     }
2371     return apdu;
2372 }
2373
2374 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2375 {
2376     if (!schema)
2377         return 0;
2378     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2379         odr_malloc(o, sizeof(Z_ElementSetNames));
2380     esn->which = Z_ElementSetNames_generic;
2381     esn->u.generic = odr_strdup(o, schema);
2382     return esn;
2383 }
2384
2385 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2386 {
2387     const char *t = 0;
2388     Yaz_ProxyConfig *cfg = check_reconfigure();
2389     if (cfg)
2390         t = cfg->get_explain_name(db, backend_db);
2391
2392     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2393     {
2394         releaseClient();
2395     }
2396
2397     if (t)
2398     {
2399         xfree(m_default_target);
2400         m_default_target = xstrdup(t);
2401     }
2402 }
2403
2404 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2405 {
2406     struct stat sbuf;
2407     if (strcmp(hreq->method, "GET"))
2408         return 0;
2409     if (hreq->path[0] != '/')
2410     {
2411         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2412         return 0;
2413     }
2414     const char *cp = hreq->path;
2415     while (*cp)
2416     {
2417         if (*cp == '/' && strchr("/.", cp[1]))
2418         {
2419             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2420             return 0;
2421         }
2422         cp++;
2423     }
2424     const char *fname = hreq->path+1;
2425     if (stat(fname, &sbuf))
2426     {
2427         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2428         return 0;
2429     }
2430     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2431     {
2432         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2433         return 0;
2434     }
2435     if (sbuf.st_size > (off_t) 1000000)
2436     {
2437         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2438         return 0;
2439     }
2440
2441     ODR o = odr_encode();
2442     Yaz_ProxyConfig *cfg = check_reconfigure();
2443     const char *ctype = cfg->check_mime_type(fname);
2444     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2445     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2446     if (m_http_version)
2447         hres->version = odr_strdup(o, m_http_version);
2448     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2449     if (m_http_keepalive)
2450         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2451     else
2452         timeout(0);
2453
2454     hres->content_len = sbuf.st_size;
2455     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2456     FILE *f = fopen(fname, "rb");
2457     if (f)
2458     {
2459         fread(hres->content_buf, 1, hres->content_len, f);
2460         fclose(f);
2461     }
2462     else
2463     {
2464         return 0;
2465     }
2466     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2467     {
2468         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2469                  fname);
2470     }
2471     int len;
2472     send_GDU(gdu, &len);
2473     recv_GDU_more(true);
2474     return 1;
2475 }
2476
2477 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2478 {
2479     if (m_s2z_odr_init)
2480     {
2481         odr_destroy(m_s2z_odr_init);
2482         m_s2z_odr_init = 0;
2483     }
2484     if (m_s2z_odr_search)
2485     {
2486         odr_destroy(m_s2z_odr_search);
2487         m_s2z_odr_search = 0;
2488     }
2489
2490     m_http_keepalive = 0;
2491     m_http_version = 0;
2492     if (!strcmp(hreq->version, "1.0"))
2493     {
2494         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2495         if (v && !strcmp(v, "Keep-Alive"))
2496             m_http_keepalive = 1;
2497         else
2498             m_http_keepalive = 0;
2499         m_http_version = "1.0";
2500     }
2501     else
2502     {
2503         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2504         if (v && !strcmp(v, "close"))
2505             m_http_keepalive = 0;
2506         else
2507             m_http_keepalive = 1;
2508         m_http_version = "1.1";
2509     }
2510
2511     Z_SRW_PDU *srw_pdu = 0;
2512     Z_SOAP *soap_package = 0;
2513     char *charset = 0;
2514     Z_SRW_diagnostic *diagnostic = 0;
2515     int num_diagnostic = 0;
2516
2517     if (file_access(hreq))
2518     {
2519         return;
2520     }
2521     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2522                             &charset) == 0
2523              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2524                                &charset, &diagnostic, &num_diagnostic) == 0)
2525     {
2526         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2527         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2528         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2529         m_s2z_init_apdu = 0;
2530         m_s2z_search_apdu = 0;
2531         m_s2z_present_apdu = 0;
2532
2533         m_s2z_stylesheet = 0;
2534
2535         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2536         {
2537             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2538
2539             const char *backend_db = srw_req->database;
2540             srw_get_client(srw_req->database, &backend_db);
2541
2542             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2543             // recordXPath unsupported.
2544             if (srw_req->recordXPath)
2545             {
2546                 yaz_add_srw_diagnostic(odr_decode(),
2547                                        &diagnostic, &num_diagnostic,
2548                                        72, 0);
2549             }
2550             // sort unsupported
2551             if (srw_req->sort_type != Z_SRW_sort_type_none)
2552             {
2553                 yaz_add_srw_diagnostic(odr_decode(),
2554                                        &diagnostic, &num_diagnostic,
2555                                        80, 0);
2556             }
2557             // save stylesheet
2558             if (srw_req->stylesheet)
2559                 m_s2z_stylesheet =
2560                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2561
2562             // set packing for response records ..
2563             if (srw_req->recordPacking &&
2564                 !strcmp(srw_req->recordPacking, "xml"))
2565                 m_s2z_packing = Z_SRW_recordPacking_XML;
2566             else
2567                 m_s2z_packing = Z_SRW_recordPacking_string;
2568
2569             if (num_diagnostic)
2570             {
2571                 Z_SRW_PDU *srw_pdu =
2572                     yaz_srw_get(odr_encode(),
2573                                 Z_SRW_searchRetrieve_response);
2574                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2575
2576                 srw_res->diagnostics = diagnostic;
2577                 srw_res->num_diagnostics = num_diagnostic;
2578                 send_srw_response(srw_pdu);
2579                 return;
2580             }
2581
2582             // prepare search PDU
2583             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2584                                           Z_APDU_searchRequest);
2585             Z_SearchRequest *z_searchRequest =
2586                 m_s2z_search_apdu->u.searchRequest;
2587
2588             z_searchRequest->num_databaseNames = 1;
2589             z_searchRequest->databaseNames = (char**)
2590                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2591             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2592                                                            backend_db);
2593
2594             // query transformation
2595             Z_Query *query = (Z_Query *)
2596                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2597             z_searchRequest->query = query;
2598
2599             if (srw_req->query_type == Z_SRW_query_type_cql)
2600             {
2601                 Z_External *ext = (Z_External *)
2602                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2603                 ext->direct_reference =
2604                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2605                 ext->indirect_reference = 0;
2606                 ext->descriptor = 0;
2607                 ext->which = Z_External_CQL;
2608                 ext->u.cql = srw_req->query.cql;
2609
2610                 query->which = Z_Query_type_104;
2611                 query->u.type_104 =  ext;
2612             }
2613             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2614             {
2615                 Z_RPNQuery *RPNquery;
2616                 YAZ_PQF_Parser pqf_parser;
2617
2618                 pqf_parser = yaz_pqf_create ();
2619
2620                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2621                                           srw_req->query.pqf);
2622                 if (!RPNquery)
2623                 {
2624                     const char *pqf_msg;
2625                     size_t off;
2626                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2627                     int ioff = off;
2628                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
2629                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2630
2631                     send_to_srw_client_error(10, 0);
2632                     return;
2633                 }
2634                 query->which = Z_Query_type_1;
2635                 query->u.type_1 =  RPNquery;
2636
2637                 yaz_pqf_destroy (pqf_parser);
2638             }
2639             else
2640             {
2641                 send_to_srw_client_error(7, "query");
2642                 return;
2643             }
2644
2645             // present
2646             m_s2z_present_apdu = 0;
2647             int max = 0;
2648             if (srw_req->maximumRecords)
2649                 max = *srw_req->maximumRecords;
2650             int start = 1;
2651             if (srw_req->startRecord)
2652                 start = *srw_req->startRecord;
2653             if (max > 0)
2654             {
2655                 // Some backend, such as Voyager doesn't honor piggyback
2656                 // So we use present always (0 &&).
2657                 if (0 && start <= 1)  // Z39.50 piggyback
2658                 {
2659                     *z_searchRequest->smallSetUpperBound = max;
2660                     *z_searchRequest->mediumSetPresentNumber = max;
2661                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2662
2663                     z_searchRequest->preferredRecordSyntax =
2664                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2665                                                VAL_TEXT_XML);
2666                     if (srw_req->recordSchema)
2667                     {
2668                         z_searchRequest->smallSetElementSetNames =
2669                             z_searchRequest->mediumSetElementSetNames =
2670                             mk_esn_from_schema(m_s2z_odr_search,
2671                                                srw_req->recordSchema);
2672                     }
2673                 }
2674                 else   // Z39.50 present
2675                 {
2676                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2677                                                    Z_APDU_presentRequest);
2678                     Z_PresentRequest *z_presentRequest =
2679                         m_s2z_present_apdu->u.presentRequest;
2680                     *z_presentRequest->resultSetStartPoint = start;
2681                     *z_presentRequest->numberOfRecordsRequested = max;
2682                     z_presentRequest->preferredRecordSyntax =
2683                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2684                                                VAL_TEXT_XML);
2685                     if (srw_req->recordSchema)
2686                     {
2687                         z_presentRequest->recordComposition =
2688                             (Z_RecordComposition *)
2689                             odr_malloc(m_s2z_odr_search,
2690                                        sizeof(Z_RecordComposition));
2691                         z_presentRequest->recordComposition->which =
2692                             Z_RecordComp_simple;
2693                         z_presentRequest->recordComposition->u.simple =
2694                             mk_esn_from_schema(m_s2z_odr_search,
2695                                                srw_req->recordSchema);
2696                     }
2697                 }
2698             }
2699             if (!m_client)
2700             {
2701                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2702                                             Z_APDU_initRequest);
2703
2704                 // prevent m_initRequest_apdu memory from being grabbed
2705                 // in Yaz_Proxy::handle_incoming_Z_PDU
2706                 m_initRequest_apdu = m_s2z_init_apdu;
2707                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2708                 return;
2709             }
2710             else
2711             {
2712                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2713                 return;
2714             }
2715         }
2716         else if (srw_pdu->which == Z_SRW_explain_request)
2717         {
2718             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2719
2720             const char *backend_db = srw_req->database;
2721             srw_get_client(srw_req->database, &backend_db);
2722
2723             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2724
2725             // save stylesheet
2726             if (srw_req->stylesheet)
2727                 m_s2z_stylesheet =
2728                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2729
2730             if (srw_req->recordPacking &&
2731                 !strcmp(srw_req->recordPacking, "xml"))
2732                 m_s2z_packing = Z_SRW_recordPacking_XML;
2733             else
2734                 m_s2z_packing = Z_SRW_recordPacking_string;
2735
2736             if (num_diagnostic)
2737             {
2738                 send_srw_explain_response(diagnostic, num_diagnostic);
2739                 return;
2740             }
2741
2742             if (!m_client)
2743             {
2744                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2745                                             Z_APDU_initRequest);
2746
2747                 // prevent m_initRequest_apdu memory from being grabbed
2748                 // in Yaz_Proxy::handle_incoming_Z_PDU
2749                 m_initRequest_apdu = m_s2z_init_apdu;
2750                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2751             }
2752             else
2753                 send_srw_explain_response(0, 0);
2754             return;
2755         }
2756         else if (srw_pdu->which == Z_SRW_scan_request)
2757         {
2758             m_s2z_database = odr_strdup(m_s2z_odr_init,
2759                                         srw_pdu->u.scan_request->database);
2760
2761             yaz_add_srw_diagnostic(odr_decode(),
2762                                    &diagnostic, &num_diagnostic,
2763                                    4, "scan");
2764             Z_SRW_PDU *srw_pdu =
2765                 yaz_srw_get(odr_encode(),
2766                             Z_SRW_scan_response);
2767             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2768
2769             srw_res->diagnostics = diagnostic;
2770             srw_res->num_diagnostics = num_diagnostic;
2771             send_srw_response(srw_pdu);
2772             return;
2773         }
2774         else
2775         {
2776             m_s2z_database = 0;
2777
2778             send_to_srw_client_error(4, 0);
2779         }
2780     }
2781     send_http_response(400);
2782 }
2783
2784 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2785 {
2786     Z_OtherInformation **oi;
2787     get_otherInfoAPDU(apdu, &oi);
2788
2789     if (apdu->u.initRequest->implementationId)
2790         yaz_log(YLOG_LOG, "%simplementationId: %s",
2791                 m_session_str, apdu->u.initRequest->implementationId);
2792     if (apdu->u.initRequest->implementationName)
2793         yaz_log(YLOG_LOG, "%simplementationName: %s",
2794                 m_session_str, apdu->u.initRequest->implementationName);
2795     if (apdu->u.initRequest->implementationVersion)
2796         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2797                 m_session_str, apdu->u.initRequest->implementationVersion);
2798     if (m_initRequest_apdu == 0)
2799     {
2800         if (m_initRequest_mem)
2801             nmem_destroy(m_initRequest_mem);
2802
2803         m_initRequest_apdu = apdu;
2804         m_initRequest_mem = odr_extract_mem(odr_decode());
2805
2806         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2807             preferredMessageSize;
2808         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2809         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2810             maximumRecordSize;
2811         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2812
2813         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2814             yaz_get_charneg_record(*oi);
2815
2816         // Save proposal charsets and langs.
2817         if (ODR_MASK_GET(apdu->u.initRequest->options,
2818                          Z_Options_negotiationModel)
2819             && charSetandLangRecord)
2820         {
2821
2822             yaz_get_proposal_charneg(m_referenceId_mem,
2823                                      charSetandLangRecord,
2824                                      &m_initRequest_oi_negotiation_charsets,
2825                                      &m_initRequest_oi_negotiation_num_charsets,
2826                                      &m_initRequest_oi_negotiation_langs,
2827                                      &m_initRequest_oi_negotiation_num_langs,
2828                                      &m_initRequest_oi_negotiation_selected);
2829
2830             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2831             {
2832                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2833                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2834                         m_initRequest_oi_negotiation_charsets[i]:"none");
2835             }
2836             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2837             {
2838                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2839                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2840                         m_initRequest_oi_negotiation_langs[i]:"none");
2841             }
2842             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2843                     m_session_str, m_initRequest_oi_negotiation_selected);
2844         }
2845         // save init options for the response..
2846         m_initRequest_options = apdu->u.initRequest->options;
2847
2848         apdu->u.initRequest->options =
2849             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2850                                        sizeof(Odr_bitmask));
2851         ODR_MASK_ZERO(apdu->u.initRequest->options);
2852         int i;
2853         for (i = 0; i<= 24; i++)
2854             ODR_MASK_SET(apdu->u.initRequest->options, i);
2855         // check negotiation option
2856         if (!ODR_MASK_GET(m_initRequest_options,
2857                           Z_Options_negotiationModel))
2858         {
2859             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2860                            Z_Options_negotiationModel);
2861         }
2862         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2863                        Z_Options_concurrentOperations);
2864         // make new version
2865         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2866         apdu->u.initRequest->protocolVersion =
2867             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2868                                        sizeof(Odr_bitmask));
2869         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2870
2871         for (i = 0; i<= 8; i++)
2872             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2873     }
2874     if (m_client->m_init_flag)
2875     {
2876         if (handle_init_response_for_invalid_session(apdu))
2877             return;
2878         if (m_client->m_initResponse)
2879         {
2880             Z_APDU *apdu2 = m_client->m_initResponse;
2881             apdu2->u.initResponse->otherInfo = 0;
2882             if (m_client->m_cookie && *m_client->m_cookie)
2883                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2884                                            m_client->m_cookie);
2885             apdu2->u.initResponse->referenceId =
2886                 apdu->u.initRequest->referenceId;
2887             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2888             apdu2->u.initResponse->protocolVersion =
2889                 m_client->m_initResponse_version;
2890
2891             send_to_client(apdu2);
2892             m_timeout_mode = timeout_normal;
2893             return;
2894         }
2895     }
2896     m_client->m_init_flag = 1;
2897
2898 #if USE_AUTH_MSG
2899     yaz_log(YLOG_LOG, "%suse_auth_msg", m_session_str);
2900     Auth_Msg *m = new Auth_Msg;
2901     m->m_proxy = this;
2902     z_APDU(odr_encode(), &apdu, 0, "encode");
2903     char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
2904     m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
2905     memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
2906     odr_reset(odr_encode());
2907
2908     inc_ref();
2909     m_my_thread->put(m);
2910 #else
2911     int ret = handle_authentication(apdu);
2912     result_authentication(apdu, ret);
2913 #endif
2914 }
2915
2916 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2917 {
2918     Z_ReferenceId **refid = get_referenceIdP(apdu);
2919     nmem_reset(m_referenceId_mem);
2920     if (refid && *refid)
2921     {
2922         m_referenceId = (Z_ReferenceId *)
2923             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2924         m_referenceId->len = m_referenceId->size = (*refid)->len;
2925         m_referenceId->buf = (unsigned char *)
2926             nmem_malloc(m_referenceId_mem, (*refid)->len);
2927         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2928     }
2929     else
2930         m_referenceId = 0;
2931
2932     if (!m_client && m_flag_invalid_session)
2933     {
2934         // Got request for a session that is invalid..
2935         m_apdu_invalid_session = apdu; // save package
2936         m_mem_invalid_session = odr_extract_mem(odr_decode());
2937         apdu = m_initRequest_apdu;     // but throw an init to the target
2938     }
2939
2940     // Determine our client.
2941     Z_OtherInformation **oi;
2942     get_otherInfoAPDU(apdu, &oi);
2943     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2944     if (!m_client)
2945     {
2946         if (m_http_version)
2947         {   // HTTP. Send not found
2948             send_http_response(404);
2949             return;
2950         }
2951         else
2952         {
2953             // Z39.50 just shutdown
2954             delete this;
2955             return;
2956         }
2957     }
2958
2959     m_client->m_server = this;
2960
2961     if (apdu->which == Z_APDU_initRequest)
2962         handle_init(apdu);
2963     else
2964         handle_incoming_Z_PDU_2(apdu);
2965 }
2966
2967 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
2968 {
2969     handle_max_record_retrieve(apdu);
2970
2971     if (apdu)
2972         apdu = handle_syntax_validation(apdu);
2973
2974     if (apdu)
2975         apdu = handle_query_transformation(apdu);
2976
2977     if (apdu)
2978         apdu = handle_target_charset_conversion(apdu);
2979
2980     if (apdu)
2981         apdu = handle_query_validation(apdu);
2982
2983     if (apdu)
2984         apdu = result_set_optimize(apdu);
2985
2986     if (!apdu)
2987     {
2988         m_client->timeout(m_target_idletime);  // mark it active even
2989         recv_GDU_more(true);
2990         // though we didn't use it
2991         return;
2992     }
2993     // Add otherInformation entry in APDU if
2994     // negotiation is in use.
2995     if (apdu)
2996         handle_charset_lang_negotiation(apdu);
2997
2998     // delete other info construct completely if 0 elements
2999     Z_OtherInformation **oi;
3000     get_otherInfoAPDU(apdu, &oi);
3001     if (oi && *oi && (*oi)->num_elements == 0)
3002         *oi = 0;
3003
3004     if (apdu->which == Z_APDU_presentRequest &&
3005         m_client->m_resultSetStartPoint == 0)
3006     {
3007         Z_PresentRequest *pr = apdu->u.presentRequest;
3008         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3009         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3010     } else {
3011         m_client->m_resultSetStartPoint = 0;
3012     }
3013     if (m_client->send_to_target(apdu) < 0)
3014     {
3015         m_client->shutdown();
3016     }
3017     else
3018         m_client->m_waiting = 1;
3019 }
3020
3021 void Yaz_Proxy::connectNotify()
3022 {
3023 }
3024
3025 void Yaz_Proxy::releaseClient()
3026 {
3027     xfree(m_proxyTarget);
3028     m_proxyTarget = 0;
3029     m_flag_invalid_session = 0;
3030     // only keep if keep_alive flag is set...
3031     if (m_client &&
3032         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3033         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3034         m_client->m_waiting == 0)
3035     {
3036         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3037                  m_session_str,
3038                  m_client->get_hostname());
3039         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3040                 m_session_str, m_client->m_pdu_recv,
3041                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3042                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3043         assert (m_client->m_waiting != 2);
3044         // Tell client (if any) that no server connection is there..
3045         m_client->m_server = 0;
3046         m_client = 0;
3047     }
3048     else if (m_client)
3049     {
3050         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
3051                  m_session_str,
3052                  m_client->get_hostname());
3053         assert (m_client->m_waiting != 2);
3054         delete m_client;
3055         m_client = 0;
3056     }
3057     else if (!m_parent)
3058     {
3059         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
3060                  m_session_str);
3061         assert (m_parent);
3062     }
3063     else
3064     {
3065         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
3066                  m_session_str);
3067     }
3068     if (m_parent)
3069         m_parent->pre_init();
3070 }
3071
3072 bool Yaz_Proxy::dec_ref(bool main_ptr)
3073 {
3074     yaz_log(YLOG_LOG, "%sdec_ref count=%d", m_session_str, m_ref_count);
3075
3076     assert(m_ref_count > 0);
3077     if (main_ptr)
3078     {
3079         if (m_main_ptr_dec)
3080             return false;
3081         m_main_ptr_dec = true;
3082     }
3083
3084     m_http_keepalive = 0;
3085
3086     --m_ref_count;
3087     if (m_ref_count > 0)
3088         return false;
3089
3090     releaseClient();
3091
3092     delete this;
3093     return true;
3094 }
3095
3096 const char *Yaz_ProxyClient::get_session_str()
3097 {
3098     if (!m_server)
3099         return "0 ";
3100     return m_server->get_session_str();
3101 }
3102
3103 void Yaz_ProxyClient::shutdown()
3104 {
3105     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3106              get_hostname());
3107
3108     if (m_server)
3109     {
3110         m_waiting = 1;      // ensure it's released from Proxy in releaseClient
3111         m_server->dec_ref(true);
3112     }
3113     else
3114         delete this;
3115 }
3116
3117 void Yaz_Proxy::failNotify()
3118 {
3119     inc_request_no();
3120     yaz_log (YLOG_LOG, "%sConnection closed by client",
3121              get_session_str());
3122     dec_ref(true);
3123 }
3124
3125 void Yaz_Proxy::send_response_fail_client(const char *addr)
3126 {
3127     yaz_log(YLOG_LOG, "%ssend_close_response", get_session_str());
3128     if (m_http_version)
3129     {
3130         Z_SRW_diagnostic *diagnostic = 0;
3131         int num_diagnostic = 0;
3132
3133         yaz_add_srw_diagnostic(odr_encode(),
3134                                &diagnostic, &num_diagnostic,
3135                                YAZ_SRW_SYSTEM_TEMPORARILY_UNAVAILABLE, addr);
3136         if (m_s2z_search_apdu)
3137             send_srw_search_response(diagnostic, num_diagnostic);
3138         else
3139             send_srw_explain_response(diagnostic, num_diagnostic);
3140     }
3141 }
3142 void Yaz_ProxyClient::failNotify()
3143 {
3144     if (m_server)
3145         m_server->inc_request_no();
3146     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
3147              get_session_str(), get_hostname());
3148
3149     if (m_server)
3150         m_server->send_response_fail_client(get_hostname());
3151     shutdown();
3152 }
3153
3154 void Yaz_ProxyClient::connectNotify()
3155 {
3156     const char *s = get_session_str();
3157     const char *h = get_hostname();
3158     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3159              m_target_idletime);
3160     timeout(m_target_idletime);
3161     if (!m_server)
3162         pre_init_client();
3163 }
3164
3165 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3166                                               *the_PDU_Observable, int fd)
3167 {
3168     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3169 }
3170
3171 Yaz_ProxyClient::~Yaz_ProxyClient()
3172 {
3173     if (m_prev)
3174         *m_prev = m_next;
3175     if (m_next)
3176         m_next->m_prev = m_prev;
3177     m_waiting = 2;     // for debugging purposes only.
3178     odr_destroy(m_init_odr);
3179     delete m_last_query;
3180     xfree (m_last_resultSetId);
3181     xfree (m_cookie);
3182 }
3183
3184 void Yaz_ProxyClient::pre_init_client()
3185 {
3186     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3187     Z_InitRequest *req = apdu->u.initRequest;
3188
3189     int i;
3190     for (i = 0; i<= 24; i++)
3191         ODR_MASK_SET(req->options, i);
3192     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3193                    Z_Options_negotiationModel);
3194     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3195                    Z_Options_concurrentOperations);
3196     for (i = 0; i<= 10; i++)
3197         ODR_MASK_SET(req->protocolVersion, i);
3198
3199     if (send_to_target(apdu) < 0)
3200     {
3201         delete this;
3202     }
3203     else
3204     {
3205         m_waiting = 1;
3206         m_init_flag = 1;
3207     }
3208 }
3209
3210 void Yaz_Proxy::pre_init()
3211 {
3212     int i;
3213     const char *name = 0;
3214     const char *zurl_in_use[MAX_ZURL_PLEX];
3215     int limit_bw, limit_pdu, limit_req, limit_search, limit_connect;
3216     int target_idletime, client_idletime;
3217     int max_clients;
3218     int keepalive_limit_bw, keepalive_limit_pdu;
3219     int pre_init;
3220     const char *cql2rpn = 0;
3221     const char *authentication = 0;
3222     const char *negotiation_charset = 0;
3223     const char *negotiation_lang = 0;
3224
3225     Yaz_ProxyConfig *cfg = check_reconfigure();
3226
3227     zurl_in_use[0] = 0;
3228
3229     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3230         set_APDU_yazlog(1);
3231     else
3232         set_APDU_yazlog(0);
3233
3234     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3235                                           &limit_bw, &limit_pdu, &limit_req,
3236                                           &limit_search, &limit_connect,
3237                                           &target_idletime, &client_idletime,
3238                                           &max_clients,
3239                                           &keepalive_limit_bw,
3240                                           &keepalive_limit_pdu,
3241                                           &pre_init,
3242                                           &cql2rpn,
3243                                           &authentication,
3244                                           &negotiation_charset,
3245                                           &negotiation_lang,
3246                                           0) ; i++)
3247     {
3248         if (pre_init)
3249         {
3250             int j;
3251             for (j = 0; zurl_in_use[j]; j++)
3252             {
3253                 Yaz_ProxyClient *c;
3254                 int spare = 0;
3255                 int spare_waiting = 0;
3256                 int in_use = 0;
3257                 int other = 0;
3258                 for (c = m_clientPool; c; c = c->m_next)
3259                 {
3260                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3261                     {
3262                         if (c->m_cookie == 0)
3263                         {
3264                             if (c->m_server == 0)
3265                                 if (c->m_waiting)
3266                                     spare_waiting++;
3267                                 else
3268                                     spare++;
3269                             else
3270                                 in_use++;
3271                         }
3272                         else
3273                             other++;
3274                     }
3275                 }
3276                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3277                         "sparew=%d preinit=%d",m_session_str,
3278                         name, zurl_in_use[j], in_use, other,
3279                         spare, spare_waiting, pre_init);
3280                 if (spare + spare_waiting < pre_init)
3281                 {
3282                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3283                     c->m_next = m_clientPool;
3284                     if (c->m_next)
3285                         c->m_next->m_prev = &c->m_next;
3286                     m_clientPool = c;
3287                     c->m_prev = &m_clientPool;
3288
3289                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3290                         c->set_APDU_yazlog(1);
3291                     else
3292                         c->set_APDU_yazlog(0);
3293
3294                     if (c->client(zurl_in_use[j]))
3295                     {
3296                         timeout(60);
3297                         delete c;
3298                         return;
3299                     }
3300                     c->timeout(30);
3301                     c->m_waiting = 1;
3302                     c->m_target_idletime = target_idletime;
3303                     c->m_seqno = m_seqno++;
3304                 }
3305             }
3306         }
3307     }
3308 }
3309
3310 void Yaz_Proxy::timeoutNotify()
3311 {
3312     if (m_parent)
3313     {
3314         GDU *gdu;
3315         switch(m_timeout_mode)
3316         {
3317         case timeout_normal:
3318         case timeout_busy:
3319             inc_request_no();
3320             m_in_queue.clear();
3321             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3322             dec_ref(true);
3323             break;
3324         case timeout_reduce:
3325             timeout(m_client_idletime);
3326             m_timeout_mode = timeout_busy;
3327             gdu = m_timeout_gdu;
3328             m_timeout_gdu = 0;
3329             recv_GDU_normal(gdu);
3330             break;
3331         case timeout_xsl:
3332             assert(m_stylesheet_nprl);
3333             convert_xsl_delay();
3334             recv_GDU_more(true);
3335         }
3336     }
3337     else
3338     {
3339         timeout(600);
3340         pre_init();
3341     }
3342 }
3343
3344 void Yaz_Proxy::markInvalid()
3345 {
3346     m_client = 0;
3347     m_flag_invalid_session = 1;
3348 }
3349
3350 void Yaz_ProxyClient::timeoutNotify()
3351 {
3352     if (m_server)
3353         m_server->inc_request_no();
3354
3355     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3356              get_hostname());
3357
3358     if (m_server)
3359         m_server->send_response_fail_client(get_hostname());
3360     shutdown();
3361
3362     m_root->pre_init();
3363 }
3364
3365 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
3366                                  Yaz_Proxy *parent) :
3367     Z_Assoc (the_PDU_Observable)
3368 {
3369     m_cookie = 0;
3370     m_next = 0;
3371     m_prev = 0;
3372     m_init_flag = 0;
3373     m_last_query = 0;
3374     m_last_resultSetId = 0;
3375     m_last_resultCount = 0;
3376     m_last_ok = 0;
3377     m_sr_transform = 0;
3378     m_waiting = 0;
3379     m_init_odr = odr_createmem (ODR_DECODE);
3380     m_initResponse = 0;
3381     m_initResponse_options = 0;
3382     m_initResponse_version = 0;
3383     m_initResponse_preferredMessageSize = 0;
3384     m_initResponse_maximumRecordSize = 0;
3385     m_resultSetStartPoint = 0;
3386     m_bytes_sent = m_bytes_recv = 0;
3387     m_pdu_recv = 0;
3388     m_server = 0;
3389     m_seqno = 0;
3390     m_target_idletime = 600;
3391     m_root = parent;
3392 }
3393
3394 const char *Yaz_Proxy::option(const char *name, const char *value)
3395 {
3396     if (!strcmp (name, "optimize")) {
3397         if (value) {
3398             xfree (m_optimize);
3399             m_optimize = xstrdup (value);
3400         }
3401         return m_optimize;
3402     }
3403     return 0;
3404 }
3405
3406 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
3407 {
3408
3409 }
3410
3411 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
3412 {
3413     if (apdu->which == Z_GDU_Z3950)
3414         recv_Z_PDU(apdu->u.z3950, len);
3415     else if (apdu->which == Z_GDU_HTTP_Response)
3416         recv_HTTP_response(apdu->u.HTTP_Response, len);
3417     else
3418         shutdown();
3419 }
3420
3421 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
3422 {
3423     if (!m_flag_invalid_session)
3424         return 0;
3425     m_flag_invalid_session = 0;
3426     handle_incoming_Z_PDU(m_apdu_invalid_session);
3427     assert (m_mem_invalid_session);
3428     nmem_destroy(m_mem_invalid_session);
3429     m_mem_invalid_session = 0;
3430     return 1;
3431 }
3432
3433 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
3434 {
3435     m_bytes_recv += len;
3436
3437     m_pdu_recv++;
3438     m_waiting = 0;
3439     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
3440         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
3441                  apdu_name(apdu), get_hostname(), len);
3442     if (apdu->which == Z_APDU_initResponse)
3443     {
3444         if (!m_server)  // if this is a pre init session , check for more
3445             m_root->pre_init();
3446         NMEM nmem = odr_extract_mem (odr_decode());
3447         odr_reset (m_init_odr);
3448         nmem_transfer (m_init_odr->mem, nmem);
3449         m_initResponse = apdu;
3450         m_initResponse_options = apdu->u.initResponse->options;
3451         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3452         m_initResponse_preferredMessageSize =
3453             *apdu->u.initResponse->preferredMessageSize;
3454         m_initResponse_maximumRecordSize =
3455             *apdu->u.initResponse->maximumRecordSize;
3456
3457         Z_InitResponse *ir = apdu->u.initResponse;
3458         char *im0 = ir->implementationName;
3459
3460         char *im1 = (char*)
3461             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3462         *im1 = '\0';
3463         if (im0)
3464         {
3465             strcat(im1, im0);
3466             strcat(im1, " ");
3467         }
3468         strcat(im1, "(YAZ Proxy)");
3469         ir->implementationName = im1;
3470
3471         nmem_destroy (nmem);
3472
3473         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3474             return;
3475     }
3476     if (apdu->which == Z_APDU_searchResponse)
3477     {
3478         Z_SearchResponse *sr = apdu->u.searchResponse;
3479         m_last_resultCount = *sr->resultCount;
3480         int status = *sr->searchStatus;
3481         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3482         {
3483             m_last_ok = 1;
3484
3485             if (sr->records && sr->records->which == Z_Records_DBOSD)
3486             {
3487                 m_cache.add(odr_decode(),
3488                             sr->records->u.databaseOrSurDiagnostics, 1,
3489                             *sr->resultCount);
3490             }
3491         }
3492     }
3493     if (apdu->which == Z_APDU_presentResponse)
3494     {
3495         Z_PresentResponse *pr = apdu->u.presentResponse;
3496         if (m_sr_transform)
3497         {
3498             m_sr_transform = 0;
3499             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3500             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3501             sr->referenceId = pr->referenceId;
3502             *sr->resultCount = m_last_resultCount;
3503             sr->records = pr->records;
3504             sr->nextResultSetPosition = pr->nextResultSetPosition;
3505             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3506             apdu = new_apdu;
3507         }
3508         if (pr->records &&
3509             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3510         {
3511             m_cache.add(odr_decode(),
3512                         pr->records->u.databaseOrSurDiagnostics,
3513                         m_resultSetStartPoint, -1);
3514             m_resultSetStartPoint = 0;
3515         }
3516     }
3517     if (m_cookie)
3518         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3519
3520     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
3521
3522     if (server)
3523         server->send_to_client(apdu);
3524     if (apdu->which == Z_APDU_close)
3525         shutdown();
3526     else if (server)
3527         server->recv_GDU_more(true);
3528 }
3529
3530 void Yaz_Proxy::low_socket_close()
3531 {
3532 #if WIN32
3533 #else
3534     int i;
3535     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3536         if  (m_lo_fd[i] >= 0)
3537             ::close(m_lo_fd[i]);
3538 #endif
3539 }
3540
3541 void Yaz_Proxy::low_socket_open()
3542 {
3543 #if WIN32
3544 #else
3545     int i;
3546     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3547         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3548 #endif
3549 }
3550
3551 int Yaz_Proxy::server(const char *addr)
3552 {
3553     int r = Z_Assoc::server(addr);
3554     if (!r)
3555     {
3556         yaz_log(YLOG_LOG, "%sStarted proxy "
3557 #ifdef VERSION
3558             VERSION
3559 #endif
3560             " on %s", m_session_str, addr);
3561         timeout(1);
3562     }
3563     return r;
3564 }
3565
3566 /*
3567  * Local variables:
3568  * c-basic-offset: 4
3569  * indent-tabs-mode: nil
3570  * End:
3571  * vim: shiftwidth=4 tabstop=8 expandtab
3572  */
3573