src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.37 2005-09-26 09:25:06 adam Exp $
   2    Copyright (c) 1998-2005, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <stdlib.h>
  42 #include <time.h>
  43 #include <fcntl.h>
  44
  45 #include <yaz/srw.h>
  46 #include <yaz/marcdisp.h>
  47 #include <yaz/yaz-iconv.h>
  48 #include <yaz/log.h>
  49 #include <yaz/diagbib1.h>
  50 #include "proxyp.h"
  51 #include <yaz/pquery.h>
  52 #include <yaz/otherinfo.h>
  53 #include <yaz/charneg.h>
  54 #include "msg-thread.h"
  55
  56 using namespace yazpp_1;
  57
  58 #define USE_AUTH_MSG 1
  59
  60 #if USE_AUTH_MSG
  61 class Auth_Msg : public IMsg_Thread {
  62 public:
  63     int m_ret;
  64     IMsg_Thread *handle();
  65     void result();
  66     Yaz_Proxy *m_proxy;
  67     NMEM m_nmem;
  68     char *m_apdu_buf;
  69     int m_apdu_len;
  70     Auth_Msg();
  71     virtual ~Auth_Msg();
  72 };
  73
  74 Auth_Msg::Auth_Msg()
  75 {
  76     m_nmem = nmem_create();
  77 }
  78
  79 Auth_Msg::~Auth_Msg()
  80 {
  81     nmem_destroy(m_nmem);
  82 }
  83
  84 IMsg_Thread *Auth_Msg::handle()
  85 {
  86     yaz_log(YLOG_LOG, "Auth_Msg:handle begin");
  87     ODR decode = odr_createmem(ODR_DECODE);
  88     Z_APDU *apdu;
  89
  90     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
  91     int r = z_APDU(decode, &apdu, 0, 0);
  92     if (!r)
  93     {
  94         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
  95     }
  96     else
  97     {
  98         m_ret = m_proxy->handle_authentication(apdu);
  99     }
 100     yaz_log(YLOG_LOG, "Auth_Msg:handle end");
 101     odr_destroy(decode);
 102     return this;
 103 }
 104
 105 void Auth_Msg::result()
 106 {
 107     yaz_log(YLOG_LOG, "Auth_Msg:result proxy ok buf=%p len=%d",
 108             m_apdu_buf, m_apdu_len);
 109     odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
 110     Z_APDU *apdu = 0;
 111     int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
 112     if (r)
 113         yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU OK");
 114     else
 115         yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
 116     if (m_proxy->dec_ref())
 117         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
 118     else
 119     {
 120         yaz_log(YLOG_LOG, "Auth_Msg::proxy still alive");
 121         m_proxy->result_authentication(apdu, m_ret);
 122     }
 123     delete this;
 124 }
 125
 126 #endif
 127
 128 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
 129 {
 130     if (apdu == 0 || ret == 0)
 131     {
 132         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
 133         *apdu_reject->u.initResponse->result = 0;
 134         send_to_client(apdu_reject);
 135         dec_ref();
 136     }
 137     else
 138         handle_incoming_Z_PDU_2(apdu);
 139 }
 140
 141 static const char *apdu_name(Z_APDU *apdu)
 142 {
 143     switch (apdu->which)
 144     {
 145     case Z_APDU_initRequest:
 146         return "initRequest";
 147     case Z_APDU_initResponse:
 148         return "initResponse";
 149     case Z_APDU_searchRequest:
 150         return "searchRequest";
 151     case Z_APDU_searchResponse:
 152         return "searchResponse";
 153     case Z_APDU_presentRequest:
 154         return "presentRequest";
 155     case Z_APDU_presentResponse:
 156         return "presentResponse";
 157     case Z_APDU_deleteResultSetRequest:
 158         return "deleteResultSetRequest";
 159     case Z_APDU_deleteResultSetResponse:
 160         return "deleteResultSetResponse";
 161     case Z_APDU_scanRequest:
 162         return "scanRequest";
 163     case Z_APDU_scanResponse:
 164         return "scanResponse";
 165     case Z_APDU_sortRequest:
 166         return "sortRequest";
 167     case Z_APDU_sortResponse:
 168         return "sortResponse";
 169     case Z_APDU_extendedServicesRequest:
 170         return "extendedServicesRequest";
 171     case Z_APDU_extendedServicesResponse:
 172         return "extendedServicesResponse";
 173     case Z_APDU_close:
 174         return "close";
 175     }
 176     return "other";
 177 }
 178
 179 static const char *gdu_name(Z_GDU *gdu)
 180 {
 181     switch(gdu->which)
 182     {
 183     case Z_GDU_Z3950:
 184         return apdu_name(gdu->u.z3950);
 185     case Z_GDU_HTTP_Request:
 186         return "HTTP Request";
 187     case Z_GDU_HTTP_Response:
 188         return "HTTP Response";
 189     }
 190     return "Unknown request/response";
 191 }
 192
 193 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
 194                      ISocketObservable *the_socket_observable,
 195                      Yaz_Proxy *parent)
 196     :
 197     Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 198 {
 199     m_PDU_Observable = the_PDU_Observable;
 200     m_socket_observable = the_socket_observable;
 201     m_client = 0;
 202     m_parent = parent;
 203     m_clientPool = 0;
 204     m_seqno = 1;
 205     m_keepalive_limit_bw = 500000;
 206     m_keepalive_limit_pdu = 1000;
 207     m_proxyTarget = 0;
 208     m_default_target = 0;
 209     m_proxy_negotiation_charset = 0;
 210     m_proxy_negotiation_lang = 0;
 211     m_charset_converter = new Yaz_CharsetConverter;
 212     m_max_clients = 150;
 213     m_log_mask = 0;
 214     m_seed = time(0);
 215     m_client_idletime = 600;
 216     m_target_idletime = 600;
 217     m_optimize = xstrdup ("1");
 218     strcpy(m_session_str, "0 ");
 219     m_session_no = 0;
 220     m_bytes_sent = 0;
 221     m_bytes_recv = 0;
 222     m_bw_max = 0;
 223     m_pdu_max = 0;
 224     m_search_max = 0;
 225     m_connect_max = 0;
 226     m_timeout_mode = timeout_normal;
 227     m_timeout_gdu = 0;
 228     m_max_record_retrieve = 0;
 229     m_reconfig_flag = 0;
 230     m_config_fname = 0;
 231     m_request_no = 0;
 232     m_flag_invalid_session = 0;
 233     m_referenceId = 0;
 234     m_referenceId_mem = nmem_create();
 235     m_config = 0;
 236     m_marcxml_mode = none;
 237     m_stylesheet_xsp = 0;
 238     m_stylesheet_nprl = 0;
 239     m_stylesheet_apdu = 0;
 240     m_s2z_stylesheet = 0;
 241     m_s2z_database = 0;
 242     m_schema = 0;
 243     m_backend_type = 0;
 244     m_backend_charset = 0;
 245     m_frontend_type = 0;
 246     m_initRequest_apdu = 0;
 247     m_initRequest_mem = 0;
 248     m_initRequest_preferredMessageSize = 0;
 249     m_initRequest_maximumRecordSize = 0;
 250     m_initRequest_options = 0;
 251     m_initRequest_version = 0;
 252     m_initRequest_oi_negotiation_charsets = 0;
 253     m_initRequest_oi_negotiation_num_charsets = 0;
 254     m_initRequest_oi_negotiation_langs = 0;
 255     m_initRequest_oi_negotiation_num_langs = 0;
 256     m_initRequest_oi_negotiation_selected = 0;
 257     m_apdu_invalid_session = 0;
 258     m_mem_invalid_session = 0;
 259     m_s2z_odr_init = 0;
 260     m_s2z_odr_search = 0;
 261     m_s2z_init_apdu = 0;
 262     m_s2z_search_apdu = 0;
 263     m_s2z_present_apdu = 0;
 264     m_http_keepalive = 0;
 265     m_http_version = 0;
 266     m_soap_ns = 0;
 267     m_s2z_packing = Z_SRW_recordPacking_string;
 268 #if HAVE_GETTIMEOFDAY
 269     m_time_tv = xmalloc(sizeof(struct timeval));
 270     struct timeval *tv = (struct timeval *) m_time_tv;
 271     tv->tv_sec = 0;
 272     tv->tv_usec = 0;
 273 #else
 274     m_time_tv = 0;
 275 #endif
 276     m_usemarcon_ini_stage1 = 0;
 277     m_usemarcon_ini_stage2 = 0;
 278     m_usemarcon = new Yaz_usemarcon();
 279     if (!m_parent)
 280         low_socket_open();
 281     m_my_thread = 0;
 282     m_ref_count = 1;
 283     m_peername = 0;
 284 }
 285
 286 void Yaz_Proxy::inc_ref()
 287 {
 288     m_ref_count++;
 289 }
 290
 291 Yaz_Proxy::~Yaz_Proxy()
 292 {
 293     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 294             m_bytes_sent, m_bytes_recv);
 295     nmem_destroy(m_initRequest_mem);
 296     nmem_destroy(m_mem_invalid_session);
 297     nmem_destroy(m_referenceId_mem);
 298
 299     xfree(m_proxyTarget);
 300     xfree(m_default_target);
 301     xfree(m_proxy_negotiation_charset);
 302     xfree(m_proxy_negotiation_lang);
 303     delete m_charset_converter;
 304     xfree(m_optimize);
 305
 306 #if HAVE_XSLT
 307     if (m_stylesheet_xsp)
 308         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 309 #endif
 310     xfree (m_time_tv);
 311
 312     xfree (m_peername);
 313     xfree (m_schema);
 314     xfree (m_backend_type);
 315     xfree (m_backend_charset);
 316     xfree (m_usemarcon_ini_stage1);
 317     xfree (m_usemarcon_ini_stage2);
 318     delete m_usemarcon;
 319     if (m_s2z_odr_init)
 320         odr_destroy(m_s2z_odr_init);
 321     if (m_s2z_odr_search)
 322         odr_destroy(m_s2z_odr_search);
 323     if (!m_parent)
 324         low_socket_close();
 325     if (!m_parent)
 326         delete m_my_thread;
 327     delete m_config;
 328 }
 329
 330 void Yaz_Proxy::set_debug_mode(int mode)
 331 {
 332     m_debug_mode = mode;
 333 }
 334
 335 int Yaz_Proxy::set_config(const char *config)
 336 {
 337     delete m_config;
 338     m_config = new Yaz_ProxyConfig();
 339     xfree(m_config_fname);
 340     m_config_fname = xstrdup(config);
 341     int r = m_config->read_xml(config);
 342     if (!r)
 343         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 344     return r;
 345 }
 346
 347 void Yaz_Proxy::set_default_target(const char *target)
 348 {
 349     xfree (m_default_target);
 350     m_default_target = 0;
 351     if (target)
 352         m_default_target = (char *) xstrdup (target);
 353 }
 354
 355 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang)
 356 {
 357     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
 358         "language to '%s'", m_session_str, charset?charset:"none",
 359         lang?lang:"none");
 360     xfree (m_proxy_negotiation_charset);
 361     xfree (m_proxy_negotiation_lang);
 362     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 363     if (charset)
 364         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 365     if (lang)
 366         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 367 }
 368
 369 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 370 {
 371     if (m_parent)
 372         return m_parent->check_reconfigure();
 373
 374     Yaz_ProxyConfig *cfg = m_config;
 375     if (m_reconfig_flag)
 376     {
 377         yaz_log(YLOG_LOG, "reconfigure");
 378         yaz_log_reopen();
 379         if (m_config_fname && cfg)
 380         {
 381             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 382             int r = cfg->read_xml(m_config_fname);
 383             if (r)
 384                 yaz_log(YLOG_WARN, "reconfigure failed");
 385             else
 386             {
 387                 m_log_mask = 0;
 388                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 389             }
 390         }
 391         else
 392             yaz_log(YLOG_LOG, "reconfigure");
 393         m_reconfig_flag = 0;
 394     }
 395     return cfg;
 396 }
 397
 398 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 399                                         *the_PDU_Observable, int fd)
 400 {
 401     check_reconfigure();
 402     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 403                                          m_socket_observable, this);
 404     new_proxy->m_config = 0;
 405     new_proxy->m_config_fname = 0;
 406     new_proxy->timeout(m_client_idletime);
 407     new_proxy->m_target_idletime = m_target_idletime;
 408     new_proxy->set_default_target(m_default_target);
 409     new_proxy->m_max_clients = m_max_clients;
 410     new_proxy->m_log_mask = m_log_mask;
 411     new_proxy->set_APDU_log(get_APDU_log());
 412     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 413         new_proxy->set_APDU_yazlog(1);
 414     else
 415         new_proxy->set_APDU_yazlog(0);
 416     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 417     m_session_no++;
 418     new_proxy->m_peername = xstrdup(the_PDU_Observable->getpeername());
 419     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 420              new_proxy->m_peername);
 421     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 422         m_proxy_negotiation_lang);
 423     // create thread object the first time we get an incoming connection
 424     if (!m_my_thread)
 425         m_my_thread = new Msg_Thread(m_socket_observable, 1);
 426     new_proxy->m_my_thread = m_my_thread;
 427     return new_proxy;
 428 }
 429
 430 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 431 {
 432     int oid[OID_SIZE];
 433     Z_OtherInformationUnit *oi;
 434     struct oident ent;
 435     ent.proto = PROTO_Z3950;
 436     ent.oclass = CLASS_USERINFO;
 437     ent.value = (oid_value) VAL_COOKIE;
 438     assert (oid_ent_to_oid (&ent, oid));
 439
 440     if (oid_ent_to_oid (&ent, oid) &&
 441         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 442         oi->which == Z_OtherInfo_characterInfo)
 443         return oi->information.characterInfo;
 444     return 0;
 445 }
 446 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 447 {
 448     int oid[OID_SIZE];
 449     Z_OtherInformationUnit *oi;
 450     struct oident ent;
 451     ent.proto = PROTO_Z3950;
 452     ent.oclass = CLASS_USERINFO;
 453     ent.value = (oid_value) VAL_PROXY;
 454     if (oid_ent_to_oid (&ent, oid) &&
 455         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 456         oi->which == Z_OtherInfo_characterInfo)
 457         return oi->information.characterInfo;
 458     return 0;
 459 }
 460 const char *Yaz_Proxy::load_balance(const char **url)
 461 {
 462     int zurl_in_use[MAX_ZURL_PLEX];
 463     int zurl_in_spare[MAX_ZURL_PLEX];
 464     Yaz_ProxyClient *c;
 465     int i;
 466
 467     for (i = 0; i<MAX_ZURL_PLEX; i++)
 468     {
 469         zurl_in_use[i] = 0;
 470         zurl_in_spare[i] = 0;
 471     }
 472     for (c = m_parent->m_clientPool; c; c = c->m_next)
 473     {
 474         for (i = 0; url[i]; i++)
 475             if (!strcmp(url[i], c->get_hostname()))
 476             {
 477                 zurl_in_use[i]++;
 478                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 479                     zurl_in_spare[i]++;
 480             }
 481     }
 482     int min_use = 100000;
 483     int spare_for_min = 0;
 484     int max_spare = 0;
 485     const char *ret_min = 0;
 486     const char *ret_spare = 0;
 487     for (i = 0; url[i]; i++)
 488     {
 489         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 490                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 491         if (min_use > zurl_in_use[i])
 492         {
 493             ret_min = url[i];
 494             min_use = zurl_in_use[i];
 495             spare_for_min = zurl_in_spare[i];
 496         }
 497         if (max_spare < zurl_in_spare[i])
 498         {
 499             ret_spare = url[i];
 500             max_spare = zurl_in_spare[i];
 501         }
 502     }
 503     return ret_min;
 504 }
 505
 506 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 507                                        const char *proxy_host)
 508 {
 509     assert (m_parent);
 510     Yaz_Proxy *parent = m_parent;
 511     Yaz_ProxyClient *c = m_client;
 512
 513     if (!m_proxyTarget)
 514     {
 515         const char *url[MAX_ZURL_PLEX];
 516         Yaz_ProxyConfig *cfg = check_reconfigure();
 517         if (proxy_host)
 518         {
 519             if (parent && parent->m_debug_mode)
 520             {
 521                 // only to be enabled for debugging...
 522                 if (!strcmp(proxy_host, "stop"))
 523                     exit(0);
 524             }
 525             xfree(m_default_target);
 526             m_default_target = xstrdup(proxy_host);
 527         }
 528         proxy_host = m_default_target;
 529         int client_idletime = -1;
 530         const char *cql2rpn_fname = 0;
 531         const char *negotiation_charset = 0;
 532         const char *negotiation_lang = 0;
 533         const char *query_charset = 0;
 534         url[0] = m_default_target;
 535         url[1] = 0;
 536         if (cfg)
 537         {
 538             int pre_init = 0;
 539             cfg->get_target_info(proxy_host, url, &m_bw_max,
 540                                  &m_pdu_max, &m_max_record_retrieve,
 541                                  &m_search_max, &m_connect_max,
 542                                  &m_target_idletime, &client_idletime,
 543                                  &parent->m_max_clients,
 544                                  &m_keepalive_limit_bw,
 545                                  &m_keepalive_limit_pdu,
 546                                  &pre_init,
 547                                  &cql2rpn_fname,
 548                                  &negotiation_charset,
 549                                  &negotiation_lang,
 550                                  &query_charset);
 551         }
 552         if (client_idletime != -1)
 553         {
 554             m_client_idletime = client_idletime;
 555             timeout(m_client_idletime);
 556         }
 557         if (cql2rpn_fname)
 558             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 559         if (negotiation_charset || negotiation_lang)
 560         {
 561             set_proxy_negotiation(negotiation_charset,
 562                 negotiation_lang);
 563         }
 564         m_charset_converter->set_target_query_charset(query_charset);
 565         if (!url[0])
 566         {
 567             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 568             return 0;
 569         }
 570         // we don't handle multiplexing for cookie session, so we just
 571         // pick the first one in this case (anonymous users will be able
 572         // to use any backend)
 573         if (cookie && *cookie)
 574             m_proxyTarget = (char*) xstrdup(url[0]);
 575         else
 576             m_proxyTarget = (char*) xstrdup(load_balance(url));
 577     }
 578     if (cookie && *cookie)
 579     {   // search in sessions with a cookie
 580         for (c = parent->m_clientPool; c; c = c->m_next)
 581         {
 582             assert (c->m_prev);
 583             assert (*c->m_prev == c);
 584             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 585                 !strcmp(m_proxyTarget, c->get_hostname()))
 586             {
 587                 // Found it in cache
 588                 // The following handles "cancel"
 589                 // If connection is busy (waiting for PDU) and
 590                 // we have an initRequest we can safely do re-open
 591                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 592                 {
 593                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 594                              c->get_hostname());
 595                     c->close();
 596                     c->m_init_flag = 0;
 597
 598                     c->m_last_ok = 0;
 599                     c->m_cache.clear();
 600                     c->m_last_resultCount = 0;
 601                     c->m_sr_transform = 0;
 602                     c->m_waiting = 0;
 603                     c->m_resultSetStartPoint = 0;
 604                     c->m_target_idletime = m_target_idletime;
 605                     if (c->client(m_proxyTarget))
 606                     {
 607                         delete c;
 608                         return 0;
 609                     }
 610                     c->timeout(30);
 611                 }
 612                 c->m_seqno = parent->m_seqno;
 613                 if (c->m_server && c->m_server != this)
 614                     c->m_server->m_client = 0;
 615                 c->m_server = this;
 616                 (parent->m_seqno)++;
 617                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 618                 return c;
 619             }
 620         }
 621     }
 622     else if (!c &&
 623             apdu->which == Z_APDU_initRequest &&
 624             apdu->u.initRequest->idAuthentication == 0 &&
 625             !ODR_MASK_GET(apdu->u.initRequest->options, Z_Options_negotiationModel))
 626     {
 627         // anonymous sessions without cookie.
 628         // if authentication is set it is NOT anonymous se we can't share them.
 629         // If charset and lang negotiation is use it is NOT anonymous session too.
 630         for (c = parent->m_clientPool; c; c = c->m_next)
 631         {
 632             assert(c->m_prev);
 633             assert(*c->m_prev == c);
 634             if (c->m_server == 0 && c->m_cookie == 0 &&
 635                 c->m_waiting == 0 &&
 636                 !strcmp(m_proxyTarget, c->get_hostname()))
 637             {
 638                 // found it in cache
 639                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 640                          m_session_str, parent->m_seqno, c->get_hostname());
 641
 642                 c->m_seqno = parent->m_seqno;
 643                 assert(c->m_server == 0);
 644                 c->m_server = this;
 645
 646                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 647                     c->set_APDU_yazlog(1);
 648                 else
 649                     c->set_APDU_yazlog(0);
 650
 651                 (parent->m_seqno)++;
 652
 653                 parent->pre_init();
 654
 655                 return c;
 656             }
 657         }
 658     }
 659     if (!m_client)
 660     {
 661         if (apdu->which != Z_APDU_initRequest)
 662         {
 663             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 664             return 0;
 665         }
 666         Z_InitRequest *initRequest = apdu->u.initRequest;
 667
 668         if (initRequest->idAuthentication)
 669         {
 670             // the client uses authentication. We set the keepalive PDU
 671             // to 0 so we don't cache it in releaseClient
 672             m_keepalive_limit_pdu = 0;
 673         }
 674         // go through list of clients - and find the lowest/oldest one.
 675         Yaz_ProxyClient *c_min = 0;
 676         int min_seq = -1;
 677         int no_of_clients = 0;
 678         if (parent->m_clientPool)
 679             yaz_log (YLOG_DEBUG, "Existing sessions");
 680         for (c = parent->m_clientPool; c; c = c->m_next)
 681         {
 682             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 683                                c->m_waiting, c->get_hostname(),
 684                                c->m_cookie ? c->m_cookie : "");
 685             no_of_clients++;
 686             if (min_seq < 0 || c->m_seqno < min_seq)
 687             {
 688                 min_seq = c->m_seqno;
 689                 c_min = c;
 690             }
 691         }
 692         if (no_of_clients >= parent->m_max_clients)
 693         {
 694             c = c_min;
 695             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 696             {
 697                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 698                          m_session_str, parent->m_max_clients, c->m_seqno);
 699                 if (c->m_server && c->m_server != this)
 700                     delete c->m_server;   // PROBLEM: m_ref_count!
 701                 c->m_server = 0;
 702             }
 703             else
 704             {
 705                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 706                          m_session_str, parent->m_max_clients,
 707                          c->m_seqno, parent->m_seqno, c->get_hostname());
 708                 xfree (c->m_cookie);
 709                 c->m_cookie = 0;
 710                 if (cookie)
 711                     c->m_cookie = xstrdup(cookie);
 712                 c->m_seqno = parent->m_seqno;
 713                 if (c->m_server && c->m_server != this)
 714                 {
 715                     c->m_server->m_client = 0;
 716                     delete c->m_server;   // PROBLEM: m_ref_count!
 717                 }
 718                 (parent->m_seqno)++;
 719                 c->m_target_idletime = m_target_idletime;
 720                 c->timeout(m_target_idletime);
 721
 722                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 723                     c->set_APDU_yazlog(1);
 724                 else
 725                     c->set_APDU_yazlog(0);
 726
 727                 return c;
 728             }
 729         }
 730         else
 731         {
 732             yaz_log (YLOG_LOG, "%sNEW %d %s",
 733                      m_session_str, parent->m_seqno, m_proxyTarget);
 734             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 735             c->m_next = parent->m_clientPool;
 736             if (c->m_next)
 737                 c->m_next->m_prev = &c->m_next;
 738             parent->m_clientPool = c;
 739             c->m_prev = &parent->m_clientPool;
 740         }
 741
 742         xfree (c->m_cookie);
 743         c->m_cookie = 0;
 744         if (cookie)
 745             c->m_cookie = xstrdup(cookie);
 746
 747         c->m_seqno = parent->m_seqno;
 748         c->m_init_flag = 0;
 749         c->m_last_resultCount = 0;
 750         c->m_last_ok = 0;
 751         c->m_cache.clear();
 752         c->m_sr_transform = 0;
 753         c->m_waiting = 0;
 754         c->m_resultSetStartPoint = 0;
 755         (parent->m_seqno)++;
 756         if (c->client(m_proxyTarget))
 757         {
 758             delete c;
 759             return 0;
 760         }
 761         c->m_target_idletime = m_target_idletime;
 762         c->timeout(30);
 763
 764         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 765             c->set_APDU_yazlog(1);
 766         else
 767             c->set_APDU_yazlog(0);
 768     }
 769     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 770     return c;
 771 }
 772
 773 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 774 {
 775     int i;
 776     for (i = 0; i<num; i++)
 777     {
 778         oident *ent;
 779         Z_DefaultDiagFormat *r;
 780         Z_DiagRec *p = pp[i];
 781         if (p->which != Z_DiagRec_defaultFormat)
 782         {
 783             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 784             return;
 785         }
 786         else
 787             r = p->u.defaultFormat;
 788         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 789             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 790             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 791         switch (r->which)
 792         {
 793         case Z_DefaultDiagFormat_v2Addinfo:
 794             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 795                     m_session_str,
 796                     *r->condition, diagbib1_str(*r->condition),
 797                     r->u.v2Addinfo);
 798             break;
 799         case Z_DefaultDiagFormat_v3Addinfo:
 800             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 801                     m_session_str,
 802                     *r->condition, diagbib1_str(*r->condition),
 803                     r->u.v3Addinfo);
 804             break;
 805         }
 806     }
 807 }
 808
 809 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 810 {
 811     if (!m_stylesheet_xsp || p->num_records <= 0)
 812     {
 813         return 0;  /* no XSLT to be done ... */
 814     }
 815
 816     m_stylesheet_offset = 0;
 817     m_stylesheet_nprl = p;
 818     m_stylesheet_apdu = apdu;
 819     m_timeout_mode = timeout_xsl;
 820
 821     timeout(0);
 822     return 1;
 823 }
 824
 825 void Yaz_Proxy::convert_xsl_delay()
 826 {
 827 #if HAVE_XSLT
 828     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 829     if (npr->which == Z_NamePlusRecord_databaseRecord)
 830     {
 831         Z_External *r = npr->u.databaseRecord;
 832         if (r->which == Z_External_octet)
 833         {
 834 #if 0
 835             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 836 #endif
 837             xmlDocPtr res, doc = xmlParseMemory(
 838                 (char*) r->u.octet_aligned->buf,
 839                 r->u.octet_aligned->len);
 840
 841
 842             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 843                     m_session_str, m_stylesheet_offset);
 844             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 845                                       doc, 0);
 846
 847             if (res)
 848             {
 849                 xmlChar *out_buf;
 850                 int out_len;
 851                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 852
 853                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 854                     u.databaseRecord =
 855                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 856                                  (char*) out_buf, out_len);
 857                 xmlFree(out_buf);
 858                 xmlFreeDoc(res);
 859             }
 860
 861             xmlFreeDoc(doc);
 862         }
 863     }
 864 #endif
 865     m_stylesheet_offset++;
 866     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 867     {
 868         m_timeout_mode = timeout_normal;
 869         m_stylesheet_nprl = 0;
 870 #if HAVE_XSLT
 871         if (m_stylesheet_xsp)
 872             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 873 #endif
 874         m_stylesheet_xsp = 0;
 875         timeout(m_client_idletime);
 876         send_PDU_convert(m_stylesheet_apdu);
 877     }
 878     else
 879         timeout(0);
 880 }
 881
 882 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 883 {
 884     if (m_frontend_type != VAL_NONE)
 885     {
 886         int i;
 887         for (i = 0; i < p->num_records; i++)
 888         {
 889             Z_NamePlusRecord *npr = p->records[i];
 890             if (npr->which == Z_NamePlusRecord_databaseRecord)
 891             {
 892                 Z_External *r = npr->u.databaseRecord;
 893                 if (r->which == Z_External_octet)
 894                 {
 895 #if HAVE_USEMARCON
 896                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 897                     {
 898                         if (!m_usemarcon->m_stage1)
 899                         {
 900                             m_usemarcon->m_stage1 = new CDetails();
 901                         }
 902                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 903                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 904                         int res = m_usemarcon->m_stage1->Start();
 905                         if (res == 0)
 906                         {
 907                             char *converted;
 908                             int convlen;
 909                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 910                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 911                             {
 912                                 if (!m_usemarcon->m_stage2)
 913                                 {
 914                                     m_usemarcon->m_stage2 = new CDetails();
 915                                 }
 916                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 917                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 918                                 res = m_usemarcon->m_stage2->Start();
 919                                 if (res == 0)
 920                                 {
 921                                     free(converted);
 922                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 923                                 }
 924                                 else
 925                                 {
 926                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 927                                 }
 928                             }
 929                             npr->u.databaseRecord =
 930                                 z_ext_record(odr_encode(),
 931                                              m_frontend_type,
 932                                              converted,
 933                                              strlen(converted));
 934                             free(converted);
 935                         }
 936                         else
 937                         {
 938                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 939                         }
 940                         continue;
 941                     }
 942 #endif
 943 /* HAVE_USEMARCON */
 944                     npr->u.databaseRecord =
 945                         z_ext_record(odr_encode(),
 946                                      m_frontend_type,
 947                                      (char*) r->u.octet_aligned->buf,
 948                                      r->u.octet_aligned->len);
 949                 }
 950             }
 951         }
 952     }
 953 }
 954
 955 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
 956                                         const char *backend_charset)
 957 {
 958     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
 959     int sel =   m_charset_converter->get_client_charset_selected();
 960     const char *client_record_charset =
 961         m_charset_converter->get_client_query_charset();
 962     if (sel && backend_charset && client_record_charset &&
 963         strcmp(backend_charset, client_record_charset))
 964     {
 965         int i;
 966         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
 967                                         backend_charset);
 968         yaz_marc_t mt = yaz_marc_create();
 969         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
 970         yaz_marc_iconv(mt, cd);
 971         for (i = 0; i < p->num_records; i++)
 972         {
 973             Z_NamePlusRecord *npr = p->records[i];
 974             if (npr->which == Z_NamePlusRecord_databaseRecord)
 975             {
 976                 Z_External *r = npr->u.databaseRecord;
 977                 oident *ent = oid_getentbyoid(r->direct_reference);
 978                 if (!ent || ent->value == VAL_NONE)
 979                     continue;
 980
 981                 if (ent->value == VAL_SUTRS)
 982                 {
 983                     WRBUF w = wrbuf_alloc();
 984
 985                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
 986                                       r->u.octet_aligned->len);
 987                     npr->u.databaseRecord =
 988                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
 989                                      wrbuf_len(w));
 990                     wrbuf_free(w, 1);
 991                 }
 992                 else if (ent->value == VAL_TEXT_XML)
 993                 {
 994                     ;
 995                 }
 996                 else if (r->which == Z_External_octet)
 997                 {
 998                     int rlen;
 999                     char *result;
1000                     if (yaz_marc_decode_buf(mt,
1001                                             (char*) r->u.octet_aligned->buf,
1002                                             r->u.octet_aligned->len,
1003                                             &result, &rlen))
1004                     {
1005                         npr->u.databaseRecord =
1006                             z_ext_record(odr_encode(), ent->value, result, rlen);
1007                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1008                                 m_session_str);
1009                     }
1010                 }
1011             }
1012         }
1013         if (cd)
1014             yaz_iconv_close(cd);
1015         yaz_marc_destroy(mt);
1016     }
1017     else
1018     {
1019         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
1020     }
1021 }
1022
1023 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1024                                    const char *backend_charset)
1025 {
1026     int i;
1027     if (!backend_charset)
1028         backend_charset = "MARC-8";
1029     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1030     yaz_marc_t mt = yaz_marc_create();
1031     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1032     yaz_marc_iconv(mt, cd);
1033     for (i = 0; i < p->num_records; i++)
1034     {
1035         Z_NamePlusRecord *npr = p->records[i];
1036         if (npr->which == Z_NamePlusRecord_databaseRecord)
1037         {
1038             Z_External *r = npr->u.databaseRecord;
1039             if (r->which == Z_External_OPAC)
1040             {
1041                 WRBUF w = wrbuf_alloc();
1042
1043                 yaz_display_OPAC(w, r->u.opac, 0);
1044                 npr->u.databaseRecord = z_ext_record(
1045                     odr_encode(), VAL_TEXT_XML,
1046                     wrbuf_buf(w), wrbuf_len(w)
1047                     );
1048                 wrbuf_free(w, 1);
1049             }
1050             else if (r->which == Z_External_octet)
1051             {
1052                 int rlen;
1053                 char *result;
1054                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1055                                         r->u.octet_aligned->len,
1056                                         &result, &rlen))
1057                 {
1058                     npr->u.databaseRecord =
1059                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
1060                 }
1061             }
1062         }
1063     }
1064     if (cd)
1065         yaz_iconv_close(cd);
1066     yaz_marc_destroy(mt);
1067 }
1068
1069 void Yaz_Proxy::logtime()
1070 {
1071 #if HAVE_GETTIMEOFDAY
1072     struct timeval *tv = (struct timeval*) m_time_tv;
1073     if (tv->tv_sec)
1074     {
1075         struct timeval tv1;
1076         gettimeofday(&tv1, 0);
1077         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1078             (tv1.tv_usec - tv->tv_usec);
1079         if (diff >= 0)
1080             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1081                     diff/1000000, (diff/1000)%1000);
1082     }
1083     tv->tv_sec = 0;
1084     tv->tv_usec = 0;
1085 #endif
1086 }
1087
1088 int Yaz_Proxy::send_http_response(int code)
1089 {
1090     ODR o = odr_encode();
1091     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1092     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1093     if (m_http_version)
1094         hres->version = odr_strdup(o, m_http_version);
1095     if (m_http_keepalive)
1096         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1097     else
1098         timeout(0);
1099
1100     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1101     {
1102         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1103                  gdu_name(gdu));
1104     }
1105     int len;
1106     int r = send_GDU(gdu, &len);
1107     m_bytes_sent += len;
1108     m_bw_stat.add_bytes(len);
1109     logtime();
1110     return r;
1111 }
1112
1113 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
1114 {
1115     ODR o = odr_encode();
1116     const char *ctype = "text/xml";
1117     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
1118     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1119     if (m_http_version)
1120         hres->version = odr_strdup(o, m_http_version);
1121     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1122     if (m_http_keepalive)
1123         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1124     else
1125         timeout(0);
1126
1127     static Z_SOAP_Handler soap_handlers[2] = {
1128 #if HAVE_XSLT
1129         {"http://www.loc.gov/zing/srw/", 0,
1130          (Z_SOAP_fun) yaz_srw_codec},
1131 #endif
1132         {0, 0, 0}
1133     };
1134
1135     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1136     soap_package->which = Z_SOAP_generic;
1137     soap_package->u.generic =
1138         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1139     soap_package->u.generic->no = 0;
1140     soap_package->u.generic->ns = soap_handlers[0].ns;
1141     soap_package->u.generic->p = (void *) srw_pdu;
1142     soap_package->ns = m_soap_ns;
1143     z_soap_codec_enc_xsl(o, &soap_package,
1144                          &hres->content_buf, &hres->content_len,
1145                          soap_handlers, 0, m_s2z_stylesheet);
1146     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1147     {
1148         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1149                  gdu_name(gdu));
1150     }
1151     int len;
1152     int r = send_GDU(gdu, &len);
1153     m_bytes_sent += len;
1154     m_bw_stat.add_bytes(len);
1155     logtime();
1156     return r;
1157 }
1158
1159 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1160 {
1161     ODR o = odr_encode();
1162     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1163     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1164
1165     srw_res->num_diagnostics = 1;
1166     srw_res->diagnostics = (Z_SRW_diagnostic *)
1167         odr_malloc(o, sizeof(*srw_res->diagnostics));
1168     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
1169     return send_srw_response(srw_pdu);
1170 }
1171
1172 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1173                              Z_DefaultDiagFormat *ddf)
1174 {
1175     int bib1_code = *ddf->condition;
1176     if (bib1_code == 109)
1177         return 404;
1178     srw_res->num_diagnostics = 1;
1179     srw_res->diagnostics = (Z_SRW_diagnostic *)
1180         odr_malloc(o, sizeof(*srw_res->diagnostics));
1181     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1182                           yaz_diag_bib1_to_srw(*ddf->condition),
1183                           ddf->u.v2Addinfo);
1184     return 0;
1185 }
1186
1187 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1188 {
1189     ODR o = odr_encode();
1190     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1191     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1192
1193     srw_res->numberOfRecords = odr_intdup (o, hits);
1194     if (records && records->which == Z_Records_DBOSD)
1195     {
1196         srw_res->num_records =
1197             records->u.databaseOrSurDiagnostics->num_records;
1198         int i;
1199         srw_res->records = (Z_SRW_record *)
1200             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1201         for (i = 0; i < srw_res->num_records; i++)
1202         {
1203             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1204             if (npr->which != Z_NamePlusRecord_databaseRecord)
1205             {
1206                 srw_res->records[i].recordSchema = "diagnostic";
1207                 srw_res->records[i].recordPacking = m_s2z_packing;
1208                 srw_res->records[i].recordData_buf = "67";
1209                 srw_res->records[i].recordData_len = 2;
1210                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1211                 continue;
1212             }
1213             Z_External *r = npr->u.databaseRecord;
1214             oident *ent = oid_getentbyoid(r->direct_reference);
1215             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1216             {
1217                 srw_res->records[i].recordSchema = m_schema;
1218                 srw_res->records[i].recordPacking = m_s2z_packing;
1219                 srw_res->records[i].recordData_buf = (char*)
1220                     r->u.octet_aligned->buf;
1221                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1222                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1223             }
1224             else
1225             {
1226                 srw_res->records[i].recordSchema = "diagnostic";
1227                 srw_res->records[i].recordPacking = m_s2z_packing;
1228                 srw_res->records[i].recordData_buf = "67";
1229                 srw_res->records[i].recordData_len = 2;
1230                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1231             }
1232         }
1233     }
1234     if (records && records->which == Z_Records_NSD)
1235     {
1236         int http_code;
1237         http_code = z_to_srw_diag(odr_encode(), srw_res,
1238                                    records->u.nonSurrogateDiagnostic);
1239         if (http_code)
1240             return send_http_response(http_code);
1241     }
1242     return send_srw_response(srw_pdu);
1243
1244 }
1245
1246 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1247                                         int num_diagnostics)
1248 {
1249     Yaz_ProxyConfig *cfg = check_reconfigure();
1250     if (cfg)
1251     {
1252         int len;
1253         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1254                                        m_s2z_database, &len);
1255         if (b)
1256         {
1257             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1258             Z_SRW_explainResponse *er = res->u.explain_response;
1259
1260             er->record.recordData_buf = b;
1261             er->record.recordData_len = len;
1262             er->record.recordPacking = m_s2z_packing;
1263             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1264
1265             er->diagnostics = diagnostics;
1266             er->num_diagnostics = num_diagnostics;
1267             return send_srw_response(res);
1268         }
1269     }
1270     return send_http_response(404);
1271 }
1272
1273 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1274 {
1275     if (m_http_version)
1276     {
1277         if (apdu->which == Z_APDU_initResponse)
1278         {
1279             Z_InitResponse *res = apdu->u.initResponse;
1280             if (*res->result == 0)
1281             {
1282                 send_to_srw_client_error(3, 0);
1283             }
1284             else if (!m_s2z_search_apdu)
1285             {
1286                 send_srw_explain_response(0, 0);
1287             }
1288             else
1289             {
1290                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1291             }
1292         }
1293         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1294         {
1295             m_s2z_search_apdu = 0;
1296             Z_SearchResponse *res = apdu->u.searchResponse;
1297             m_s2z_hit_count = *res->resultCount;
1298             if (res->records && res->records->which == Z_Records_NSD)
1299             {
1300                 send_to_srw_client_ok(0, res->records, 1);
1301             }
1302             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1303             {
1304                 // adjust
1305                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1306
1307                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1308                 {
1309                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1310                         > m_s2z_hit_count)
1311                         *pr->numberOfRecordsRequested =
1312                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1313                 }
1314                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1315             }
1316             else
1317             {
1318                 m_s2z_present_apdu = 0;
1319                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1320             }
1321         }
1322         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1323         {
1324             int start =
1325                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1326
1327             m_s2z_present_apdu = 0;
1328             Z_PresentResponse *res = apdu->u.presentResponse;
1329             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1330         }
1331     }
1332     else
1333     {
1334         int len = 0;
1335         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1336             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1337                      apdu_name(apdu));
1338         int r = send_Z_PDU(apdu, &len);
1339         m_bytes_sent += len;
1340         m_bw_stat.add_bytes(len);
1341         logtime();
1342         return r;
1343     }
1344     return 0;
1345 }
1346
1347 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1348 {
1349     int kill_session = 0;
1350     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1351
1352     if (new_id)
1353         *new_id = m_referenceId;
1354
1355     if (apdu->which == Z_APDU_searchResponse)
1356     {
1357         Z_SearchResponse *sr = apdu->u.searchResponse;
1358         Z_Records *p = sr->records;
1359         if (p && p->which == Z_Records_NSD)
1360         {
1361             Z_DiagRec dr, *dr_p = &dr;
1362             dr.which = Z_DiagRec_defaultFormat;
1363             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1364
1365             *sr->searchStatus = 0;
1366             display_diagrecs(&dr_p, 1);
1367         }
1368         else
1369         {
1370             if (p && p->which == Z_Records_DBOSD)
1371             {
1372                 if (m_backend_type
1373 #if HAVE_USEMARCON
1374                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1375 #endif
1376                     )
1377                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1378                 if (m_marcxml_mode == marcxml)
1379                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1380                                        m_backend_charset);
1381                 else
1382                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1383                                             m_backend_charset);
1384                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1385                     return 0;
1386
1387             }
1388             if (sr->resultCount)
1389             {
1390                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1391                         *sr->resultCount);
1392                 if (*sr->resultCount < 0)
1393                 {
1394                     m_flag_invalid_session = 1;
1395                     kill_session = 1;
1396
1397                     *sr->searchStatus = 0;
1398                     sr->records =
1399                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1400                     *sr->resultCount = 0;
1401                 }
1402             }
1403         }
1404     }
1405     else if (apdu->which == Z_APDU_presentResponse)
1406     {
1407         Z_PresentResponse *sr = apdu->u.presentResponse;
1408         Z_Records *p = sr->records;
1409         if (p && p->which == Z_Records_NSD)
1410         {
1411             Z_DiagRec dr, *dr_p = &dr;
1412             dr.which = Z_DiagRec_defaultFormat;
1413             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1414             if (*sr->presentStatus == Z_PresentStatus_success)
1415                 *sr->presentStatus = Z_PresentStatus_failure;
1416             display_diagrecs(&dr_p, 1);
1417         }
1418         if (p && p->which == Z_Records_DBOSD)
1419         {
1420             if (m_backend_type
1421 #if HAVE_USEMARCON
1422                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1423 #endif
1424                 )
1425                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1426             if (m_marcxml_mode == marcxml)
1427                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1428                                    m_backend_charset);
1429             else
1430                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1431                                         m_backend_charset);
1432             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1433                 return 0;
1434         }
1435     }
1436     else if (apdu->which == Z_APDU_initResponse)
1437     {
1438         //Get and check negotiation record
1439         //from init response.
1440         handle_charset_lang_negotiation(apdu);
1441
1442         if (m_initRequest_options)
1443         {
1444             Z_Options *nopt =
1445                 (Odr_bitmask *)odr_malloc(odr_encode(),
1446                                           sizeof(Odr_bitmask));
1447             ODR_MASK_ZERO(nopt);
1448
1449             int i;
1450             for (i = 0; i<24; i++)
1451                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1452                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1453                     ODR_MASK_SET(nopt, i);
1454             apdu->u.initResponse->options = nopt;
1455         }
1456         if (m_initRequest_version)
1457         {
1458             Z_ProtocolVersion *nopt =
1459                 (Odr_bitmask *)odr_malloc(odr_encode(),
1460                                           sizeof(Odr_bitmask));
1461             ODR_MASK_ZERO(nopt);
1462
1463             int i;
1464             for (i = 0; i<8; i++)
1465                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1466                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1467                     ODR_MASK_SET(nopt, i);
1468             apdu->u.initResponse->protocolVersion = nopt;
1469         }
1470         apdu->u.initResponse->preferredMessageSize =
1471             odr_intdup(odr_encode(),
1472                        m_client->m_initResponse_preferredMessageSize >
1473                        m_initRequest_preferredMessageSize ?
1474                        m_initRequest_preferredMessageSize :
1475                        m_client->m_initResponse_preferredMessageSize);
1476         apdu->u.initResponse->maximumRecordSize =
1477             odr_intdup(odr_encode(),
1478                        m_client->m_initResponse_maximumRecordSize >
1479                        m_initRequest_maximumRecordSize ?
1480                        m_initRequest_maximumRecordSize :
1481                        m_client->m_initResponse_maximumRecordSize);
1482     }
1483
1484     int r = send_PDU_convert(apdu);
1485     if (r)
1486         return r;
1487     if (kill_session)
1488     {
1489         delete m_client;
1490         m_client = 0;
1491         m_parent->pre_init();
1492     }
1493     return r;
1494 }
1495
1496 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1497 {
1498     int len = 0;
1499     const char *apdu_name_tmp = apdu_name(apdu);
1500     int r = send_Z_PDU(apdu, &len);
1501     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1502         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1503                  get_session_str(),
1504                  apdu_name_tmp, get_hostname(), len);
1505     m_bytes_sent += len;
1506     return r;
1507 }
1508
1509 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1510 {
1511     if (apdu->which == Z_APDU_presentRequest)
1512     {
1513         Z_PresentRequest *pr = apdu->u.presentRequest;
1514         int toget = *pr->numberOfRecordsRequested;
1515         int start = *pr->resultSetStartPoint;
1516
1517         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1518                 pr->resultSetId, start, toget);
1519
1520         if (*m_parent->m_optimize == '0')
1521             return apdu;
1522
1523         if (!m_client->m_last_resultSetId)
1524         {
1525             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1526             new_apdu->u.presentResponse->records =
1527                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1528                                                pr->resultSetId);
1529             send_to_client(new_apdu);
1530             return 0;
1531         }
1532         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1533         {
1534             if (start+toget-1 > m_client->m_last_resultCount)
1535             {
1536                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1537                 new_apdu->u.presentResponse->records =
1538                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1539                 send_to_client(new_apdu);
1540                 return 0;
1541             }
1542             Z_NamePlusRecordList *npr;
1543 #if 0
1544             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1545                     m_session_str, start, toget, yaz_z3950oid_to_str(
1546                         pr->preferredRecordSyntax, &oclass));
1547 #endif
1548             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1549                                           pr->preferredRecordSyntax,
1550                                           pr->recordComposition))
1551             {
1552                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1553                          m_session_str);
1554                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1555                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1556
1557                 new_apdu->u.presentResponse->numberOfRecordsReturned
1558                     = odr_intdup(odr_encode(), toget);
1559
1560                 new_apdu->u.presentResponse->records = (Z_Records*)
1561                     odr_malloc(odr_encode(), sizeof(Z_Records));
1562                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1563                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1564                 new_apdu->u.presentResponse->nextResultSetPosition =
1565                     odr_intdup(odr_encode(), start+toget);
1566
1567                 send_to_client(new_apdu);
1568                 return 0;
1569             }
1570         }
1571     }
1572
1573     if (apdu->which != Z_APDU_searchRequest)
1574         return apdu;
1575     Z_SearchRequest *sr = apdu->u.searchRequest;
1576     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1577     Yaz_Z_Databases this_databases;
1578
1579     this_databases.set(sr->num_databaseNames, (const char **)
1580                        sr->databaseNames);
1581
1582     this_query->set_Z_Query(sr->query);
1583
1584     char query_str[120];
1585     this_query->print(query_str, sizeof(query_str)-1);
1586     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1587
1588     if (*m_parent->m_optimize != '0' &&
1589         m_client->m_last_ok && m_client->m_last_query &&
1590         m_client->m_last_query->match(this_query) &&
1591         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1592         m_client->m_last_databases.match(this_databases))
1593     {
1594         delete this_query;
1595         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1596             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1597         {
1598             Z_NamePlusRecordList *npr;
1599             int toget = *sr->mediumSetPresentNumber;
1600             Z_RecordComposition *comp = 0;
1601
1602             if (toget > m_client->m_last_resultCount)
1603                 toget = m_client->m_last_resultCount;
1604
1605             if (sr->mediumSetElementSetNames)
1606             {
1607                 comp = (Z_RecordComposition *)
1608                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1609                 comp->which = Z_RecordComp_simple;
1610                 comp->u.simple = sr->mediumSetElementSetNames;
1611             }
1612
1613             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1614                                           sr->preferredRecordSyntax, comp))
1615             {
1616                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1617                          m_session_str);
1618                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1619                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1620                 new_apdu->u.searchResponse->resultCount =
1621                     &m_client->m_last_resultCount;
1622
1623                 new_apdu->u.searchResponse->numberOfRecordsReturned
1624                     = odr_intdup(odr_encode(), toget);
1625
1626                 new_apdu->u.searchResponse->presentStatus =
1627                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1628                 new_apdu->u.searchResponse->records = (Z_Records*)
1629                     odr_malloc(odr_encode(), sizeof(Z_Records));
1630                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1631                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1632                 new_apdu->u.searchResponse->nextResultSetPosition =
1633                     odr_intdup(odr_encode(), toget+1);
1634                 send_to_client(new_apdu);
1635                 return 0;
1636             }
1637             else
1638             {
1639                 // medium Set
1640                 // send present request (medium size)
1641                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1642                          m_session_str);
1643
1644                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1645                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1646                 pr->referenceId = sr->referenceId;
1647                 pr->resultSetId = sr->resultSetName;
1648                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1649                 *pr->numberOfRecordsRequested = toget;
1650                 pr->recordComposition = comp;
1651                 m_client->m_sr_transform = 1;
1652                 return new_apdu;
1653             }
1654         }
1655         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1656             m_client->m_last_resultCount <= 0)
1657         {
1658             // large set. Return pseudo-search response immediately
1659             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1660                      m_session_str);
1661             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1662             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1663             new_apdu->u.searchResponse->resultCount =
1664                 &m_client->m_last_resultCount;
1665             send_to_client(new_apdu);
1666             return 0;
1667         }
1668         else
1669         {
1670             Z_NamePlusRecordList *npr;
1671             int toget = m_client->m_last_resultCount;
1672             Z_RecordComposition *comp = 0;
1673             // small set
1674             // send a present request (small set)
1675
1676             if (sr->smallSetElementSetNames)
1677             {
1678                 comp = (Z_RecordComposition *)
1679                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1680                 comp->which = Z_RecordComp_simple;
1681                 comp->u.simple = sr->smallSetElementSetNames;
1682             }
1683
1684             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1685                                           sr->preferredRecordSyntax, comp))
1686             {
1687                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1688                          m_session_str);
1689                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1690                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1691                 new_apdu->u.searchResponse->resultCount =
1692                     &m_client->m_last_resultCount;
1693
1694                 new_apdu->u.searchResponse->numberOfRecordsReturned
1695                     = odr_intdup(odr_encode(), toget);
1696
1697                 new_apdu->u.searchResponse->presentStatus =
1698                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1699                 new_apdu->u.searchResponse->records = (Z_Records*)
1700                     odr_malloc(odr_encode(), sizeof(Z_Records));
1701                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1702                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1703                 new_apdu->u.searchResponse->nextResultSetPosition =
1704                     odr_intdup(odr_encode(), toget+1);
1705                 send_to_client(new_apdu);
1706                 return 0;
1707             }
1708             else
1709             {
1710                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1711                          m_session_str);
1712                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1713                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1714                 pr->referenceId = sr->referenceId;
1715                 pr->resultSetId = sr->resultSetName;
1716                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1717                 *pr->numberOfRecordsRequested = toget;
1718                 pr->recordComposition = comp;
1719                 m_client->m_sr_transform = 1;
1720                 return new_apdu;
1721             }
1722         }
1723     }
1724     else  // query doesn't match
1725     {
1726         delete m_client->m_last_query;
1727         m_client->m_last_query = this_query;
1728         m_client->m_last_ok = 0;
1729         m_client->m_cache.clear();
1730         m_client->m_resultSetStartPoint = 0;
1731
1732         xfree (m_client->m_last_resultSetId);
1733         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1734
1735         m_client->m_last_databases.set(sr->num_databaseNames,
1736                                        (const char **) sr->databaseNames);
1737     }
1738     return apdu;
1739 }
1740
1741
1742 void Yaz_Proxy::inc_request_no()
1743 {
1744     char *cp = strchr(m_session_str, ' ');
1745     m_request_no++;
1746     if (cp)
1747         sprintf(cp+1, "%d ", m_request_no);
1748 }
1749
1750 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1751 {
1752     inc_request_no();
1753
1754     m_bytes_recv += len;
1755
1756     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1757         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1758                  m_session_str, gdu_name(apdu), len);
1759
1760 #if HAVE_GETTIMEOFDAY
1761     gettimeofday((struct timeval *) m_time_tv, 0);
1762 #endif
1763     m_bw_stat.add_bytes(len);
1764     m_pdu_stat.add_bytes(1);
1765
1766     GDU *gdu = new GDU(apdu);
1767     m_in_queue.enqueue(gdu);
1768
1769     recv_GDU_more(false);
1770 }
1771
1772 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
1773 {
1774     int bw_total = m_bw_stat.get_total();
1775     int pdu_total = m_pdu_stat.get_total();
1776     int reduce = 0;
1777
1778     assert(m_timeout_mode == timeout_busy);
1779     assert(m_timeout_gdu == 0);
1780
1781     if (m_bw_max)
1782     {
1783         if (bw_total > m_bw_max)
1784         {
1785             reduce = (bw_total/m_bw_max);
1786         }
1787     }
1788     if (m_pdu_max)
1789     {
1790         if (pdu_total > m_pdu_max)
1791         {
1792             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1793             reduce = (reduce > nreduce) ? reduce : nreduce;
1794         }
1795     }
1796     m_http_version = 0;
1797
1798 #if 0
1799     /* uncomment to force a big reduce */
1800     m_timeout_mode = timeout_reduce;
1801     m_timeout_gdu = gdu;
1802     timeout(3);       // call us reduce seconds later
1803     return;
1804 #endif
1805     if (reduce)
1806     {
1807         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1808                 m_session_str, reduce, bw_total, pdu_total,
1809                 m_bw_max, m_pdu_max);
1810
1811         m_timeout_mode = timeout_reduce;
1812         m_timeout_gdu = gdu;
1813         timeout(reduce);       // call us reduce seconds later
1814     }
1815     else
1816         recv_GDU_normal(gdu);
1817 }
1818
1819 void Yaz_Proxy::recv_GDU_more(bool normal)
1820 {
1821     GDU *g;
1822     if (normal && m_timeout_mode == timeout_busy)
1823         m_timeout_mode = timeout_normal;
1824     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
1825     {
1826         m_timeout_mode = timeout_busy;
1827         recv_GDU_reduce(g);
1828     }
1829 }
1830
1831 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
1832 {
1833     Z_GDU *apdu = gdu->get();
1834     gdu->extract_odr_to(odr_decode());
1835     delete gdu;
1836
1837     if (apdu->which == Z_GDU_Z3950)
1838         handle_incoming_Z_PDU(apdu->u.z3950);
1839     else if (apdu->which == Z_GDU_HTTP_Request)
1840         handle_incoming_HTTP(apdu->u.HTTP_Request);
1841 }
1842
1843 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1844 {
1845     if (m_max_record_retrieve)
1846     {
1847         if (apdu->which == Z_APDU_presentRequest)
1848         {
1849             Z_PresentRequest *pr = apdu->u.presentRequest;
1850             if (pr->numberOfRecordsRequested &&
1851                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1852                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1853         }
1854     }
1855 }
1856
1857 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1858 {
1859     if (apdu->which == Z_APDU_initRequest)
1860     {
1861         yaz_log(YLOG_LOG, "%shandle_charset_lang_negotiation",
1862                 m_session_str);
1863         if (m_initRequest_options &&
1864             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1865             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1866         {
1867             // There is no negotiation proposal from
1868             // client's side. OK. The proxy negotiation
1869             // in use, only.
1870             Z_InitRequest *initRequest = apdu->u.initRequest;
1871             Z_OtherInformation **otherInfo;
1872             Z_OtherInformationUnit *oi;
1873             get_otherInfoAPDU(apdu, &otherInfo);
1874             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1875             if (oi)
1876             {
1877                 ODR_MASK_SET(initRequest->options,
1878                     Z_Options_negotiationModel);
1879                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1880                 oi->information.externallyDefinedInfo =
1881                 yaz_set_proposal_charneg(odr_encode(),
1882                     (const char**)&m_proxy_negotiation_charset,
1883                     m_proxy_negotiation_charset ? 1:0,
1884                     (const char**)&m_proxy_negotiation_lang,
1885                     m_proxy_negotiation_lang ? 1:0,
1886                     1);
1887             }
1888         }
1889         else if (m_initRequest_options &&
1890                  ODR_MASK_GET(m_initRequest_options,
1891                               Z_Options_negotiationModel) &&
1892                  m_charset_converter->get_target_query_charset())
1893         {
1894             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1895                     m_session_str,
1896                     m_charset_converter->get_target_query_charset());
1897             Z_InitRequest *initRequest = apdu->u.initRequest;
1898             Z_CharSetandLanguageNegotiation *negotiation =
1899                 yaz_get_charneg_record (initRequest->otherInfo);
1900             if (negotiation &&
1901                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1902             {
1903                 NMEM nmem = nmem_create();
1904                 char **charsets = 0;
1905                 int num_charsets = 0;
1906                 char **langs = 0;
1907                 int num_langs = 0;
1908                 int selected = 0;
1909                 yaz_get_proposal_charneg (nmem, negotiation,
1910                                           &charsets, &num_charsets,
1911                                           &langs, &num_langs, &selected);
1912                 int i;
1913                 for (i = 0; i<num_charsets; i++)
1914                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
1915                             charsets[i]);
1916                 for (i = 0; i<num_langs; i++)
1917                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
1918                             langs[i]);
1919
1920                 const char *t_charset =
1921                     m_charset_converter->get_target_query_charset();
1922                 // sweep through charsets and pick the first supported
1923                 // conversion
1924                 for (i = 0; i<num_charsets; i++)
1925                 {
1926                     const char *c_charset = charsets[i];
1927                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
1928                         break;
1929                 }
1930                 if (i != num_charsets)
1931                 {
1932                     // got one .. set up ODR for reverse direction
1933                     const char *c_charset = charsets[i];
1934                     odr_set_charset(odr_encode(), c_charset, t_charset);
1935                     m_charset_converter->set_client_query_charset(c_charset);
1936                     m_charset_converter->set_client_charset_selected(selected);
1937                 }
1938                 nmem_destroy(nmem);
1939                 ODR_MASK_CLEAR(m_initRequest_options,
1940                                Z_Options_negotiationModel);
1941                 yaz_del_charneg_record(&initRequest->otherInfo);
1942             }
1943             else
1944             {
1945                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
1946                         m_session_str);
1947             }
1948         }
1949     }
1950     else if (apdu->which == Z_APDU_initResponse)
1951     {
1952         Z_InitResponse *initResponse = apdu->u.initResponse;
1953         Z_OtherInformation **otherInfo;
1954
1955         if (ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel))
1956         {
1957             char *charset = 0;
1958             char *lang = 0;
1959             int selected = 0;
1960
1961             get_otherInfoAPDU(apdu, &otherInfo);
1962
1963             if (!otherInfo && !(*otherInfo))
1964                 return;
1965
1966             Z_CharSetandLanguageNegotiation *charneg =
1967                 yaz_get_charneg_record(*otherInfo);
1968
1969             if (!charneg)
1970                 return;
1971
1972             yaz_get_response_charneg(m_referenceId_mem, charneg,
1973                 &charset, &lang, &selected);
1974
1975             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
1976                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
1977
1978             if (m_initRequest_options &&
1979                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
1980             {
1981                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
1982                     m_session_str);
1983             }
1984             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1985             {
1986                 // negotiation-charset, negotiation-lang
1987                 // elements of config file in use.
1988
1989                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
1990                     m_session_str);
1991
1992                 // clear negotiation option.
1993                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
1994
1995                 // Delete negotiation (charneg-3) entry.
1996                 yaz_del_charneg_record(otherInfo);
1997             }
1998         }
1999         else
2000         {
2001             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2002             {
2003                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2004                         m_session_str);
2005             }
2006             else if (m_charset_converter->get_client_query_charset())
2007             {
2008                 Z_OtherInformation **otherInfo;
2009                 Z_OtherInformationUnit *oi;
2010                 get_otherInfoAPDU(apdu, &otherInfo);
2011                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2012                 if (oi)
2013                 {
2014                     ODR_MASK_SET(initResponse->options,
2015                                  Z_Options_negotiationModel);
2016                     ODR_MASK_SET(m_initRequest_options,
2017                                  Z_Options_negotiationModel);
2018
2019                     oi->which = Z_OtherInfo_externallyDefinedInfo;
2020                     oi->information.externallyDefinedInfo =
2021                         yaz_set_response_charneg(
2022                             odr_encode(),
2023                             m_charset_converter->get_client_query_charset(),
2024                             0 /* no lang */,
2025                             m_charset_converter->get_client_charset_selected());
2026                 }
2027             }
2028         }
2029     }
2030 }
2031
2032 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2033                                                      int error,
2034                                                      const char *addinfo)
2035 {
2036     Z_Records *rec = (Z_Records *)
2037         odr_malloc (odr, sizeof(*rec));
2038     int *err = (int *)
2039         odr_malloc (odr, sizeof(*err));
2040     Z_DiagRec *drec = (Z_DiagRec *)
2041         odr_malloc (odr, sizeof(*drec));
2042     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2043         odr_malloc (odr, sizeof(*dr));
2044     *err = error;
2045     rec->which = Z_Records_NSD;
2046     rec->u.nonSurrogateDiagnostic = dr;
2047     dr->diagnosticSetId =
2048         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
2049     dr->condition = err;
2050     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2051     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2052     return rec;
2053 }
2054
2055 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2056 {
2057     if (apdu->which == Z_APDU_searchRequest &&
2058         apdu->u.searchRequest->query &&
2059         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2060         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2061     {
2062         Z_RPNQuery *rpnquery = 0;
2063         Z_SearchRequest *sr = apdu->u.searchRequest;
2064         char *addinfo = 0;
2065
2066         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2067                 sr->query->u.type_104->u.cql);
2068
2069         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2070                                           &rpnquery, odr_encode(),
2071                                           &addinfo);
2072         if (r == -3)
2073             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2074         else if (r)
2075         {
2076             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2077             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2078
2079             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2080             new_apdu->u.searchResponse->records =
2081                 create_nonSurrogateDiagnostics(odr_encode(),
2082                                                yaz_diag_srw_to_bib1(r),
2083                                                addinfo);
2084             *new_apdu->u.searchResponse->searchStatus = 0;
2085
2086             send_to_client(new_apdu);
2087
2088             return 0;
2089         }
2090         else
2091         {
2092             sr->query->which = Z_Query_type_1;
2093             sr->query->u.type_1 = rpnquery;
2094         }
2095         return apdu;
2096     }
2097     return apdu;
2098 }
2099
2100 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2101 {
2102     if (apdu->which == Z_APDU_searchRequest &&
2103         apdu->u.searchRequest->query)
2104     {
2105         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2106             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2107         {
2108             if (m_http_version)
2109                 m_charset_converter->set_client_query_charset("UTF-8");
2110             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2111             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2112         }
2113     }
2114     return apdu;
2115 }
2116
2117
2118 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2119 {
2120     if (apdu->which == Z_APDU_searchRequest)
2121     {
2122         Z_SearchRequest *sr = apdu->u.searchRequest;
2123         int err = 0;
2124         char *addinfo = 0;
2125
2126         Yaz_ProxyConfig *cfg = check_reconfigure();
2127         if (cfg)
2128             err = cfg->check_query(odr_encode(), m_default_target,
2129                                    sr->query, &addinfo);
2130         if (err)
2131         {
2132             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2133
2134             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2135             new_apdu->u.searchResponse->records =
2136                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2137             *new_apdu->u.searchResponse->searchStatus = 0;
2138
2139             send_to_client(new_apdu);
2140
2141             return 0;
2142         }
2143     }
2144     return apdu;
2145 }
2146
2147 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2148 {
2149     if (apdu->which != Z_APDU_initRequest)
2150         return 1;  // pass if no init request
2151     Z_InitRequest *req = apdu->u.initRequest;
2152
2153     Yaz_ProxyConfig *cfg = check_reconfigure();
2154     if (!cfg)
2155         return 1;  // pass if no config
2156
2157     int ret;
2158     if (req->idAuthentication == 0)
2159     {
2160         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2161                                          m_peername);
2162     }
2163     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2164     {
2165         ret = cfg->client_authentication(
2166             m_default_target,
2167             req->idAuthentication->u.idPass->userId,
2168             req->idAuthentication->u.idPass->groupId,
2169             req->idAuthentication->u.idPass->password,
2170             m_peername);
2171     }
2172     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2173     {
2174         char user[64], pass[64];
2175         *user = '\0';
2176         *pass = '\0';
2177         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2178         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2179                                          m_peername);
2180     }
2181     else
2182         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2183                                          m_peername);
2184
2185     cfg->target_authentication(m_default_target, odr_encode(), req);
2186
2187     return ret;
2188 }
2189
2190 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2191 {
2192     m_marcxml_mode = none;
2193     if (apdu->which == Z_APDU_searchRequest)
2194     {
2195         Z_SearchRequest *sr = apdu->u.searchRequest;
2196         int err = 0;
2197         char *addinfo = 0;
2198         Yaz_ProxyConfig *cfg = check_reconfigure();
2199
2200         Z_RecordComposition rc_temp, *rc = 0;
2201         if (sr->smallSetElementSetNames)
2202         {
2203             rc_temp.which = Z_RecordComp_simple;
2204             rc_temp.u.simple = sr->smallSetElementSetNames;
2205             rc = &rc_temp;
2206         }
2207
2208         if (sr->preferredRecordSyntax)
2209         {
2210             struct oident *ent;
2211             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2212             m_frontend_type = ent->value;
2213         }
2214         else
2215             m_frontend_type = VAL_NONE;
2216
2217         char *stylesheet_name = 0;
2218         if (cfg)
2219             err = cfg->check_syntax(odr_encode(),
2220                                     m_default_target,
2221                                     sr->preferredRecordSyntax, rc,
2222                                     &addinfo, &stylesheet_name, &m_schema,
2223                                     &m_backend_type, &m_backend_charset,
2224                                     &m_usemarcon_ini_stage1,
2225                                     &m_usemarcon_ini_stage2);
2226         if (stylesheet_name)
2227         {
2228             m_parent->low_socket_close();
2229
2230 #if HAVE_XSLT
2231             if (m_stylesheet_xsp)
2232                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2233             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2234                                                        stylesheet_name);
2235 #endif
2236             m_stylesheet_offset = 0;
2237             xfree(stylesheet_name);
2238
2239             m_parent->low_socket_open();
2240         }
2241         if (err == -1)
2242         {
2243             sr->smallSetElementSetNames = 0;
2244             sr->mediumSetElementSetNames = 0;
2245             m_marcxml_mode = marcxml;
2246             if (m_backend_type)
2247             {
2248
2249                 sr->preferredRecordSyntax =
2250                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2251                                         m_backend_type);
2252             }
2253             else
2254                 sr->preferredRecordSyntax =
2255                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2256                                            VAL_USMARC);
2257         }
2258         else if (err)
2259         {
2260             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2261
2262             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2263             new_apdu->u.searchResponse->records =
2264                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2265             *new_apdu->u.searchResponse->searchStatus = 0;
2266
2267             send_to_client(new_apdu);
2268
2269             return 0;
2270         }
2271         else if (m_backend_type)
2272         {
2273             sr->preferredRecordSyntax =
2274                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2275         }
2276     }
2277     else if (apdu->which == Z_APDU_presentRequest)
2278     {
2279         Z_PresentRequest *pr = apdu->u.presentRequest;
2280         int err = 0;
2281         char *addinfo = 0;
2282         Yaz_ProxyConfig *cfg = check_reconfigure();
2283
2284         if (pr->preferredRecordSyntax)
2285         {
2286             struct oident *ent;
2287             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2288             m_frontend_type = ent->value;
2289         }
2290         else
2291             m_frontend_type = VAL_NONE;
2292
2293         char *stylesheet_name = 0;
2294         if (cfg)
2295             err = cfg->check_syntax(odr_encode(), m_default_target,
2296                                     pr->preferredRecordSyntax,
2297                                     pr->recordComposition,
2298                                     &addinfo, &stylesheet_name, &m_schema,
2299                                     &m_backend_type, &m_backend_charset,
2300                                     &m_usemarcon_ini_stage1,
2301                                     &m_usemarcon_ini_stage2
2302                                     );
2303         if (stylesheet_name)
2304         {
2305             m_parent->low_socket_close();
2306
2307 #if HAVE_XSLT
2308             if (m_stylesheet_xsp)
2309                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2310             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2311                                                        stylesheet_name);
2312 #endif
2313             m_stylesheet_offset = 0;
2314             xfree(stylesheet_name);
2315
2316             m_parent->low_socket_open();
2317         }
2318         if (err == -1)
2319         {
2320             pr->recordComposition = 0;
2321             m_marcxml_mode = marcxml;
2322             if (m_backend_type)
2323             {
2324
2325                 pr->preferredRecordSyntax =
2326                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2327                                         m_backend_type);
2328             }
2329             else
2330                 pr->preferredRecordSyntax =
2331                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2332                                            VAL_USMARC);
2333         }
2334         else if (err)
2335         {
2336             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2337
2338             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2339             new_apdu->u.presentResponse->records =
2340                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2341             *new_apdu->u.presentResponse->presentStatus =
2342                 Z_PresentStatus_failure;
2343
2344             send_to_client(new_apdu);
2345
2346             return 0;
2347         }
2348         else if (m_backend_type)
2349         {
2350             pr->preferredRecordSyntax =
2351                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2352         }
2353     }
2354     return apdu;
2355 }
2356
2357 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2358 {
2359     if (!schema)
2360         return 0;
2361     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2362         odr_malloc(o, sizeof(Z_ElementSetNames));
2363     esn->which = Z_ElementSetNames_generic;
2364     esn->u.generic = odr_strdup(o, schema);
2365     return esn;
2366 }
2367
2368 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2369 {
2370     const char *t = 0;
2371     Yaz_ProxyConfig *cfg = check_reconfigure();
2372     if (cfg)
2373         t = cfg->get_explain_name(db, backend_db);
2374
2375     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2376     {
2377         releaseClient();
2378     }
2379
2380     if (t)
2381     {
2382         xfree(m_default_target);
2383         m_default_target = xstrdup(t);
2384     }
2385 }
2386
2387 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2388 {
2389     struct stat sbuf;
2390     if (strcmp(hreq->method, "GET"))
2391         return 0;
2392     if (hreq->path[0] != '/')
2393     {
2394         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2395         return 0;
2396     }
2397     const char *cp = hreq->path;
2398     while (*cp)
2399     {
2400         if (*cp == '/' && strchr("/.", cp[1]))
2401         {
2402             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2403             return 0;
2404         }
2405         cp++;
2406     }
2407     const char *fname = hreq->path+1;
2408     if (stat(fname, &sbuf))
2409     {
2410         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2411         return 0;
2412     }
2413     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2414     {
2415         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2416         return 0;
2417     }
2418     if (sbuf.st_size > (off_t) 1000000)
2419     {
2420         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2421         return 0;
2422     }
2423
2424     ODR o = odr_encode();
2425     Yaz_ProxyConfig *cfg = check_reconfigure();
2426     const char *ctype = cfg->check_mime_type(fname);
2427     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2428     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2429     if (m_http_version)
2430         hres->version = odr_strdup(o, m_http_version);
2431     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2432     if (m_http_keepalive)
2433         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2434     else
2435         timeout(0);
2436
2437     hres->content_len = sbuf.st_size;
2438     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2439     FILE *f = fopen(fname, "rb");
2440     if (f)
2441     {
2442         fread(hres->content_buf, 1, hres->content_len, f);
2443         fclose(f);
2444     }
2445     else
2446     {
2447         return 0;
2448     }
2449     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2450     {
2451         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2452                  fname);
2453     }
2454     int len;
2455     send_GDU(gdu, &len);
2456     return 1;
2457 }
2458
2459 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2460 {
2461     if (m_s2z_odr_init)
2462     {
2463         odr_destroy(m_s2z_odr_init);
2464         m_s2z_odr_init = 0;
2465     }
2466     if (m_s2z_odr_search)
2467     {
2468         odr_destroy(m_s2z_odr_search);
2469         m_s2z_odr_search = 0;
2470     }
2471
2472     m_http_keepalive = 0;
2473     m_http_version = 0;
2474     if (!strcmp(hreq->version, "1.0"))
2475     {
2476         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2477         if (v && !strcmp(v, "Keep-Alive"))
2478             m_http_keepalive = 1;
2479         else
2480             m_http_keepalive = 0;
2481         m_http_version = "1.0";
2482     }
2483     else
2484     {
2485         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2486         if (v && !strcmp(v, "close"))
2487             m_http_keepalive = 0;
2488         else
2489             m_http_keepalive = 1;
2490         m_http_version = "1.1";
2491     }
2492
2493     Z_SRW_PDU *srw_pdu = 0;
2494     Z_SOAP *soap_package = 0;
2495     char *charset = 0;
2496     Z_SRW_diagnostic *diagnostic = 0;
2497     int num_diagnostic = 0;
2498
2499     if (file_access(hreq))
2500     {
2501         return;
2502     }
2503     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2504                             &charset) == 0
2505              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2506                                &charset, &diagnostic, &num_diagnostic) == 0)
2507     {
2508         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2509         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2510         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2511         m_s2z_init_apdu = 0;
2512         m_s2z_search_apdu = 0;
2513         m_s2z_present_apdu = 0;
2514
2515         m_s2z_stylesheet = 0;
2516
2517         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2518         {
2519             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2520
2521             const char *backend_db = srw_req->database;
2522             srw_get_client(srw_req->database, &backend_db);
2523
2524             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2525             // recordXPath unsupported.
2526             if (srw_req->recordXPath)
2527             {
2528                 yaz_add_srw_diagnostic(odr_decode(),
2529                                        &diagnostic, &num_diagnostic,
2530                                        72, 0);
2531             }
2532             // sort unsupported
2533             if (srw_req->sort_type != Z_SRW_sort_type_none)
2534             {
2535                 yaz_add_srw_diagnostic(odr_decode(),
2536                                        &diagnostic, &num_diagnostic,
2537                                        80, 0);
2538             }
2539             // save stylesheet
2540             if (srw_req->stylesheet)
2541                 m_s2z_stylesheet =
2542                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2543
2544             // set packing for response records ..
2545             if (srw_req->recordPacking &&
2546                 !strcmp(srw_req->recordPacking, "xml"))
2547                 m_s2z_packing = Z_SRW_recordPacking_XML;
2548             else
2549                 m_s2z_packing = Z_SRW_recordPacking_string;
2550
2551             if (num_diagnostic)
2552             {
2553                 Z_SRW_PDU *srw_pdu =
2554                     yaz_srw_get(odr_encode(),
2555                                 Z_SRW_searchRetrieve_response);
2556                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2557
2558                 srw_res->diagnostics = diagnostic;
2559                 srw_res->num_diagnostics = num_diagnostic;
2560                 send_srw_response(srw_pdu);
2561                 return;
2562             }
2563
2564             // prepare search PDU
2565             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2566                                           Z_APDU_searchRequest);
2567             Z_SearchRequest *z_searchRequest =
2568                 m_s2z_search_apdu->u.searchRequest;
2569
2570             z_searchRequest->num_databaseNames = 1;
2571             z_searchRequest->databaseNames = (char**)
2572                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2573             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2574                                                            backend_db);
2575
2576             // query transformation
2577             Z_Query *query = (Z_Query *)
2578                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2579             z_searchRequest->query = query;
2580
2581             if (srw_req->query_type == Z_SRW_query_type_cql)
2582             {
2583                 Z_External *ext = (Z_External *)
2584                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2585                 ext->direct_reference =
2586                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2587                 ext->indirect_reference = 0;
2588                 ext->descriptor = 0;
2589                 ext->which = Z_External_CQL;
2590                 ext->u.cql = srw_req->query.cql;
2591
2592                 query->which = Z_Query_type_104;
2593                 query->u.type_104 =  ext;
2594             }
2595             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2596             {
2597                 Z_RPNQuery *RPNquery;
2598                 YAZ_PQF_Parser pqf_parser;
2599
2600                 pqf_parser = yaz_pqf_create ();
2601
2602                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2603                                           srw_req->query.pqf);
2604                 if (!RPNquery)
2605                 {
2606                     const char *pqf_msg;
2607                     size_t off;
2608                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2609                     yaz_log(YLOG_LOG, "%*s^\n", off+4, "");
2610                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2611
2612                     send_to_srw_client_error(10, 0);
2613                     return;
2614                 }
2615                 query->which = Z_Query_type_1;
2616                 query->u.type_1 =  RPNquery;
2617
2618                 yaz_pqf_destroy (pqf_parser);
2619             }
2620             else
2621             {
2622                 send_to_srw_client_error(7, "query");
2623                 return;
2624             }
2625
2626             // present
2627             m_s2z_present_apdu = 0;
2628             int max = 0;
2629             if (srw_req->maximumRecords)
2630                 max = *srw_req->maximumRecords;
2631             int start = 1;
2632             if (srw_req->startRecord)
2633                 start = *srw_req->startRecord;
2634             if (max > 0)
2635             {
2636                 // Some backend, such as Voyager doesn't honor piggyback
2637                 // So we use present always (0 &&).
2638                 if (0 && start <= 1)  // Z39.50 piggyback
2639                 {
2640                     *z_searchRequest->smallSetUpperBound = max;
2641                     *z_searchRequest->mediumSetPresentNumber = max;
2642                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2643
2644                     z_searchRequest->preferredRecordSyntax =
2645                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2646                                                VAL_TEXT_XML);
2647                     if (srw_req->recordSchema)
2648                     {
2649                         z_searchRequest->smallSetElementSetNames =
2650                             z_searchRequest->mediumSetElementSetNames =
2651                             mk_esn_from_schema(m_s2z_odr_search,
2652                                                srw_req->recordSchema);
2653                     }
2654                 }
2655                 else   // Z39.50 present
2656                 {
2657                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2658                                                    Z_APDU_presentRequest);
2659                     Z_PresentRequest *z_presentRequest =
2660                         m_s2z_present_apdu->u.presentRequest;
2661                     *z_presentRequest->resultSetStartPoint = start;
2662                     *z_presentRequest->numberOfRecordsRequested = max;
2663                     z_presentRequest->preferredRecordSyntax =
2664                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2665                                                VAL_TEXT_XML);
2666                     if (srw_req->recordSchema)
2667                     {
2668                         z_presentRequest->recordComposition =
2669                             (Z_RecordComposition *)
2670                             odr_malloc(m_s2z_odr_search,
2671                                        sizeof(Z_RecordComposition));
2672                         z_presentRequest->recordComposition->which =
2673                             Z_RecordComp_simple;
2674                         z_presentRequest->recordComposition->u.simple =
2675                             mk_esn_from_schema(m_s2z_odr_search,
2676                                                srw_req->recordSchema);
2677                     }
2678                 }
2679             }
2680             if (!m_client)
2681             {
2682                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2683                                             Z_APDU_initRequest);
2684
2685                 // prevent m_initRequest_apdu memory from being grabbed
2686                 // in Yaz_Proxy::handle_incoming_Z_PDU
2687                 m_initRequest_apdu = m_s2z_init_apdu;
2688                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2689                 return;
2690             }
2691             else
2692             {
2693                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2694                 return;
2695             }
2696         }
2697         else if (srw_pdu->which == Z_SRW_explain_request)
2698         {
2699             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2700
2701             const char *backend_db = srw_req->database;
2702             srw_get_client(srw_req->database, &backend_db);
2703
2704             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2705
2706             // save stylesheet
2707             if (srw_req->stylesheet)
2708                 m_s2z_stylesheet =
2709                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2710
2711             if (srw_req->recordPacking &&
2712                 !strcmp(srw_req->recordPacking, "xml"))
2713                 m_s2z_packing = Z_SRW_recordPacking_XML;
2714             else
2715                 m_s2z_packing = Z_SRW_recordPacking_string;
2716
2717             if (num_diagnostic)
2718             {
2719                 send_srw_explain_response(diagnostic, num_diagnostic);
2720                 return;
2721             }
2722
2723             if (!m_client)
2724             {
2725                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2726                                             Z_APDU_initRequest);
2727
2728                 // prevent m_initRequest_apdu memory from being grabbed
2729                 // in Yaz_Proxy::handle_incoming_Z_PDU
2730                 m_initRequest_apdu = m_s2z_init_apdu;
2731                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2732             }
2733             else
2734                 send_srw_explain_response(0, 0);
2735             return;
2736         }
2737         else if (srw_pdu->which == Z_SRW_scan_request)
2738         {
2739             m_s2z_database = odr_strdup(m_s2z_odr_init,
2740                                         srw_pdu->u.scan_request->database);
2741
2742             yaz_add_srw_diagnostic(odr_decode(),
2743                                    &diagnostic, &num_diagnostic,
2744                                    4, "scan");
2745             Z_SRW_PDU *srw_pdu =
2746                 yaz_srw_get(odr_encode(),
2747                             Z_SRW_scan_response);
2748             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2749
2750             srw_res->diagnostics = diagnostic;
2751             srw_res->num_diagnostics = num_diagnostic;
2752             send_srw_response(srw_pdu);
2753             return;
2754         }
2755         else
2756         {
2757             m_s2z_database = 0;
2758
2759             send_to_srw_client_error(4, 0);
2760         }
2761     }
2762     send_http_response(400);
2763 }
2764
2765 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2766 {
2767     Z_OtherInformation **oi;
2768     get_otherInfoAPDU(apdu, &oi);
2769
2770     if (apdu->u.initRequest->implementationId)
2771         yaz_log(YLOG_LOG, "%simplementationId: %s",
2772                 m_session_str, apdu->u.initRequest->implementationId);
2773     if (apdu->u.initRequest->implementationName)
2774         yaz_log(YLOG_LOG, "%simplementationName: %s",
2775                 m_session_str, apdu->u.initRequest->implementationName);
2776     if (apdu->u.initRequest->implementationVersion)
2777         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2778                 m_session_str, apdu->u.initRequest->implementationVersion);
2779     if (m_initRequest_apdu == 0)
2780     {
2781         if (m_initRequest_mem)
2782             nmem_destroy(m_initRequest_mem);
2783
2784         m_initRequest_apdu = apdu;
2785         m_initRequest_mem = odr_extract_mem(odr_decode());
2786
2787         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2788             preferredMessageSize;
2789         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2790         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2791             maximumRecordSize;
2792         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2793
2794         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2795             yaz_get_charneg_record(*oi);
2796
2797         // Save proposal charsets and langs.
2798         if (ODR_MASK_GET(apdu->u.initRequest->options,
2799                          Z_Options_negotiationModel)
2800             && charSetandLangRecord)
2801         {
2802
2803             yaz_get_proposal_charneg(m_referenceId_mem,
2804                                      charSetandLangRecord,
2805                                      &m_initRequest_oi_negotiation_charsets,
2806                                      &m_initRequest_oi_negotiation_num_charsets,
2807                                      &m_initRequest_oi_negotiation_langs,
2808                                      &m_initRequest_oi_negotiation_num_langs,
2809                                      &m_initRequest_oi_negotiation_selected);
2810
2811             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2812             {
2813                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2814                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2815                         m_initRequest_oi_negotiation_charsets[i]:"none");
2816             }
2817             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2818             {
2819                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2820                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2821                         m_initRequest_oi_negotiation_langs[i]:"none");
2822             }
2823             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2824                     m_session_str, m_initRequest_oi_negotiation_selected);
2825         }
2826         // save init options for the response..
2827         m_initRequest_options = apdu->u.initRequest->options;
2828
2829         apdu->u.initRequest->options =
2830             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2831                                        sizeof(Odr_bitmask));
2832         ODR_MASK_ZERO(apdu->u.initRequest->options);
2833         int i;
2834         for (i = 0; i<= 24; i++)
2835             ODR_MASK_SET(apdu->u.initRequest->options, i);
2836         // check negotiation option
2837         if (!ODR_MASK_GET(m_initRequest_options,
2838                           Z_Options_negotiationModel))
2839         {
2840             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2841                            Z_Options_negotiationModel);
2842         }
2843         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2844                        Z_Options_concurrentOperations);
2845         // make new version
2846         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2847         apdu->u.initRequest->protocolVersion =
2848             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2849                                        sizeof(Odr_bitmask));
2850         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2851
2852         for (i = 0; i<= 8; i++)
2853             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2854     }
2855     if (m_client->m_init_flag)
2856     {
2857         if (handle_init_response_for_invalid_session(apdu))
2858             return;
2859         if (m_client->m_initResponse)
2860         {
2861             Z_APDU *apdu2 = m_client->m_initResponse;
2862             apdu2->u.initResponse->otherInfo = 0;
2863             if (m_client->m_cookie && *m_client->m_cookie)
2864                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2865                                            m_client->m_cookie);
2866             apdu2->u.initResponse->referenceId =
2867                 apdu->u.initRequest->referenceId;
2868             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2869             apdu2->u.initResponse->protocolVersion =
2870                 m_client->m_initResponse_version;
2871
2872             send_to_client(apdu2);
2873             m_timeout_mode = timeout_normal;
2874             return;
2875         }
2876     }
2877     m_client->m_init_flag = 1;
2878
2879 #if USE_AUTH_MSG
2880     yaz_log(YLOG_LOG, "%suse_auth_msg", m_session_str);
2881     Auth_Msg *m = new Auth_Msg;
2882     m->m_proxy = this;
2883     z_APDU(odr_encode(), &apdu, 0, "encode");
2884     char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
2885     m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
2886     memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
2887     odr_reset(odr_encode());
2888
2889     inc_ref();
2890     m_my_thread->put(m);
2891 #else
2892     int ret = handle_authentication(apdu);
2893     result_authentication(apdu, ret);
2894 #endif
2895 }
2896
2897 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2898 {
2899     Z_ReferenceId **refid = get_referenceIdP(apdu);
2900     nmem_reset(m_referenceId_mem);
2901     if (refid && *refid)
2902     {
2903         m_referenceId = (Z_ReferenceId *)
2904             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2905         m_referenceId->len = m_referenceId->size = (*refid)->len;
2906         m_referenceId->buf = (unsigned char *)
2907             nmem_malloc(m_referenceId_mem, (*refid)->len);
2908         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2909     }
2910     else
2911         m_referenceId = 0;
2912
2913     if (!m_client && m_flag_invalid_session)
2914     {
2915         // Got request for a session that is invalid..
2916         m_apdu_invalid_session = apdu; // save package
2917         m_mem_invalid_session = odr_extract_mem(odr_decode());
2918         apdu = m_initRequest_apdu;     // but throw an init to the target
2919     }
2920
2921     // Determine our client.
2922     Z_OtherInformation **oi;
2923     get_otherInfoAPDU(apdu, &oi);
2924     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2925     if (!m_client)
2926     {
2927         if (m_http_version)
2928         {   // HTTP. Send not found
2929             send_http_response(404);
2930             return;
2931         }
2932         else
2933         {
2934             // Z39.50 just shutdown
2935             delete this;
2936             return;
2937         }
2938     }
2939
2940     m_client->m_server = this;
2941
2942     if (apdu->which == Z_APDU_initRequest)
2943         handle_init(apdu);
2944     else
2945         handle_incoming_Z_PDU_2(apdu);
2946 }
2947
2948 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
2949 {
2950     handle_max_record_retrieve(apdu);
2951
2952     if (apdu)
2953         apdu = handle_syntax_validation(apdu);
2954
2955     if (apdu)
2956         apdu = handle_query_transformation(apdu);
2957
2958     if (apdu)
2959         apdu = handle_target_charset_conversion(apdu);
2960
2961     if (apdu)
2962         apdu = handle_query_validation(apdu);
2963
2964     if (apdu)
2965         apdu = result_set_optimize(apdu);
2966
2967     if (!apdu)
2968     {
2969         m_client->timeout(m_target_idletime);  // mark it active even
2970         recv_GDU_more(true);
2971         // though we didn't use it
2972         return;
2973     }
2974     // Add otherInformation entry in APDU if
2975     // negotiation is in use.
2976     if (apdu)
2977         handle_charset_lang_negotiation(apdu);
2978
2979     // delete other info construct completely if 0 elements
2980     Z_OtherInformation **oi;
2981     get_otherInfoAPDU(apdu, &oi);
2982     if (oi && *oi && (*oi)->num_elements == 0)
2983         *oi = 0;
2984
2985     if (apdu->which == Z_APDU_presentRequest &&
2986         m_client->m_resultSetStartPoint == 0)
2987     {
2988         Z_PresentRequest *pr = apdu->u.presentRequest;
2989         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2990         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2991     } else {
2992         m_client->m_resultSetStartPoint = 0;
2993     }
2994     if (m_client->send_to_target(apdu) < 0)
2995     {
2996         m_client->shutdown();
2997     }
2998     else
2999         m_client->m_waiting = 1;
3000 }
3001
3002 void Yaz_Proxy::connectNotify()
3003 {
3004 }
3005
3006 void Yaz_Proxy::releaseClient()
3007 {
3008     xfree(m_proxyTarget);
3009     m_proxyTarget = 0;
3010     m_flag_invalid_session = 0;
3011     // only keep if keep_alive flag is set...
3012     if (m_client &&
3013         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3014         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3015         m_client->m_waiting == 0)
3016     {
3017         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3018                  m_session_str,
3019                  m_client->get_hostname());
3020         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3021                 m_session_str, m_client->m_pdu_recv,
3022                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3023                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3024         assert (m_client->m_waiting != 2);
3025         // Tell client (if any) that no server connection is there..
3026         m_client->m_server = 0;
3027         m_client = 0;
3028     }
3029     else if (m_client)
3030     {
3031         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
3032                  m_session_str,
3033                  m_client->get_hostname());
3034         assert (m_client->m_waiting != 2);
3035         delete m_client;
3036         m_client = 0;
3037     }
3038     else if (!m_parent)
3039     {
3040         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
3041                  m_session_str);
3042         assert (m_parent);
3043     }
3044     else
3045     {
3046         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
3047                  m_session_str);
3048     }
3049     if (m_parent)
3050         m_parent->pre_init();
3051 }
3052
3053 bool Yaz_Proxy::dec_ref()
3054 {
3055     yaz_log(YLOG_LOG, "%sdec_ref count=%d", m_session_str, m_ref_count);
3056     assert(m_ref_count > 0);
3057
3058     --m_ref_count;
3059     if (m_ref_count > 0)
3060         return false;
3061
3062     releaseClient();
3063
3064     delete this;
3065     return true;
3066 }
3067
3068 const char *Yaz_ProxyClient::get_session_str()
3069 {
3070     if (!m_server)
3071         return "0 ";
3072     return m_server->get_session_str();
3073 }
3074
3075 void Yaz_ProxyClient::shutdown()
3076 {
3077     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3078              get_hostname());
3079
3080     if (m_server)
3081     {
3082         m_waiting = 1;      // ensure it's released from Proxy in releaseClient
3083         m_server->dec_ref();
3084     }
3085     else
3086         delete this;
3087 }
3088
3089 void Yaz_Proxy::failNotify()
3090 {
3091     inc_request_no();
3092     yaz_log (YLOG_LOG, "%sConnection closed by client",
3093              get_session_str());
3094     dec_ref();
3095 }
3096
3097 void Yaz_ProxyClient::failNotify()
3098 {
3099     if (m_server)
3100         m_server->inc_request_no();
3101     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
3102              get_session_str(), get_hostname());
3103     shutdown();
3104 }
3105
3106 void Yaz_ProxyClient::connectNotify()
3107 {
3108     const char *s = get_session_str();
3109     const char *h = get_hostname();
3110     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3111              m_target_idletime);
3112     timeout(m_target_idletime);
3113     if (!m_server)
3114         pre_init_client();
3115 }
3116
3117 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3118                                               *the_PDU_Observable, int fd)
3119 {
3120     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3121 }
3122
3123 Yaz_ProxyClient::~Yaz_ProxyClient()
3124 {
3125     if (m_prev)
3126         *m_prev = m_next;
3127     if (m_next)
3128         m_next->m_prev = m_prev;
3129     m_waiting = 2;     // for debugging purposes only.
3130     odr_destroy(m_init_odr);
3131     delete m_last_query;
3132     xfree (m_last_resultSetId);
3133     xfree (m_cookie);
3134 }
3135
3136 void Yaz_ProxyClient::pre_init_client()
3137 {
3138     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3139     Z_InitRequest *req = apdu->u.initRequest;
3140
3141     int i;
3142     for (i = 0; i<= 24; i++)
3143         ODR_MASK_SET(req->options, i);
3144     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3145                    Z_Options_negotiationModel);
3146     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3147                    Z_Options_concurrentOperations);
3148     for (i = 0; i<= 10; i++)
3149         ODR_MASK_SET(req->protocolVersion, i);
3150
3151     if (send_to_target(apdu) < 0)
3152     {
3153         delete this;
3154     }
3155     else
3156     {
3157         m_waiting = 1;
3158         m_init_flag = 1;
3159     }
3160 }
3161
3162 void Yaz_Proxy::pre_init()
3163 {
3164     int i;
3165     const char *name = 0;
3166     const char *zurl_in_use[MAX_ZURL_PLEX];
3167     int limit_bw, limit_pdu, limit_req, limit_search, limit_connect;
3168     int target_idletime, client_idletime;
3169     int max_clients;
3170     int keepalive_limit_bw, keepalive_limit_pdu;
3171     int pre_init;
3172     const char *cql2rpn = 0;
3173     const char *authentication = 0;
3174     const char *negotiation_charset = 0;
3175     const char *negotiation_lang = 0;
3176
3177     Yaz_ProxyConfig *cfg = check_reconfigure();
3178
3179     zurl_in_use[0] = 0;
3180
3181     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3182         set_APDU_yazlog(1);
3183     else
3184         set_APDU_yazlog(0);
3185
3186     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3187                                           &limit_bw, &limit_pdu, &limit_req,
3188                                           &limit_search, &limit_connect,
3189                                           &target_idletime, &client_idletime,
3190                                           &max_clients,
3191                                           &keepalive_limit_bw,
3192                                           &keepalive_limit_pdu,
3193                                           &pre_init,
3194                                           &cql2rpn,
3195                                           &authentication,
3196                                           &negotiation_charset,
3197                                           &negotiation_lang,
3198                                           0) ; i++)
3199     {
3200         if (pre_init)
3201         {
3202             int j;
3203             for (j = 0; zurl_in_use[j]; j++)
3204             {
3205                 Yaz_ProxyClient *c;
3206                 int spare = 0;
3207                 int spare_waiting = 0;
3208                 int in_use = 0;
3209                 int other = 0;
3210                 for (c = m_clientPool; c; c = c->m_next)
3211                 {
3212                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3213                     {
3214                         if (c->m_cookie == 0)
3215                         {
3216                             if (c->m_server == 0)
3217                                 if (c->m_waiting)
3218                                     spare_waiting++;
3219                                 else
3220                                     spare++;
3221                             else
3222                                 in_use++;
3223                         }
3224                         else
3225                             other++;
3226                     }
3227                 }
3228                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3229                         "sparew=%d preinit=%d",m_session_str,
3230                         name, zurl_in_use[j], in_use, other,
3231                         spare, spare_waiting, pre_init);
3232                 if (spare + spare_waiting < pre_init)
3233                 {
3234                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3235                     c->m_next = m_clientPool;
3236                     if (c->m_next)
3237                         c->m_next->m_prev = &c->m_next;
3238                     m_clientPool = c;
3239                     c->m_prev = &m_clientPool;
3240
3241                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3242                         c->set_APDU_yazlog(1);
3243                     else
3244                         c->set_APDU_yazlog(0);
3245
3246                     if (c->client(zurl_in_use[j]))
3247                     {
3248                         timeout(60);
3249                         delete c;
3250                         return;
3251                     }
3252                     c->timeout(30);
3253                     c->m_waiting = 1;
3254                     c->m_target_idletime = target_idletime;
3255                     c->m_seqno = m_seqno++;
3256                 }
3257             }
3258         }
3259     }
3260 }
3261
3262 void Yaz_Proxy::timeoutNotify()
3263 {
3264     if (m_parent)
3265     {
3266         GDU *gdu;
3267         switch(m_timeout_mode)
3268         {
3269         case timeout_normal:
3270         case timeout_busy:
3271             inc_request_no();
3272             m_in_queue.clear();
3273             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3274             dec_ref();
3275             break;
3276         case timeout_reduce:
3277             timeout(m_client_idletime);
3278             m_timeout_mode = timeout_busy;
3279             gdu = m_timeout_gdu;
3280             m_timeout_gdu = 0;
3281             recv_GDU_normal(gdu);
3282             break;
3283         case timeout_xsl:
3284             assert(m_stylesheet_nprl);
3285             convert_xsl_delay();
3286             recv_GDU_more(true);
3287         }
3288     }
3289     else
3290     {
3291         timeout(600);
3292         pre_init();
3293     }
3294 }
3295
3296 void Yaz_Proxy::markInvalid()
3297 {
3298     m_client = 0;
3299     m_flag_invalid_session = 1;
3300 }
3301
3302 void Yaz_ProxyClient::timeoutNotify()
3303 {
3304     if (m_server)
3305         m_server->inc_request_no();
3306
3307     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3308              get_hostname());
3309     m_waiting = 1;
3310     m_root->pre_init();
3311     if (m_server && m_init_flag)
3312     {
3313         // target timed out in a session that was properly initialized
3314         // server object stay alive but we mark it as invalid so it
3315         // gets initialized again
3316         m_server->markInvalid();
3317         m_server = 0;
3318     }
3319     shutdown();
3320 }
3321
3322 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
3323                                  Yaz_Proxy *parent) :
3324     Z_Assoc (the_PDU_Observable)
3325 {
3326     m_cookie = 0;
3327     m_next = 0;
3328     m_prev = 0;
3329     m_init_flag = 0;
3330     m_last_query = 0;
3331     m_last_resultSetId = 0;
3332     m_last_resultCount = 0;
3333     m_last_ok = 0;
3334     m_sr_transform = 0;
3335     m_waiting = 0;
3336     m_init_odr = odr_createmem (ODR_DECODE);
3337     m_initResponse = 0;
3338     m_initResponse_options = 0;
3339     m_initResponse_version = 0;
3340     m_initResponse_preferredMessageSize = 0;
3341     m_initResponse_maximumRecordSize = 0;
3342     m_resultSetStartPoint = 0;
3343     m_bytes_sent = m_bytes_recv = 0;
3344     m_pdu_recv = 0;
3345     m_server = 0;
3346     m_seqno = 0;
3347     m_target_idletime = 600;
3348     m_root = parent;
3349 }
3350
3351 const char *Yaz_Proxy::option(const char *name, const char *value)
3352 {
3353     if (!strcmp (name, "optimize")) {
3354         if (value) {
3355             xfree (m_optimize);
3356             m_optimize = xstrdup (value);
3357         }
3358         return m_optimize;
3359     }
3360     return 0;
3361 }
3362
3363 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
3364 {
3365
3366 }
3367
3368 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
3369 {
3370     if (apdu->which == Z_GDU_Z3950)
3371         recv_Z_PDU(apdu->u.z3950, len);
3372     else if (apdu->which == Z_GDU_HTTP_Response)
3373         recv_HTTP_response(apdu->u.HTTP_Response, len);
3374     else
3375         shutdown();
3376 }
3377
3378 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
3379 {
3380     if (!m_flag_invalid_session)
3381         return 0;
3382     m_flag_invalid_session = 0;
3383     handle_incoming_Z_PDU(m_apdu_invalid_session);
3384     assert (m_mem_invalid_session);
3385     nmem_destroy(m_mem_invalid_session);
3386     m_mem_invalid_session = 0;
3387     return 1;
3388 }
3389
3390 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
3391 {
3392     m_bytes_recv += len;
3393
3394     m_pdu_recv++;
3395     m_waiting = 0;
3396     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
3397         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
3398                  apdu_name(apdu), get_hostname(), len);
3399     if (apdu->which == Z_APDU_initResponse)
3400     {
3401         if (!m_server)  // if this is a pre init session , check for more
3402             m_root->pre_init();
3403         NMEM nmem = odr_extract_mem (odr_decode());
3404         odr_reset (m_init_odr);
3405         nmem_transfer (m_init_odr->mem, nmem);
3406         m_initResponse = apdu;
3407         m_initResponse_options = apdu->u.initResponse->options;
3408         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3409         m_initResponse_preferredMessageSize =
3410             *apdu->u.initResponse->preferredMessageSize;
3411         m_initResponse_maximumRecordSize =
3412             *apdu->u.initResponse->maximumRecordSize;
3413
3414         Z_InitResponse *ir = apdu->u.initResponse;
3415         char *im0 = ir->implementationName;
3416
3417         char *im1 = (char*)
3418             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3419         *im1 = '\0';
3420         if (im0)
3421         {
3422             strcat(im1, im0);
3423             strcat(im1, " ");
3424         }
3425         strcat(im1, "(YAZ Proxy)");
3426         ir->implementationName = im1;
3427
3428         nmem_destroy (nmem);
3429
3430         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3431             return;
3432     }
3433     if (apdu->which == Z_APDU_searchResponse)
3434     {
3435         Z_SearchResponse *sr = apdu->u.searchResponse;
3436         m_last_resultCount = *sr->resultCount;
3437         int status = *sr->searchStatus;
3438         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3439         {
3440             m_last_ok = 1;
3441
3442             if (sr->records && sr->records->which == Z_Records_DBOSD)
3443             {
3444                 m_cache.add(odr_decode(),
3445                             sr->records->u.databaseOrSurDiagnostics, 1,
3446                             *sr->resultCount);
3447             }
3448         }
3449     }
3450     if (apdu->which == Z_APDU_presentResponse)
3451     {
3452         Z_PresentResponse *pr = apdu->u.presentResponse;
3453         if (m_sr_transform)
3454         {
3455             m_sr_transform = 0;
3456             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3457             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3458             sr->referenceId = pr->referenceId;
3459             *sr->resultCount = m_last_resultCount;
3460             sr->records = pr->records;
3461             sr->nextResultSetPosition = pr->nextResultSetPosition;
3462             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3463             apdu = new_apdu;
3464         }
3465         if (pr->records &&
3466             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3467         {
3468             m_cache.add(odr_decode(),
3469                         pr->records->u.databaseOrSurDiagnostics,
3470                         m_resultSetStartPoint, -1);
3471             m_resultSetStartPoint = 0;
3472         }
3473     }
3474     if (m_cookie)
3475         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3476
3477     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
3478
3479     if (server)
3480         server->send_to_client(apdu);
3481     if (apdu->which == Z_APDU_close)
3482         shutdown();
3483     else if (server)
3484         server->recv_GDU_more(true);
3485 }
3486
3487 void Yaz_Proxy::low_socket_close()
3488 {
3489 #if WIN32
3490 #else
3491     int i;
3492     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3493         if  (m_lo_fd[i] >= 0)
3494             ::close(m_lo_fd[i]);
3495 #endif
3496 }
3497
3498 void Yaz_Proxy::low_socket_open()
3499 {
3500 #if WIN32
3501 #else
3502     int i;
3503     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3504         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3505 #endif
3506 }
3507
3508 int Yaz_Proxy::server(const char *addr)
3509 {
3510     int r = Z_Assoc::server(addr);
3511     if (!r)
3512     {
3513         yaz_log(YLOG_LOG, "%sStarted proxy "
3514 #ifdef VERSION
3515             VERSION
3516 #endif
3517             " on %s", m_session_str, addr);
3518         timeout(1);
3519     }
3520     return r;
3521 }
3522
3523 /*
3524  * Local variables:
3525  * c-basic-offset: 4
3526  * indent-tabs-mode: nil
3527  * End:
3528  * vim: shiftwidth=4 tabstop=8 expandtab
3529  */
3530