src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.39 2005-11-25 19:37:28 adam Exp $
   2    Copyright (c) 1998-2005, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <stdlib.h>
  42 #include <time.h>
  43 #include <fcntl.h>
  44
  45 #include <yaz/srw.h>
  46 #include <yaz/marcdisp.h>
  47 #include <yaz/yaz-iconv.h>
  48 #include <yaz/log.h>
  49 #include <yaz/diagbib1.h>
  50 #include "proxyp.h"
  51 #include <yaz/pquery.h>
  52 #include <yaz/otherinfo.h>
  53 #include <yaz/charneg.h>
  54 #include "msg-thread.h"
  55
  56 using namespace yazpp_1;
  57
  58 #define USE_AUTH_MSG 1
  59
  60 #if USE_AUTH_MSG
  61 class Auth_Msg : public IMsg_Thread {
  62 public:
  63     int m_ret;
  64     IMsg_Thread *handle();
  65     void result();
  66     Yaz_Proxy *m_proxy;
  67     NMEM m_nmem;
  68     char *m_apdu_buf;
  69     int m_apdu_len;
  70     Auth_Msg();
  71     virtual ~Auth_Msg();
  72 };
  73
  74 Auth_Msg::Auth_Msg()
  75 {
  76     m_nmem = nmem_create();
  77 }
  78
  79 Auth_Msg::~Auth_Msg()
  80 {
  81     nmem_destroy(m_nmem);
  82 }
  83
  84 IMsg_Thread *Auth_Msg::handle()
  85 {
  86     yaz_log(YLOG_LOG, "Auth_Msg:handle begin");
  87     ODR decode = odr_createmem(ODR_DECODE);
  88     Z_APDU *apdu;
  89
  90     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
  91     int r = z_APDU(decode, &apdu, 0, 0);
  92     if (!r)
  93     {
  94         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
  95     }
  96     else
  97     {
  98         m_ret = m_proxy->handle_authentication(apdu);
  99     }
 100     yaz_log(YLOG_LOG, "Auth_Msg:handle end");
 101     odr_destroy(decode);
 102     return this;
 103 }
 104
 105 void Auth_Msg::result()
 106 {
 107     yaz_log(YLOG_LOG, "Auth_Msg:result proxy ok buf=%p len=%d",
 108             m_apdu_buf, m_apdu_len);
 109     odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
 110     Z_APDU *apdu = 0;
 111     int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
 112     if (r)
 113         yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU OK");
 114     else
 115         yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
 116     if (m_proxy->dec_ref())
 117         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
 118     else
 119     {
 120         yaz_log(YLOG_LOG, "Auth_Msg::proxy still alive");
 121         m_proxy->result_authentication(apdu, m_ret);
 122     }
 123     delete this;
 124 }
 125
 126 #endif
 127
 128 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
 129 {
 130     if (apdu == 0 || ret == 0)
 131     {
 132         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
 133         *apdu_reject->u.initResponse->result = 0;
 134         send_to_client(apdu_reject);
 135         dec_ref();
 136     }
 137     else
 138         handle_incoming_Z_PDU_2(apdu);
 139 }
 140
 141 static const char *apdu_name(Z_APDU *apdu)
 142 {
 143     switch (apdu->which)
 144     {
 145     case Z_APDU_initRequest:
 146         return "initRequest";
 147     case Z_APDU_initResponse:
 148         return "initResponse";
 149     case Z_APDU_searchRequest:
 150         return "searchRequest";
 151     case Z_APDU_searchResponse:
 152         return "searchResponse";
 153     case Z_APDU_presentRequest:
 154         return "presentRequest";
 155     case Z_APDU_presentResponse:
 156         return "presentResponse";
 157     case Z_APDU_deleteResultSetRequest:
 158         return "deleteResultSetRequest";
 159     case Z_APDU_deleteResultSetResponse:
 160         return "deleteResultSetResponse";
 161     case Z_APDU_scanRequest:
 162         return "scanRequest";
 163     case Z_APDU_scanResponse:
 164         return "scanResponse";
 165     case Z_APDU_sortRequest:
 166         return "sortRequest";
 167     case Z_APDU_sortResponse:
 168         return "sortResponse";
 169     case Z_APDU_extendedServicesRequest:
 170         return "extendedServicesRequest";
 171     case Z_APDU_extendedServicesResponse:
 172         return "extendedServicesResponse";
 173     case Z_APDU_close:
 174         return "close";
 175     }
 176     return "other";
 177 }
 178
 179 static const char *gdu_name(Z_GDU *gdu)
 180 {
 181     switch(gdu->which)
 182     {
 183     case Z_GDU_Z3950:
 184         return apdu_name(gdu->u.z3950);
 185     case Z_GDU_HTTP_Request:
 186         return "HTTP Request";
 187     case Z_GDU_HTTP_Response:
 188         return "HTTP Response";
 189     }
 190     return "Unknown request/response";
 191 }
 192
 193 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
 194                      ISocketObservable *the_socket_observable,
 195                      Yaz_Proxy *parent)
 196     :
 197     Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 198 {
 199     m_PDU_Observable = the_PDU_Observable;
 200     m_socket_observable = the_socket_observable;
 201     m_client = 0;
 202     m_parent = parent;
 203     m_clientPool = 0;
 204     m_seqno = 1;
 205     m_keepalive_limit_bw = 500000;
 206     m_keepalive_limit_pdu = 1000;
 207     m_proxyTarget = 0;
 208     m_default_target = 0;
 209     m_proxy_negotiation_charset = 0;
 210     m_proxy_negotiation_lang = 0;
 211     m_charset_converter = new Yaz_CharsetConverter;
 212     m_max_clients = 150;
 213     m_log_mask = 0;
 214     m_seed = time(0);
 215     m_client_idletime = 600;
 216     m_target_idletime = 600;
 217     m_optimize = xstrdup ("1");
 218     strcpy(m_session_str, "0 ");
 219     m_session_no = 0;
 220     m_bytes_sent = 0;
 221     m_bytes_recv = 0;
 222     m_bw_max = 0;
 223     m_pdu_max = 0;
 224     m_search_max = 0;
 225     m_connect_max = 0;
 226     m_timeout_mode = timeout_normal;
 227     m_timeout_gdu = 0;
 228     m_max_record_retrieve = 0;
 229     m_reconfig_flag = 0;
 230     m_config_fname = 0;
 231     m_request_no = 0;
 232     m_flag_invalid_session = 0;
 233     m_referenceId = 0;
 234     m_referenceId_mem = nmem_create();
 235     m_config = 0;
 236     m_marcxml_mode = none;
 237     m_stylesheet_xsp = 0;
 238     m_stylesheet_nprl = 0;
 239     m_stylesheet_apdu = 0;
 240     m_s2z_stylesheet = 0;
 241     m_s2z_database = 0;
 242     m_schema = 0;
 243     m_backend_type = 0;
 244     m_backend_charset = 0;
 245     m_frontend_type = 0;
 246     m_initRequest_apdu = 0;
 247     m_initRequest_mem = 0;
 248     m_initRequest_preferredMessageSize = 0;
 249     m_initRequest_maximumRecordSize = 0;
 250     m_initRequest_options = 0;
 251     m_initRequest_version = 0;
 252     m_initRequest_oi_negotiation_charsets = 0;
 253     m_initRequest_oi_negotiation_num_charsets = 0;
 254     m_initRequest_oi_negotiation_langs = 0;
 255     m_initRequest_oi_negotiation_num_langs = 0;
 256     m_initRequest_oi_negotiation_selected = 0;
 257     m_apdu_invalid_session = 0;
 258     m_mem_invalid_session = 0;
 259     m_s2z_odr_init = 0;
 260     m_s2z_odr_search = 0;
 261     m_s2z_init_apdu = 0;
 262     m_s2z_search_apdu = 0;
 263     m_s2z_present_apdu = 0;
 264     m_http_keepalive = 0;
 265     m_http_version = 0;
 266     m_soap_ns = 0;
 267     m_s2z_packing = Z_SRW_recordPacking_string;
 268 #if HAVE_GETTIMEOFDAY
 269     m_time_tv = xmalloc(sizeof(struct timeval));
 270     struct timeval *tv = (struct timeval *) m_time_tv;
 271     tv->tv_sec = 0;
 272     tv->tv_usec = 0;
 273 #else
 274     m_time_tv = 0;
 275 #endif
 276     m_usemarcon_ini_stage1 = 0;
 277     m_usemarcon_ini_stage2 = 0;
 278     m_usemarcon = new Yaz_usemarcon();
 279     if (!m_parent)
 280         low_socket_open();
 281     m_my_thread = 0;
 282     m_ref_count = 1;
 283     m_peername = 0;
 284 }
 285
 286 void Yaz_Proxy::inc_ref()
 287 {
 288     m_ref_count++;
 289 }
 290
 291 Yaz_Proxy::~Yaz_Proxy()
 292 {
 293     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 294             m_bytes_sent, m_bytes_recv);
 295     nmem_destroy(m_initRequest_mem);
 296     nmem_destroy(m_mem_invalid_session);
 297     nmem_destroy(m_referenceId_mem);
 298
 299     xfree(m_proxyTarget);
 300     xfree(m_default_target);
 301     xfree(m_proxy_negotiation_charset);
 302     xfree(m_proxy_negotiation_lang);
 303     delete m_charset_converter;
 304     xfree(m_optimize);
 305
 306 #if HAVE_XSLT
 307     if (m_stylesheet_xsp)
 308         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 309 #endif
 310     xfree (m_time_tv);
 311
 312     xfree (m_peername);
 313     xfree (m_schema);
 314     xfree (m_backend_type);
 315     xfree (m_backend_charset);
 316     xfree (m_usemarcon_ini_stage1);
 317     xfree (m_usemarcon_ini_stage2);
 318     delete m_usemarcon;
 319     if (m_s2z_odr_init)
 320         odr_destroy(m_s2z_odr_init);
 321     if (m_s2z_odr_search)
 322         odr_destroy(m_s2z_odr_search);
 323     if (!m_parent)
 324         low_socket_close();
 325     if (!m_parent)
 326         delete m_my_thread;
 327     delete m_config;
 328 }
 329
 330 void Yaz_Proxy::set_debug_mode(int mode)
 331 {
 332     m_debug_mode = mode;
 333 }
 334
 335 int Yaz_Proxy::set_config(const char *config)
 336 {
 337     delete m_config;
 338     m_config = new Yaz_ProxyConfig();
 339     xfree(m_config_fname);
 340     m_config_fname = xstrdup(config);
 341     int r = m_config->read_xml(config);
 342     if (!r)
 343         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 344     return r;
 345 }
 346
 347 void Yaz_Proxy::set_default_target(const char *target)
 348 {
 349     xfree (m_default_target);
 350     m_default_target = 0;
 351     if (target)
 352         m_default_target = (char *) xstrdup (target);
 353 }
 354
 355 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang)
 356 {
 357     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
 358         "language to '%s'", m_session_str, charset?charset:"none",
 359         lang?lang:"none");
 360     xfree (m_proxy_negotiation_charset);
 361     xfree (m_proxy_negotiation_lang);
 362     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 363     if (charset)
 364         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 365     if (lang)
 366         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 367 }
 368
 369 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 370 {
 371     if (m_parent)
 372         return m_parent->check_reconfigure();
 373
 374     Yaz_ProxyConfig *cfg = m_config;
 375     if (m_reconfig_flag)
 376     {
 377         yaz_log(YLOG_LOG, "reconfigure");
 378         yaz_log_reopen();
 379         if (m_config_fname && cfg)
 380         {
 381             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 382             int r = cfg->read_xml(m_config_fname);
 383             if (r)
 384                 yaz_log(YLOG_WARN, "reconfigure failed");
 385             else
 386             {
 387                 m_log_mask = 0;
 388                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 389             }
 390         }
 391         else
 392             yaz_log(YLOG_LOG, "reconfigure");
 393         m_reconfig_flag = 0;
 394     }
 395     return cfg;
 396 }
 397
 398 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 399                                         *the_PDU_Observable, int fd)
 400 {
 401     check_reconfigure();
 402     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 403                                          m_socket_observable, this);
 404     new_proxy->m_config = 0;
 405     new_proxy->m_config_fname = 0;
 406     new_proxy->timeout(m_client_idletime);
 407     new_proxy->m_target_idletime = m_target_idletime;
 408     new_proxy->set_default_target(m_default_target);
 409     new_proxy->m_max_clients = m_max_clients;
 410     new_proxy->m_log_mask = m_log_mask;
 411     new_proxy->set_APDU_log(get_APDU_log());
 412     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 413         new_proxy->set_APDU_yazlog(1);
 414     else
 415         new_proxy->set_APDU_yazlog(0);
 416     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 417     m_session_no++;
 418     new_proxy->m_peername = xstrdup(the_PDU_Observable->getpeername());
 419     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 420              new_proxy->m_peername);
 421     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 422         m_proxy_negotiation_lang);
 423     // create thread object the first time we get an incoming connection
 424     if (!m_my_thread)
 425         m_my_thread = new Msg_Thread(m_socket_observable, 1);
 426     new_proxy->m_my_thread = m_my_thread;
 427     return new_proxy;
 428 }
 429
 430 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 431 {
 432     int oid[OID_SIZE];
 433     Z_OtherInformationUnit *oi;
 434     struct oident ent;
 435     ent.proto = PROTO_Z3950;
 436     ent.oclass = CLASS_USERINFO;
 437     ent.value = (oid_value) VAL_COOKIE;
 438     assert (oid_ent_to_oid (&ent, oid));
 439
 440     if (oid_ent_to_oid (&ent, oid) &&
 441         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 442         oi->which == Z_OtherInfo_characterInfo)
 443         return oi->information.characterInfo;
 444     return 0;
 445 }
 446 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 447 {
 448     int oid[OID_SIZE];
 449     Z_OtherInformationUnit *oi;
 450     struct oident ent;
 451     ent.proto = PROTO_Z3950;
 452     ent.oclass = CLASS_USERINFO;
 453     ent.value = (oid_value) VAL_PROXY;
 454     if (oid_ent_to_oid (&ent, oid) &&
 455         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 456         oi->which == Z_OtherInfo_characterInfo)
 457         return oi->information.characterInfo;
 458     return 0;
 459 }
 460 const char *Yaz_Proxy::load_balance(const char **url)
 461 {
 462     int zurl_in_use[MAX_ZURL_PLEX];
 463     int zurl_in_spare[MAX_ZURL_PLEX];
 464     Yaz_ProxyClient *c;
 465     int i;
 466
 467     for (i = 0; i<MAX_ZURL_PLEX; i++)
 468     {
 469         zurl_in_use[i] = 0;
 470         zurl_in_spare[i] = 0;
 471     }
 472     for (c = m_parent->m_clientPool; c; c = c->m_next)
 473     {
 474         for (i = 0; url[i]; i++)
 475             if (!strcmp(url[i], c->get_hostname()))
 476             {
 477                 zurl_in_use[i]++;
 478                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 479                     zurl_in_spare[i]++;
 480             }
 481     }
 482     int min_use = 100000;
 483     int spare_for_min = 0;
 484     int max_spare = 0;
 485     const char *ret_min = 0;
 486     const char *ret_spare = 0;
 487     for (i = 0; url[i]; i++)
 488     {
 489         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 490                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 491         if (min_use > zurl_in_use[i])
 492         {
 493             ret_min = url[i];
 494             min_use = zurl_in_use[i];
 495             spare_for_min = zurl_in_spare[i];
 496         }
 497         if (max_spare < zurl_in_spare[i])
 498         {
 499             ret_spare = url[i];
 500             max_spare = zurl_in_spare[i];
 501         }
 502     }
 503     return ret_min;
 504 }
 505
 506 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 507                                        const char *proxy_host)
 508 {
 509     assert (m_parent);
 510     Yaz_Proxy *parent = m_parent;
 511     Yaz_ProxyClient *c = m_client;
 512
 513     if (!m_proxyTarget)
 514     {
 515         const char *url[MAX_ZURL_PLEX];
 516         Yaz_ProxyConfig *cfg = check_reconfigure();
 517         if (proxy_host)
 518         {
 519             if (parent && parent->m_debug_mode)
 520             {
 521                 // only to be enabled for debugging...
 522                 if (!strcmp(proxy_host, "stop"))
 523                     exit(0);
 524             }
 525             xfree(m_default_target);
 526             m_default_target = xstrdup(proxy_host);
 527         }
 528         proxy_host = m_default_target;
 529         int client_idletime = -1;
 530         const char *cql2rpn_fname = 0;
 531         const char *negotiation_charset = 0;
 532         const char *negotiation_lang = 0;
 533         const char *query_charset = 0;
 534         url[0] = m_default_target;
 535         url[1] = 0;
 536         if (cfg)
 537         {
 538             int pre_init = 0;
 539             cfg->get_target_info(proxy_host, url, &m_bw_max,
 540                                  &m_pdu_max, &m_max_record_retrieve,
 541                                  &m_search_max, &m_connect_max,
 542                                  &m_target_idletime, &client_idletime,
 543                                  &parent->m_max_clients,
 544                                  &m_keepalive_limit_bw,
 545                                  &m_keepalive_limit_pdu,
 546                                  &pre_init,
 547                                  &cql2rpn_fname,
 548                                  &negotiation_charset,
 549                                  &negotiation_lang,
 550                                  &query_charset);
 551         }
 552         if (client_idletime != -1)
 553         {
 554             m_client_idletime = client_idletime;
 555             timeout(m_client_idletime);
 556         }
 557         if (cql2rpn_fname)
 558             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 559         if (negotiation_charset || negotiation_lang)
 560         {
 561             set_proxy_negotiation(negotiation_charset,
 562                 negotiation_lang);
 563         }
 564         m_charset_converter->set_target_query_charset(query_charset);
 565         if (!url[0])
 566         {
 567             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 568             return 0;
 569         }
 570         // we don't handle multiplexing for cookie session, so we just
 571         // pick the first one in this case (anonymous users will be able
 572         // to use any backend)
 573         if (cookie && *cookie)
 574             m_proxyTarget = (char*) xstrdup(url[0]);
 575         else
 576             m_proxyTarget = (char*) xstrdup(load_balance(url));
 577     }
 578     if (cookie && *cookie)
 579     {   // search in sessions with a cookie
 580         for (c = parent->m_clientPool; c; c = c->m_next)
 581         {
 582             assert (c->m_prev);
 583             assert (*c->m_prev == c);
 584             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 585                 !strcmp(m_proxyTarget, c->get_hostname()))
 586             {
 587                 // Found it in cache
 588                 // The following handles "cancel"
 589                 // If connection is busy (waiting for PDU) and
 590                 // we have an initRequest we can safely do re-open
 591                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 592                 {
 593                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 594                              c->get_hostname());
 595                     c->close();
 596                     c->m_init_flag = 0;
 597
 598                     c->m_last_ok = 0;
 599                     c->m_cache.clear();
 600                     c->m_last_resultCount = 0;
 601                     c->m_sr_transform = 0;
 602                     c->m_waiting = 0;
 603                     c->m_resultSetStartPoint = 0;
 604                     c->m_target_idletime = m_target_idletime;
 605                     if (c->client(m_proxyTarget))
 606                     {
 607                         delete c;
 608                         return 0;
 609                     }
 610                     c->timeout(30);
 611                 }
 612                 c->m_seqno = parent->m_seqno;
 613                 if (c->m_server && c->m_server != this)
 614                     c->m_server->m_client = 0;
 615                 c->m_server = this;
 616                 (parent->m_seqno)++;
 617                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 618                 return c;
 619             }
 620         }
 621     }
 622     else if (!c &&
 623             apdu->which == Z_APDU_initRequest &&
 624             apdu->u.initRequest->idAuthentication == 0 &&
 625             !ODR_MASK_GET(apdu->u.initRequest->options, Z_Options_negotiationModel))
 626     {
 627         // anonymous sessions without cookie.
 628         // if authentication is set it is NOT anonymous se we can't share them.
 629         // If charset and lang negotiation is use it is NOT anonymous session too.
 630         for (c = parent->m_clientPool; c; c = c->m_next)
 631         {
 632             assert(c->m_prev);
 633             assert(*c->m_prev == c);
 634             if (c->m_server == 0 && c->m_cookie == 0 &&
 635                 c->m_waiting == 0 &&
 636                 !strcmp(m_proxyTarget, c->get_hostname()))
 637             {
 638                 // found it in cache
 639                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 640                          m_session_str, parent->m_seqno, c->get_hostname());
 641
 642                 c->m_seqno = parent->m_seqno;
 643                 assert(c->m_server == 0);
 644                 c->m_server = this;
 645
 646                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 647                     c->set_APDU_yazlog(1);
 648                 else
 649                     c->set_APDU_yazlog(0);
 650
 651                 (parent->m_seqno)++;
 652
 653                 parent->pre_init();
 654
 655                 return c;
 656             }
 657         }
 658     }
 659     if (!m_client)
 660     {
 661         if (apdu->which != Z_APDU_initRequest)
 662         {
 663             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 664             return 0;
 665         }
 666         Z_InitRequest *initRequest = apdu->u.initRequest;
 667
 668         if (initRequest->idAuthentication)
 669         {
 670             // the client uses authentication. We set the keepalive PDU
 671             // to 0 so we don't cache it in releaseClient
 672             m_keepalive_limit_pdu = 0;
 673         }
 674         // go through list of clients - and find the lowest/oldest one.
 675         Yaz_ProxyClient *c_min = 0;
 676         int min_seq = -1;
 677         int no_of_clients = 0;
 678         if (parent->m_clientPool)
 679             yaz_log (YLOG_DEBUG, "Existing sessions");
 680         for (c = parent->m_clientPool; c; c = c->m_next)
 681         {
 682             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 683                                c->m_waiting, c->get_hostname(),
 684                                c->m_cookie ? c->m_cookie : "");
 685             no_of_clients++;
 686             if (min_seq < 0 || c->m_seqno < min_seq)
 687             {
 688                 min_seq = c->m_seqno;
 689                 c_min = c;
 690             }
 691         }
 692         if (no_of_clients >= parent->m_max_clients)
 693         {
 694             c = c_min;
 695             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 696             {
 697                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 698                          m_session_str, parent->m_max_clients, c->m_seqno);
 699                 if (c->m_server && c->m_server != this)
 700                     delete c->m_server;   // PROBLEM: m_ref_count!
 701                 c->m_server = 0;
 702             }
 703             else
 704             {
 705                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 706                          m_session_str, parent->m_max_clients,
 707                          c->m_seqno, parent->m_seqno, c->get_hostname());
 708                 xfree (c->m_cookie);
 709                 c->m_cookie = 0;
 710                 if (cookie)
 711                     c->m_cookie = xstrdup(cookie);
 712                 c->m_seqno = parent->m_seqno;
 713                 if (c->m_server && c->m_server != this)
 714                 {
 715                     c->m_server->m_client = 0;
 716                     delete c->m_server;   // PROBLEM: m_ref_count!
 717                 }
 718                 (parent->m_seqno)++;
 719                 c->m_target_idletime = m_target_idletime;
 720                 c->timeout(m_target_idletime);
 721
 722                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 723                     c->set_APDU_yazlog(1);
 724                 else
 725                     c->set_APDU_yazlog(0);
 726
 727                 return c;
 728             }
 729         }
 730         else
 731         {
 732             yaz_log (YLOG_LOG, "%sNEW %d %s",
 733                      m_session_str, parent->m_seqno, m_proxyTarget);
 734             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 735             c->m_next = parent->m_clientPool;
 736             if (c->m_next)
 737                 c->m_next->m_prev = &c->m_next;
 738             parent->m_clientPool = c;
 739             c->m_prev = &parent->m_clientPool;
 740         }
 741
 742         xfree (c->m_cookie);
 743         c->m_cookie = 0;
 744         if (cookie)
 745             c->m_cookie = xstrdup(cookie);
 746
 747         c->m_seqno = parent->m_seqno;
 748         c->m_init_flag = 0;
 749         c->m_last_resultCount = 0;
 750         c->m_last_ok = 0;
 751         c->m_cache.clear();
 752         c->m_sr_transform = 0;
 753         c->m_waiting = 0;
 754         c->m_resultSetStartPoint = 0;
 755         (parent->m_seqno)++;
 756         if (c->client(m_proxyTarget))
 757         {
 758             delete c;
 759             return 0;
 760         }
 761         c->m_target_idletime = m_target_idletime;
 762         c->timeout(30);
 763
 764         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 765             c->set_APDU_yazlog(1);
 766         else
 767             c->set_APDU_yazlog(0);
 768     }
 769     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 770     return c;
 771 }
 772
 773 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 774 {
 775     int i;
 776     for (i = 0; i<num; i++)
 777     {
 778         oident *ent;
 779         Z_DefaultDiagFormat *r;
 780         Z_DiagRec *p = pp[i];
 781         if (p->which != Z_DiagRec_defaultFormat)
 782         {
 783             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 784             return;
 785         }
 786         else
 787             r = p->u.defaultFormat;
 788         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 789             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 790             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 791         switch (r->which)
 792         {
 793         case Z_DefaultDiagFormat_v2Addinfo:
 794             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 795                     m_session_str,
 796                     *r->condition, diagbib1_str(*r->condition),
 797                     r->u.v2Addinfo);
 798             break;
 799         case Z_DefaultDiagFormat_v3Addinfo:
 800             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 801                     m_session_str,
 802                     *r->condition, diagbib1_str(*r->condition),
 803                     r->u.v3Addinfo);
 804             break;
 805         }
 806     }
 807 }
 808
 809 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 810 {
 811     if (!m_stylesheet_xsp || p->num_records <= 0)
 812     {
 813         return 0;  /* no XSLT to be done ... */
 814     }
 815
 816     m_stylesheet_offset = 0;
 817     m_stylesheet_nprl = p;
 818     m_stylesheet_apdu = apdu;
 819     m_timeout_mode = timeout_xsl;
 820
 821     timeout(0);
 822     return 1;
 823 }
 824
 825 void Yaz_Proxy::convert_xsl_delay()
 826 {
 827 #if HAVE_XSLT
 828     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 829     if (npr->which == Z_NamePlusRecord_databaseRecord)
 830     {
 831         Z_External *r = npr->u.databaseRecord;
 832         if (r->which == Z_External_octet)
 833         {
 834 #if 0
 835             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 836 #endif
 837             xmlDocPtr res, doc = xmlParseMemory(
 838                 (char*) r->u.octet_aligned->buf,
 839                 r->u.octet_aligned->len);
 840
 841
 842             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 843                     m_session_str, m_stylesheet_offset);
 844             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 845                                       doc, 0);
 846
 847             if (res)
 848             {
 849                 xmlChar *out_buf;
 850                 int out_len;
 851                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 852
 853                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 854                     u.databaseRecord =
 855                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 856                                  (char*) out_buf, out_len);
 857                 xmlFree(out_buf);
 858                 xmlFreeDoc(res);
 859             }
 860
 861             xmlFreeDoc(doc);
 862         }
 863     }
 864 #endif
 865     m_stylesheet_offset++;
 866     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 867     {
 868         m_timeout_mode = timeout_normal;
 869         m_stylesheet_nprl = 0;
 870 #if HAVE_XSLT
 871         if (m_stylesheet_xsp)
 872             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 873 #endif
 874         m_stylesheet_xsp = 0;
 875         timeout(m_client_idletime);
 876         send_PDU_convert(m_stylesheet_apdu);
 877     }
 878     else
 879         timeout(0);
 880 }
 881
 882 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 883 {
 884     if (m_frontend_type != VAL_NONE)
 885     {
 886         int i;
 887         for (i = 0; i < p->num_records; i++)
 888         {
 889             Z_NamePlusRecord *npr = p->records[i];
 890             if (npr->which == Z_NamePlusRecord_databaseRecord)
 891             {
 892                 Z_External *r = npr->u.databaseRecord;
 893                 if (r->which == Z_External_octet)
 894                 {
 895 #if HAVE_USEMARCON
 896                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 897                     {
 898                         if (!m_usemarcon->m_stage1)
 899                         {
 900                             m_usemarcon->m_stage1 = new CDetails();
 901                         }
 902                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 903                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 904                         int res = m_usemarcon->m_stage1->Start();
 905                         if (res == 0)
 906                         {
 907                             char *converted;
 908                             int convlen;
 909                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 910                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 911                             {
 912                                 if (!m_usemarcon->m_stage2)
 913                                 {
 914                                     m_usemarcon->m_stage2 = new CDetails();
 915                                 }
 916                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 917                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 918                                 res = m_usemarcon->m_stage2->Start();
 919                                 if (res == 0)
 920                                 {
 921                                     free(converted);
 922                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 923                                 }
 924                                 else
 925                                 {
 926                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 927                                 }
 928                             }
 929                             npr->u.databaseRecord =
 930                                 z_ext_record(odr_encode(),
 931                                              m_frontend_type,
 932                                              converted,
 933                                              strlen(converted));
 934                             free(converted);
 935                         }
 936                         else
 937                         {
 938                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 939                         }
 940                         continue;
 941                     }
 942 #endif
 943 /* HAVE_USEMARCON */
 944                     npr->u.databaseRecord =
 945                         z_ext_record(odr_encode(),
 946                                      m_frontend_type,
 947                                      (char*) r->u.octet_aligned->buf,
 948                                      r->u.octet_aligned->len);
 949                 }
 950             }
 951         }
 952     }
 953 }
 954
 955 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
 956                                         const char *backend_charset)
 957 {
 958     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
 959     int sel =   m_charset_converter->get_client_charset_selected();
 960     const char *client_record_charset =
 961         m_charset_converter->get_client_query_charset();
 962     if (sel && backend_charset && client_record_charset &&
 963         strcmp(backend_charset, client_record_charset))
 964     {
 965         int i;
 966         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
 967                                         backend_charset);
 968         yaz_marc_t mt = yaz_marc_create();
 969         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
 970         yaz_marc_iconv(mt, cd);
 971         for (i = 0; i < p->num_records; i++)
 972         {
 973             Z_NamePlusRecord *npr = p->records[i];
 974             if (npr->which == Z_NamePlusRecord_databaseRecord)
 975             {
 976                 Z_External *r = npr->u.databaseRecord;
 977                 oident *ent = oid_getentbyoid(r->direct_reference);
 978                 if (!ent || ent->value == VAL_NONE)
 979                     continue;
 980
 981                 if (ent->value == VAL_SUTRS)
 982                 {
 983                     WRBUF w = wrbuf_alloc();
 984
 985                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
 986                                       r->u.octet_aligned->len);
 987                     npr->u.databaseRecord =
 988                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
 989                                      wrbuf_len(w));
 990                     wrbuf_free(w, 1);
 991                 }
 992                 else if (ent->value == VAL_TEXT_XML)
 993                 {
 994                     ;
 995                 }
 996                 else if (r->which == Z_External_octet)
 997                 {
 998                     int rlen;
 999                     char *result;
1000                     if (yaz_marc_decode_buf(mt,
1001                                             (char*) r->u.octet_aligned->buf,
1002                                             r->u.octet_aligned->len,
1003                                             &result, &rlen))
1004                     {
1005                         npr->u.databaseRecord =
1006                             z_ext_record(odr_encode(), ent->value, result, rlen);
1007                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1008                                 m_session_str);
1009                     }
1010                 }
1011             }
1012         }
1013         if (cd)
1014             yaz_iconv_close(cd);
1015         yaz_marc_destroy(mt);
1016     }
1017     else
1018     {
1019         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
1020     }
1021 }
1022
1023 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1024                                    const char *backend_charset)
1025 {
1026     int i;
1027     if (!backend_charset)
1028         backend_charset = "MARC-8";
1029     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1030     yaz_marc_t mt = yaz_marc_create();
1031     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1032     yaz_marc_iconv(mt, cd);
1033     for (i = 0; i < p->num_records; i++)
1034     {
1035         Z_NamePlusRecord *npr = p->records[i];
1036         if (npr->which == Z_NamePlusRecord_databaseRecord)
1037         {
1038             Z_External *r = npr->u.databaseRecord;
1039             if (r->which == Z_External_OPAC)
1040             {
1041                 WRBUF w = wrbuf_alloc();
1042
1043                 yaz_display_OPAC(w, r->u.opac, 0);
1044                 npr->u.databaseRecord = z_ext_record(
1045                     odr_encode(), VAL_TEXT_XML,
1046                     wrbuf_buf(w), wrbuf_len(w)
1047                     );
1048                 wrbuf_free(w, 1);
1049             }
1050             else if (r->which == Z_External_octet)
1051             {
1052                 int rlen;
1053                 char *result;
1054                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1055                                         r->u.octet_aligned->len,
1056                                         &result, &rlen))
1057                 {
1058                     npr->u.databaseRecord =
1059                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
1060                 }
1061             }
1062         }
1063     }
1064     if (cd)
1065         yaz_iconv_close(cd);
1066     yaz_marc_destroy(mt);
1067 }
1068
1069 void Yaz_Proxy::logtime()
1070 {
1071 #if HAVE_GETTIMEOFDAY
1072     struct timeval *tv = (struct timeval*) m_time_tv;
1073     if (tv->tv_sec)
1074     {
1075         struct timeval tv1;
1076         gettimeofday(&tv1, 0);
1077         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1078             (tv1.tv_usec - tv->tv_usec);
1079         if (diff >= 0)
1080             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1081                     diff/1000000, (diff/1000)%1000);
1082     }
1083     tv->tv_sec = 0;
1084     tv->tv_usec = 0;
1085 #endif
1086 }
1087
1088 int Yaz_Proxy::send_http_response(int code)
1089 {
1090     ODR o = odr_encode();
1091     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1092     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1093     if (m_http_version)
1094         hres->version = odr_strdup(o, m_http_version);
1095     if (m_http_keepalive)
1096         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1097     else
1098         timeout(0);
1099
1100     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1101     {
1102         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1103                  gdu_name(gdu));
1104     }
1105     int len;
1106     int r = send_GDU(gdu, &len);
1107     m_bytes_sent += len;
1108     m_bw_stat.add_bytes(len);
1109     logtime();
1110
1111     recv_GDU_more(true);
1112
1113     return r;
1114 }
1115
1116 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
1117 {
1118     ODR o = odr_encode();
1119     const char *ctype = "text/xml";
1120     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
1121     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1122     if (m_http_version)
1123         hres->version = odr_strdup(o, m_http_version);
1124     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1125     if (m_http_keepalive)
1126         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1127     else
1128         timeout(0);
1129
1130     static Z_SOAP_Handler soap_handlers[2] = {
1131 #if HAVE_XSLT
1132         {"http://www.loc.gov/zing/srw/", 0,
1133          (Z_SOAP_fun) yaz_srw_codec},
1134 #endif
1135         {0, 0, 0}
1136     };
1137
1138     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1139     soap_package->which = Z_SOAP_generic;
1140     soap_package->u.generic =
1141         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1142     soap_package->u.generic->no = 0;
1143     soap_package->u.generic->ns = soap_handlers[0].ns;
1144     soap_package->u.generic->p = (void *) srw_pdu;
1145     soap_package->ns = m_soap_ns;
1146     z_soap_codec_enc_xsl(o, &soap_package,
1147                          &hres->content_buf, &hres->content_len,
1148                          soap_handlers, 0, m_s2z_stylesheet);
1149     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1150     {
1151         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1152                  gdu_name(gdu));
1153     }
1154     int len;
1155     int r = send_GDU(gdu, &len);
1156     m_bytes_sent += len;
1157     m_bw_stat.add_bytes(len);
1158     logtime();
1159
1160     recv_GDU_more(true);
1161
1162     return r;
1163 }
1164
1165 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1166 {
1167     ODR o = odr_encode();
1168     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1169     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1170
1171     srw_res->num_diagnostics = 1;
1172     srw_res->diagnostics = (Z_SRW_diagnostic *)
1173         odr_malloc(o, sizeof(*srw_res->diagnostics));
1174     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
1175     return send_srw_response(srw_pdu);
1176 }
1177
1178 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1179                              Z_DefaultDiagFormat *ddf)
1180 {
1181     int bib1_code = *ddf->condition;
1182     if (bib1_code == 109)
1183         return 404;
1184     srw_res->num_diagnostics = 1;
1185     srw_res->diagnostics = (Z_SRW_diagnostic *)
1186         odr_malloc(o, sizeof(*srw_res->diagnostics));
1187     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1188                           yaz_diag_bib1_to_srw(*ddf->condition),
1189                           ddf->u.v2Addinfo);
1190     return 0;
1191 }
1192
1193 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1194 {
1195     ODR o = odr_encode();
1196     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1197     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1198
1199     srw_res->numberOfRecords = odr_intdup (o, hits);
1200     if (records && records->which == Z_Records_DBOSD)
1201     {
1202         srw_res->num_records =
1203             records->u.databaseOrSurDiagnostics->num_records;
1204         int i;
1205         srw_res->records = (Z_SRW_record *)
1206             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1207         for (i = 0; i < srw_res->num_records; i++)
1208         {
1209             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1210             if (npr->which != Z_NamePlusRecord_databaseRecord)
1211             {
1212                 srw_res->records[i].recordSchema = "diagnostic";
1213                 srw_res->records[i].recordPacking = m_s2z_packing;
1214                 srw_res->records[i].recordData_buf = "67";
1215                 srw_res->records[i].recordData_len = 2;
1216                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1217                 continue;
1218             }
1219             Z_External *r = npr->u.databaseRecord;
1220             oident *ent = oid_getentbyoid(r->direct_reference);
1221             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1222             {
1223                 srw_res->records[i].recordSchema = m_schema;
1224                 srw_res->records[i].recordPacking = m_s2z_packing;
1225                 srw_res->records[i].recordData_buf = (char*)
1226                     r->u.octet_aligned->buf;
1227                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1228                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1229             }
1230             else
1231             {
1232                 srw_res->records[i].recordSchema = "diagnostic";
1233                 srw_res->records[i].recordPacking = m_s2z_packing;
1234                 srw_res->records[i].recordData_buf = "67";
1235                 srw_res->records[i].recordData_len = 2;
1236                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1237             }
1238         }
1239     }
1240     if (records && records->which == Z_Records_NSD)
1241     {
1242         int http_code;
1243         http_code = z_to_srw_diag(odr_encode(), srw_res,
1244                                    records->u.nonSurrogateDiagnostic);
1245         if (http_code)
1246             return send_http_response(http_code);
1247     }
1248     return send_srw_response(srw_pdu);
1249
1250 }
1251
1252 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1253                                         int num_diagnostics)
1254 {
1255     Yaz_ProxyConfig *cfg = check_reconfigure();
1256     if (cfg)
1257     {
1258         int len;
1259         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1260                                        m_s2z_database, &len);
1261         if (b)
1262         {
1263             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1264             Z_SRW_explainResponse *er = res->u.explain_response;
1265
1266             er->record.recordData_buf = b;
1267             er->record.recordData_len = len;
1268             er->record.recordPacking = m_s2z_packing;
1269             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1270
1271             er->diagnostics = diagnostics;
1272             er->num_diagnostics = num_diagnostics;
1273             return send_srw_response(res);
1274         }
1275     }
1276     return send_http_response(404);
1277 }
1278
1279 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1280 {
1281     if (m_http_version)
1282     {
1283         if (apdu->which == Z_APDU_initResponse)
1284         {
1285             Z_InitResponse *res = apdu->u.initResponse;
1286             if (*res->result == 0)
1287             {
1288                 send_to_srw_client_error(3, 0);
1289             }
1290             else if (!m_s2z_search_apdu)
1291             {
1292                 send_srw_explain_response(0, 0);
1293             }
1294             else
1295             {
1296                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1297             }
1298         }
1299         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1300         {
1301             m_s2z_search_apdu = 0;
1302             Z_SearchResponse *res = apdu->u.searchResponse;
1303             m_s2z_hit_count = *res->resultCount;
1304             if (res->records && res->records->which == Z_Records_NSD)
1305             {
1306                 send_to_srw_client_ok(0, res->records, 1);
1307             }
1308             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1309             {
1310                 // adjust
1311                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1312
1313                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1314                 {
1315                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1316                         > m_s2z_hit_count)
1317                         *pr->numberOfRecordsRequested =
1318                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1319                 }
1320                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1321             }
1322             else
1323             {
1324                 m_s2z_present_apdu = 0;
1325                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1326             }
1327         }
1328         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1329         {
1330             int start =
1331                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1332
1333             m_s2z_present_apdu = 0;
1334             Z_PresentResponse *res = apdu->u.presentResponse;
1335             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1336         }
1337     }
1338     else
1339     {
1340         int len = 0;
1341         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1342             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1343                      apdu_name(apdu));
1344         int r = send_Z_PDU(apdu, &len);
1345         m_bytes_sent += len;
1346         m_bw_stat.add_bytes(len);
1347         logtime();
1348         return r;
1349     }
1350     return 0;
1351 }
1352
1353 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1354 {
1355     int kill_session = 0;
1356     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1357
1358     if (new_id)
1359         *new_id = m_referenceId;
1360
1361     if (apdu->which == Z_APDU_searchResponse)
1362     {
1363         Z_SearchResponse *sr = apdu->u.searchResponse;
1364         Z_Records *p = sr->records;
1365         if (p && p->which == Z_Records_NSD)
1366         {
1367             Z_DiagRec dr, *dr_p = &dr;
1368             dr.which = Z_DiagRec_defaultFormat;
1369             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1370
1371             *sr->searchStatus = 0;
1372             display_diagrecs(&dr_p, 1);
1373         }
1374         else
1375         {
1376             if (p && p->which == Z_Records_DBOSD)
1377             {
1378                 if (m_backend_type
1379 #if HAVE_USEMARCON
1380                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1381 #endif
1382                     )
1383                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1384                 if (m_marcxml_mode == marcxml)
1385                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1386                                        m_backend_charset);
1387                 else
1388                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1389                                             m_backend_charset);
1390                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1391                     return 0;
1392
1393             }
1394             if (sr->resultCount)
1395             {
1396                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1397                         *sr->resultCount);
1398                 if (*sr->resultCount < 0)
1399                 {
1400                     m_flag_invalid_session = 1;
1401                     kill_session = 1;
1402
1403                     *sr->searchStatus = 0;
1404                     sr->records =
1405                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1406                     *sr->resultCount = 0;
1407                 }
1408             }
1409         }
1410     }
1411     else if (apdu->which == Z_APDU_presentResponse)
1412     {
1413         Z_PresentResponse *sr = apdu->u.presentResponse;
1414         Z_Records *p = sr->records;
1415         if (p && p->which == Z_Records_NSD)
1416         {
1417             Z_DiagRec dr, *dr_p = &dr;
1418             dr.which = Z_DiagRec_defaultFormat;
1419             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1420             if (*sr->presentStatus == Z_PresentStatus_success)
1421                 *sr->presentStatus = Z_PresentStatus_failure;
1422             display_diagrecs(&dr_p, 1);
1423         }
1424         if (p && p->which == Z_Records_DBOSD)
1425         {
1426             if (m_backend_type
1427 #if HAVE_USEMARCON
1428                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1429 #endif
1430                 )
1431                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1432             if (m_marcxml_mode == marcxml)
1433                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1434                                    m_backend_charset);
1435             else
1436                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1437                                         m_backend_charset);
1438             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1439                 return 0;
1440         }
1441     }
1442     else if (apdu->which == Z_APDU_initResponse)
1443     {
1444         //Get and check negotiation record
1445         //from init response.
1446         handle_charset_lang_negotiation(apdu);
1447
1448         if (m_initRequest_options)
1449         {
1450             Z_Options *nopt =
1451                 (Odr_bitmask *)odr_malloc(odr_encode(),
1452                                           sizeof(Odr_bitmask));
1453             ODR_MASK_ZERO(nopt);
1454
1455             int i;
1456             for (i = 0; i<24; i++)
1457                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1458                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1459                     ODR_MASK_SET(nopt, i);
1460             apdu->u.initResponse->options = nopt;
1461         }
1462         if (m_initRequest_version)
1463         {
1464             Z_ProtocolVersion *nopt =
1465                 (Odr_bitmask *)odr_malloc(odr_encode(),
1466                                           sizeof(Odr_bitmask));
1467             ODR_MASK_ZERO(nopt);
1468
1469             int i;
1470             for (i = 0; i<8; i++)
1471                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1472                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1473                     ODR_MASK_SET(nopt, i);
1474             apdu->u.initResponse->protocolVersion = nopt;
1475         }
1476         apdu->u.initResponse->preferredMessageSize =
1477             odr_intdup(odr_encode(),
1478                        m_client->m_initResponse_preferredMessageSize >
1479                        m_initRequest_preferredMessageSize ?
1480                        m_initRequest_preferredMessageSize :
1481                        m_client->m_initResponse_preferredMessageSize);
1482         apdu->u.initResponse->maximumRecordSize =
1483             odr_intdup(odr_encode(),
1484                        m_client->m_initResponse_maximumRecordSize >
1485                        m_initRequest_maximumRecordSize ?
1486                        m_initRequest_maximumRecordSize :
1487                        m_client->m_initResponse_maximumRecordSize);
1488     }
1489
1490     int r = send_PDU_convert(apdu);
1491     if (r)
1492         return r;
1493     if (kill_session)
1494     {
1495         delete m_client;
1496         m_client = 0;
1497         m_parent->pre_init();
1498     }
1499     return r;
1500 }
1501
1502 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1503 {
1504     int len = 0;
1505     const char *apdu_name_tmp = apdu_name(apdu);
1506     int r = send_Z_PDU(apdu, &len);
1507     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1508         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1509                  get_session_str(),
1510                  apdu_name_tmp, get_hostname(), len);
1511     m_bytes_sent += len;
1512     return r;
1513 }
1514
1515 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1516 {
1517     if (apdu->which == Z_APDU_presentRequest)
1518     {
1519         Z_PresentRequest *pr = apdu->u.presentRequest;
1520         int toget = *pr->numberOfRecordsRequested;
1521         int start = *pr->resultSetStartPoint;
1522
1523         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1524                 pr->resultSetId, start, toget);
1525
1526         if (*m_parent->m_optimize == '0')
1527             return apdu;
1528
1529         if (!m_client->m_last_resultSetId)
1530         {
1531             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1532             new_apdu->u.presentResponse->records =
1533                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1534                                                pr->resultSetId);
1535             send_to_client(new_apdu);
1536             return 0;
1537         }
1538         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1539         {
1540             if (start+toget-1 > m_client->m_last_resultCount)
1541             {
1542                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1543                 new_apdu->u.presentResponse->records =
1544                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1545                 send_to_client(new_apdu);
1546                 return 0;
1547             }
1548             Z_NamePlusRecordList *npr;
1549 #if 0
1550             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1551                     m_session_str, start, toget, yaz_z3950oid_to_str(
1552                         pr->preferredRecordSyntax, &oclass));
1553 #endif
1554             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1555                                           pr->preferredRecordSyntax,
1556                                           pr->recordComposition))
1557             {
1558                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1559                          m_session_str);
1560                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1561                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1562
1563                 new_apdu->u.presentResponse->numberOfRecordsReturned
1564                     = odr_intdup(odr_encode(), toget);
1565
1566                 new_apdu->u.presentResponse->records = (Z_Records*)
1567                     odr_malloc(odr_encode(), sizeof(Z_Records));
1568                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1569                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1570                 new_apdu->u.presentResponse->nextResultSetPosition =
1571                     odr_intdup(odr_encode(), start+toget);
1572
1573                 send_to_client(new_apdu);
1574                 return 0;
1575             }
1576         }
1577     }
1578
1579     if (apdu->which != Z_APDU_searchRequest)
1580         return apdu;
1581     Z_SearchRequest *sr = apdu->u.searchRequest;
1582     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1583     Yaz_Z_Databases this_databases;
1584
1585     this_databases.set(sr->num_databaseNames, (const char **)
1586                        sr->databaseNames);
1587
1588     this_query->set_Z_Query(sr->query);
1589
1590     char query_str[120];
1591     this_query->print(query_str, sizeof(query_str)-1);
1592     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1593
1594     if (*m_parent->m_optimize != '0' &&
1595         m_client->m_last_ok && m_client->m_last_query &&
1596         m_client->m_last_query->match(this_query) &&
1597         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1598         m_client->m_last_databases.match(this_databases))
1599     {
1600         delete this_query;
1601         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1602             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1603         {
1604             Z_NamePlusRecordList *npr;
1605             int toget = *sr->mediumSetPresentNumber;
1606             Z_RecordComposition *comp = 0;
1607
1608             if (toget > m_client->m_last_resultCount)
1609                 toget = m_client->m_last_resultCount;
1610
1611             if (sr->mediumSetElementSetNames)
1612             {
1613                 comp = (Z_RecordComposition *)
1614                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1615                 comp->which = Z_RecordComp_simple;
1616                 comp->u.simple = sr->mediumSetElementSetNames;
1617             }
1618
1619             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1620                                           sr->preferredRecordSyntax, comp))
1621             {
1622                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1623                          m_session_str);
1624                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1625                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1626                 new_apdu->u.searchResponse->resultCount =
1627                     &m_client->m_last_resultCount;
1628
1629                 new_apdu->u.searchResponse->numberOfRecordsReturned
1630                     = odr_intdup(odr_encode(), toget);
1631
1632                 new_apdu->u.searchResponse->presentStatus =
1633                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1634                 new_apdu->u.searchResponse->records = (Z_Records*)
1635                     odr_malloc(odr_encode(), sizeof(Z_Records));
1636                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1637                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1638                 new_apdu->u.searchResponse->nextResultSetPosition =
1639                     odr_intdup(odr_encode(), toget+1);
1640                 send_to_client(new_apdu);
1641                 return 0;
1642             }
1643             else
1644             {
1645                 // medium Set
1646                 // send present request (medium size)
1647                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1648                          m_session_str);
1649
1650                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1651                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1652                 pr->referenceId = sr->referenceId;
1653                 pr->resultSetId = sr->resultSetName;
1654                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1655                 *pr->numberOfRecordsRequested = toget;
1656                 pr->recordComposition = comp;
1657                 m_client->m_sr_transform = 1;
1658                 return new_apdu;
1659             }
1660         }
1661         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1662             m_client->m_last_resultCount <= 0)
1663         {
1664             // large set. Return pseudo-search response immediately
1665             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1666                      m_session_str);
1667             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1668             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1669             new_apdu->u.searchResponse->resultCount =
1670                 &m_client->m_last_resultCount;
1671             send_to_client(new_apdu);
1672             return 0;
1673         }
1674         else
1675         {
1676             Z_NamePlusRecordList *npr;
1677             int toget = m_client->m_last_resultCount;
1678             Z_RecordComposition *comp = 0;
1679             // small set
1680             // send a present request (small set)
1681
1682             if (sr->smallSetElementSetNames)
1683             {
1684                 comp = (Z_RecordComposition *)
1685                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1686                 comp->which = Z_RecordComp_simple;
1687                 comp->u.simple = sr->smallSetElementSetNames;
1688             }
1689
1690             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1691                                           sr->preferredRecordSyntax, comp))
1692             {
1693                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1694                          m_session_str);
1695                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1696                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1697                 new_apdu->u.searchResponse->resultCount =
1698                     &m_client->m_last_resultCount;
1699
1700                 new_apdu->u.searchResponse->numberOfRecordsReturned
1701                     = odr_intdup(odr_encode(), toget);
1702
1703                 new_apdu->u.searchResponse->presentStatus =
1704                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1705                 new_apdu->u.searchResponse->records = (Z_Records*)
1706                     odr_malloc(odr_encode(), sizeof(Z_Records));
1707                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1708                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1709                 new_apdu->u.searchResponse->nextResultSetPosition =
1710                     odr_intdup(odr_encode(), toget+1);
1711                 send_to_client(new_apdu);
1712                 return 0;
1713             }
1714             else
1715             {
1716                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1717                          m_session_str);
1718                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1719                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1720                 pr->referenceId = sr->referenceId;
1721                 pr->resultSetId = sr->resultSetName;
1722                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1723                 *pr->numberOfRecordsRequested = toget;
1724                 pr->recordComposition = comp;
1725                 m_client->m_sr_transform = 1;
1726                 return new_apdu;
1727             }
1728         }
1729     }
1730     else  // query doesn't match
1731     {
1732         delete m_client->m_last_query;
1733         m_client->m_last_query = this_query;
1734         m_client->m_last_ok = 0;
1735         m_client->m_cache.clear();
1736         m_client->m_resultSetStartPoint = 0;
1737
1738         xfree (m_client->m_last_resultSetId);
1739         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1740
1741         m_client->m_last_databases.set(sr->num_databaseNames,
1742                                        (const char **) sr->databaseNames);
1743     }
1744     return apdu;
1745 }
1746
1747
1748 void Yaz_Proxy::inc_request_no()
1749 {
1750     char *cp = strchr(m_session_str, ' ');
1751     m_request_no++;
1752     if (cp)
1753         sprintf(cp+1, "%d ", m_request_no);
1754 }
1755
1756 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1757 {
1758     inc_request_no();
1759
1760     m_bytes_recv += len;
1761
1762     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1763         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1764                  m_session_str, gdu_name(apdu), len);
1765
1766 #if HAVE_GETTIMEOFDAY
1767     gettimeofday((struct timeval *) m_time_tv, 0);
1768 #endif
1769     m_bw_stat.add_bytes(len);
1770     m_pdu_stat.add_bytes(1);
1771
1772     GDU *gdu = new GDU(apdu);
1773     m_in_queue.enqueue(gdu);
1774
1775     recv_GDU_more(false);
1776 }
1777
1778 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
1779 {
1780     int bw_total = m_bw_stat.get_total();
1781     int pdu_total = m_pdu_stat.get_total();
1782     int reduce = 0;
1783
1784     assert(m_timeout_mode == timeout_busy);
1785     assert(m_timeout_gdu == 0);
1786
1787     if (m_bw_max)
1788     {
1789         if (bw_total > m_bw_max)
1790         {
1791             reduce = (bw_total/m_bw_max);
1792         }
1793     }
1794     if (m_pdu_max)
1795     {
1796         if (pdu_total > m_pdu_max)
1797         {
1798             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1799             reduce = (reduce > nreduce) ? reduce : nreduce;
1800         }
1801     }
1802     m_http_version = 0;
1803
1804 #if 0
1805     /* uncomment to force a big reduce */
1806     m_timeout_mode = timeout_reduce;
1807     m_timeout_gdu = gdu;
1808     timeout(3);       // call us reduce seconds later
1809     return;
1810 #endif
1811     if (reduce)
1812     {
1813         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1814                 m_session_str, reduce, bw_total, pdu_total,
1815                 m_bw_max, m_pdu_max);
1816
1817         m_timeout_mode = timeout_reduce;
1818         m_timeout_gdu = gdu;
1819         timeout(reduce);       // call us reduce seconds later
1820     }
1821     else
1822         recv_GDU_normal(gdu);
1823 }
1824
1825 void Yaz_Proxy::recv_GDU_more(bool normal)
1826 {
1827     GDU *g;
1828     if (normal && m_timeout_mode == timeout_busy)
1829         m_timeout_mode = timeout_normal;
1830     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
1831     {
1832         m_timeout_mode = timeout_busy;
1833         recv_GDU_reduce(g);
1834     }
1835 }
1836
1837 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
1838 {
1839     Z_GDU *apdu = 0;
1840     gdu->move_away_gdu(odr_decode(), &apdu);
1841     delete gdu;
1842
1843     if (apdu->which == Z_GDU_Z3950)
1844         handle_incoming_Z_PDU(apdu->u.z3950);
1845     else if (apdu->which == Z_GDU_HTTP_Request)
1846         handle_incoming_HTTP(apdu->u.HTTP_Request);
1847 }
1848
1849 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1850 {
1851     if (m_max_record_retrieve)
1852     {
1853         if (apdu->which == Z_APDU_presentRequest)
1854         {
1855             Z_PresentRequest *pr = apdu->u.presentRequest;
1856             if (pr->numberOfRecordsRequested &&
1857                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1858                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1859         }
1860     }
1861 }
1862
1863 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1864 {
1865     if (apdu->which == Z_APDU_initRequest)
1866     {
1867         yaz_log(YLOG_LOG, "%shandle_charset_lang_negotiation",
1868                 m_session_str);
1869         if (m_initRequest_options &&
1870             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1871             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1872         {
1873             // There is no negotiation proposal from
1874             // client's side. OK. The proxy negotiation
1875             // in use, only.
1876             Z_InitRequest *initRequest = apdu->u.initRequest;
1877             Z_OtherInformation **otherInfo;
1878             Z_OtherInformationUnit *oi;
1879             get_otherInfoAPDU(apdu, &otherInfo);
1880             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1881             if (oi)
1882             {
1883                 ODR_MASK_SET(initRequest->options,
1884                     Z_Options_negotiationModel);
1885                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1886                 oi->information.externallyDefinedInfo =
1887                 yaz_set_proposal_charneg(odr_encode(),
1888                     (const char**)&m_proxy_negotiation_charset,
1889                     m_proxy_negotiation_charset ? 1:0,
1890                     (const char**)&m_proxy_negotiation_lang,
1891                     m_proxy_negotiation_lang ? 1:0,
1892                     1);
1893             }
1894         }
1895         else if (m_initRequest_options &&
1896                  ODR_MASK_GET(m_initRequest_options,
1897                               Z_Options_negotiationModel) &&
1898                  m_charset_converter->get_target_query_charset())
1899         {
1900             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1901                     m_session_str,
1902                     m_charset_converter->get_target_query_charset());
1903             Z_InitRequest *initRequest = apdu->u.initRequest;
1904             Z_CharSetandLanguageNegotiation *negotiation =
1905                 yaz_get_charneg_record (initRequest->otherInfo);
1906             if (negotiation &&
1907                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1908             {
1909                 NMEM nmem = nmem_create();
1910                 char **charsets = 0;
1911                 int num_charsets = 0;
1912                 char **langs = 0;
1913                 int num_langs = 0;
1914                 int selected = 0;
1915                 yaz_get_proposal_charneg (nmem, negotiation,
1916                                           &charsets, &num_charsets,
1917                                           &langs, &num_langs, &selected);
1918                 int i;
1919                 for (i = 0; i<num_charsets; i++)
1920                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
1921                             charsets[i]);
1922                 for (i = 0; i<num_langs; i++)
1923                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
1924                             langs[i]);
1925
1926                 const char *t_charset =
1927                     m_charset_converter->get_target_query_charset();
1928                 // sweep through charsets and pick the first supported
1929                 // conversion
1930                 for (i = 0; i<num_charsets; i++)
1931                 {
1932                     const char *c_charset = charsets[i];
1933                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
1934                         break;
1935                 }
1936                 if (i != num_charsets)
1937                 {
1938                     // got one .. set up ODR for reverse direction
1939                     const char *c_charset = charsets[i];
1940                     odr_set_charset(odr_encode(), c_charset, t_charset);
1941                     m_charset_converter->set_client_query_charset(c_charset);
1942                     m_charset_converter->set_client_charset_selected(selected);
1943                 }
1944                 nmem_destroy(nmem);
1945                 ODR_MASK_CLEAR(m_initRequest_options,
1946                                Z_Options_negotiationModel);
1947                 yaz_del_charneg_record(&initRequest->otherInfo);
1948             }
1949             else
1950             {
1951                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
1952                         m_session_str);
1953             }
1954         }
1955     }
1956     else if (apdu->which == Z_APDU_initResponse)
1957     {
1958         Z_InitResponse *initResponse = apdu->u.initResponse;
1959         Z_OtherInformation **otherInfo;
1960
1961         if (ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel))
1962         {
1963             char *charset = 0;
1964             char *lang = 0;
1965             int selected = 0;
1966
1967             get_otherInfoAPDU(apdu, &otherInfo);
1968
1969             if (!otherInfo && !(*otherInfo))
1970                 return;
1971
1972             Z_CharSetandLanguageNegotiation *charneg =
1973                 yaz_get_charneg_record(*otherInfo);
1974
1975             if (!charneg)
1976                 return;
1977
1978             yaz_get_response_charneg(m_referenceId_mem, charneg,
1979                 &charset, &lang, &selected);
1980
1981             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
1982                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
1983
1984             if (m_initRequest_options &&
1985                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
1986             {
1987                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
1988                     m_session_str);
1989             }
1990             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1991             {
1992                 // negotiation-charset, negotiation-lang
1993                 // elements of config file in use.
1994
1995                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
1996                     m_session_str);
1997
1998                 // clear negotiation option.
1999                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2000
2001                 // Delete negotiation (charneg-3) entry.
2002                 yaz_del_charneg_record(otherInfo);
2003             }
2004         }
2005         else
2006         {
2007             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2008             {
2009                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2010                         m_session_str);
2011             }
2012             else if (m_charset_converter->get_client_query_charset())
2013             {
2014                 Z_OtherInformation **otherInfo;
2015                 Z_OtherInformationUnit *oi;
2016                 get_otherInfoAPDU(apdu, &otherInfo);
2017                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2018                 if (oi)
2019                 {
2020                     ODR_MASK_SET(initResponse->options,
2021                                  Z_Options_negotiationModel);
2022                     ODR_MASK_SET(m_initRequest_options,
2023                                  Z_Options_negotiationModel);
2024
2025                     oi->which = Z_OtherInfo_externallyDefinedInfo;
2026                     oi->information.externallyDefinedInfo =
2027                         yaz_set_response_charneg(
2028                             odr_encode(),
2029                             m_charset_converter->get_client_query_charset(),
2030                             0 /* no lang */,
2031                             m_charset_converter->get_client_charset_selected());
2032                 }
2033             }
2034         }
2035     }
2036 }
2037
2038 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2039                                                      int error,
2040                                                      const char *addinfo)
2041 {
2042     Z_Records *rec = (Z_Records *)
2043         odr_malloc (odr, sizeof(*rec));
2044     int *err = (int *)
2045         odr_malloc (odr, sizeof(*err));
2046     Z_DiagRec *drec = (Z_DiagRec *)
2047         odr_malloc (odr, sizeof(*drec));
2048     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2049         odr_malloc (odr, sizeof(*dr));
2050     *err = error;
2051     rec->which = Z_Records_NSD;
2052     rec->u.nonSurrogateDiagnostic = dr;
2053     dr->diagnosticSetId =
2054         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
2055     dr->condition = err;
2056     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2057     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2058     return rec;
2059 }
2060
2061 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2062 {
2063     if (apdu->which == Z_APDU_searchRequest &&
2064         apdu->u.searchRequest->query &&
2065         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2066         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2067     {
2068         Z_RPNQuery *rpnquery = 0;
2069         Z_SearchRequest *sr = apdu->u.searchRequest;
2070         char *addinfo = 0;
2071
2072         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2073                 sr->query->u.type_104->u.cql);
2074
2075         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2076                                           &rpnquery, odr_encode(),
2077                                           &addinfo);
2078         if (r == -3)
2079             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2080         else if (r)
2081         {
2082             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2083             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2084
2085             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2086             new_apdu->u.searchResponse->records =
2087                 create_nonSurrogateDiagnostics(odr_encode(),
2088                                                yaz_diag_srw_to_bib1(r),
2089                                                addinfo);
2090             *new_apdu->u.searchResponse->searchStatus = 0;
2091
2092             send_to_client(new_apdu);
2093
2094             return 0;
2095         }
2096         else
2097         {
2098             sr->query->which = Z_Query_type_1;
2099             sr->query->u.type_1 = rpnquery;
2100         }
2101         return apdu;
2102     }
2103     return apdu;
2104 }
2105
2106 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2107 {
2108     if (apdu->which == Z_APDU_searchRequest &&
2109         apdu->u.searchRequest->query)
2110     {
2111         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2112             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2113         {
2114             if (m_http_version)
2115                 m_charset_converter->set_client_query_charset("UTF-8");
2116             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2117             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2118         }
2119     }
2120     return apdu;
2121 }
2122
2123
2124 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2125 {
2126     if (apdu->which == Z_APDU_searchRequest)
2127     {
2128         Z_SearchRequest *sr = apdu->u.searchRequest;
2129         int err = 0;
2130         char *addinfo = 0;
2131
2132         Yaz_ProxyConfig *cfg = check_reconfigure();
2133         if (cfg)
2134             err = cfg->check_query(odr_encode(), m_default_target,
2135                                    sr->query, &addinfo);
2136         if (err)
2137         {
2138             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2139
2140             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2141             new_apdu->u.searchResponse->records =
2142                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2143             *new_apdu->u.searchResponse->searchStatus = 0;
2144
2145             send_to_client(new_apdu);
2146
2147             return 0;
2148         }
2149     }
2150     return apdu;
2151 }
2152
2153 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2154 {
2155     if (apdu->which != Z_APDU_initRequest)
2156         return 1;  // pass if no init request
2157     Z_InitRequest *req = apdu->u.initRequest;
2158
2159     Yaz_ProxyConfig *cfg = check_reconfigure();
2160     if (!cfg)
2161         return 1;  // pass if no config
2162
2163     int ret;
2164     if (req->idAuthentication == 0)
2165     {
2166         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2167                                          m_peername);
2168     }
2169     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2170     {
2171         ret = cfg->client_authentication(
2172             m_default_target,
2173             req->idAuthentication->u.idPass->userId,
2174             req->idAuthentication->u.idPass->groupId,
2175             req->idAuthentication->u.idPass->password,
2176             m_peername);
2177     }
2178     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2179     {
2180         char user[64], pass[64];
2181         *user = '\0';
2182         *pass = '\0';
2183         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2184         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2185                                          m_peername);
2186     }
2187     else
2188         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2189                                          m_peername);
2190
2191     cfg->target_authentication(m_default_target, odr_encode(), req);
2192
2193     return ret;
2194 }
2195
2196 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2197 {
2198     m_marcxml_mode = none;
2199     if (apdu->which == Z_APDU_searchRequest)
2200     {
2201         Z_SearchRequest *sr = apdu->u.searchRequest;
2202         int err = 0;
2203         char *addinfo = 0;
2204         Yaz_ProxyConfig *cfg = check_reconfigure();
2205
2206         Z_RecordComposition rc_temp, *rc = 0;
2207         if (sr->smallSetElementSetNames)
2208         {
2209             rc_temp.which = Z_RecordComp_simple;
2210             rc_temp.u.simple = sr->smallSetElementSetNames;
2211             rc = &rc_temp;
2212         }
2213
2214         if (sr->preferredRecordSyntax)
2215         {
2216             struct oident *ent;
2217             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2218             m_frontend_type = ent->value;
2219         }
2220         else
2221             m_frontend_type = VAL_NONE;
2222
2223         char *stylesheet_name = 0;
2224         if (cfg)
2225             err = cfg->check_syntax(odr_encode(),
2226                                     m_default_target,
2227                                     sr->preferredRecordSyntax, rc,
2228                                     &addinfo, &stylesheet_name, &m_schema,
2229                                     &m_backend_type, &m_backend_charset,
2230                                     &m_usemarcon_ini_stage1,
2231                                     &m_usemarcon_ini_stage2);
2232         if (stylesheet_name)
2233         {
2234             m_parent->low_socket_close();
2235
2236 #if HAVE_XSLT
2237             if (m_stylesheet_xsp)
2238                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2239             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2240                                                        stylesheet_name);
2241 #endif
2242             m_stylesheet_offset = 0;
2243             xfree(stylesheet_name);
2244
2245             m_parent->low_socket_open();
2246         }
2247         if (err == -1)
2248         {
2249             sr->smallSetElementSetNames = 0;
2250             sr->mediumSetElementSetNames = 0;
2251             m_marcxml_mode = marcxml;
2252             if (m_backend_type)
2253             {
2254
2255                 sr->preferredRecordSyntax =
2256                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2257                                         m_backend_type);
2258             }
2259             else
2260                 sr->preferredRecordSyntax =
2261                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2262                                            VAL_USMARC);
2263         }
2264         else if (err)
2265         {
2266             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2267
2268             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2269             new_apdu->u.searchResponse->records =
2270                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2271             *new_apdu->u.searchResponse->searchStatus = 0;
2272
2273             send_to_client(new_apdu);
2274
2275             return 0;
2276         }
2277         else if (m_backend_type)
2278         {
2279             sr->preferredRecordSyntax =
2280                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2281         }
2282     }
2283     else if (apdu->which == Z_APDU_presentRequest)
2284     {
2285         Z_PresentRequest *pr = apdu->u.presentRequest;
2286         int err = 0;
2287         char *addinfo = 0;
2288         Yaz_ProxyConfig *cfg = check_reconfigure();
2289
2290         if (pr->preferredRecordSyntax)
2291         {
2292             struct oident *ent;
2293             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2294             m_frontend_type = ent->value;
2295         }
2296         else
2297             m_frontend_type = VAL_NONE;
2298
2299         char *stylesheet_name = 0;
2300         if (cfg)
2301             err = cfg->check_syntax(odr_encode(), m_default_target,
2302                                     pr->preferredRecordSyntax,
2303                                     pr->recordComposition,
2304                                     &addinfo, &stylesheet_name, &m_schema,
2305                                     &m_backend_type, &m_backend_charset,
2306                                     &m_usemarcon_ini_stage1,
2307                                     &m_usemarcon_ini_stage2
2308                                     );
2309         if (stylesheet_name)
2310         {
2311             m_parent->low_socket_close();
2312
2313 #if HAVE_XSLT
2314             if (m_stylesheet_xsp)
2315                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2316             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2317                                                        stylesheet_name);
2318 #endif
2319             m_stylesheet_offset = 0;
2320             xfree(stylesheet_name);
2321
2322             m_parent->low_socket_open();
2323         }
2324         if (err == -1)
2325         {
2326             pr->recordComposition = 0;
2327             m_marcxml_mode = marcxml;
2328             if (m_backend_type)
2329             {
2330
2331                 pr->preferredRecordSyntax =
2332                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2333                                         m_backend_type);
2334             }
2335             else
2336                 pr->preferredRecordSyntax =
2337                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2338                                            VAL_USMARC);
2339         }
2340         else if (err)
2341         {
2342             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2343
2344             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2345             new_apdu->u.presentResponse->records =
2346                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2347             *new_apdu->u.presentResponse->presentStatus =
2348                 Z_PresentStatus_failure;
2349
2350             send_to_client(new_apdu);
2351
2352             return 0;
2353         }
2354         else if (m_backend_type)
2355         {
2356             pr->preferredRecordSyntax =
2357                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2358         }
2359     }
2360     return apdu;
2361 }
2362
2363 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2364 {
2365     if (!schema)
2366         return 0;
2367     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2368         odr_malloc(o, sizeof(Z_ElementSetNames));
2369     esn->which = Z_ElementSetNames_generic;
2370     esn->u.generic = odr_strdup(o, schema);
2371     return esn;
2372 }
2373
2374 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2375 {
2376     const char *t = 0;
2377     Yaz_ProxyConfig *cfg = check_reconfigure();
2378     if (cfg)
2379         t = cfg->get_explain_name(db, backend_db);
2380
2381     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2382     {
2383         releaseClient();
2384     }
2385
2386     if (t)
2387     {
2388         xfree(m_default_target);
2389         m_default_target = xstrdup(t);
2390     }
2391 }
2392
2393 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2394 {
2395     struct stat sbuf;
2396     if (strcmp(hreq->method, "GET"))
2397         return 0;
2398     if (hreq->path[0] != '/')
2399     {
2400         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2401         return 0;
2402     }
2403     const char *cp = hreq->path;
2404     while (*cp)
2405     {
2406         if (*cp == '/' && strchr("/.", cp[1]))
2407         {
2408             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2409             return 0;
2410         }
2411         cp++;
2412     }
2413     const char *fname = hreq->path+1;
2414     if (stat(fname, &sbuf))
2415     {
2416         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2417         return 0;
2418     }
2419     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2420     {
2421         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2422         return 0;
2423     }
2424     if (sbuf.st_size > (off_t) 1000000)
2425     {
2426         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2427         return 0;
2428     }
2429
2430     ODR o = odr_encode();
2431     Yaz_ProxyConfig *cfg = check_reconfigure();
2432     const char *ctype = cfg->check_mime_type(fname);
2433     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2434     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2435     if (m_http_version)
2436         hres->version = odr_strdup(o, m_http_version);
2437     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2438     if (m_http_keepalive)
2439         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2440     else
2441         timeout(0);
2442
2443     hres->content_len = sbuf.st_size;
2444     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2445     FILE *f = fopen(fname, "rb");
2446     if (f)
2447     {
2448         fread(hres->content_buf, 1, hres->content_len, f);
2449         fclose(f);
2450     }
2451     else
2452     {
2453         return 0;
2454     }
2455     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2456     {
2457         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2458                  fname);
2459     }
2460     int len;
2461     send_GDU(gdu, &len);
2462     recv_GDU_more(true);
2463     return 1;
2464 }
2465
2466 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2467 {
2468     if (m_s2z_odr_init)
2469     {
2470         odr_destroy(m_s2z_odr_init);
2471         m_s2z_odr_init = 0;
2472     }
2473     if (m_s2z_odr_search)
2474     {
2475         odr_destroy(m_s2z_odr_search);
2476         m_s2z_odr_search = 0;
2477     }
2478
2479     m_http_keepalive = 0;
2480     m_http_version = 0;
2481     if (!strcmp(hreq->version, "1.0"))
2482     {
2483         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2484         if (v && !strcmp(v, "Keep-Alive"))
2485             m_http_keepalive = 1;
2486         else
2487             m_http_keepalive = 0;
2488         m_http_version = "1.0";
2489     }
2490     else
2491     {
2492         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2493         if (v && !strcmp(v, "close"))
2494             m_http_keepalive = 0;
2495         else
2496             m_http_keepalive = 1;
2497         m_http_version = "1.1";
2498     }
2499
2500     Z_SRW_PDU *srw_pdu = 0;
2501     Z_SOAP *soap_package = 0;
2502     char *charset = 0;
2503     Z_SRW_diagnostic *diagnostic = 0;
2504     int num_diagnostic = 0;
2505
2506     if (file_access(hreq))
2507     {
2508         return;
2509     }
2510     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2511                             &charset) == 0
2512              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2513                                &charset, &diagnostic, &num_diagnostic) == 0)
2514     {
2515         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2516         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2517         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2518         m_s2z_init_apdu = 0;
2519         m_s2z_search_apdu = 0;
2520         m_s2z_present_apdu = 0;
2521
2522         m_s2z_stylesheet = 0;
2523
2524         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2525         {
2526             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2527
2528             const char *backend_db = srw_req->database;
2529             srw_get_client(srw_req->database, &backend_db);
2530
2531             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2532             // recordXPath unsupported.
2533             if (srw_req->recordXPath)
2534             {
2535                 yaz_add_srw_diagnostic(odr_decode(),
2536                                        &diagnostic, &num_diagnostic,
2537                                        72, 0);
2538             }
2539             // sort unsupported
2540             if (srw_req->sort_type != Z_SRW_sort_type_none)
2541             {
2542                 yaz_add_srw_diagnostic(odr_decode(),
2543                                        &diagnostic, &num_diagnostic,
2544                                        80, 0);
2545             }
2546             // save stylesheet
2547             if (srw_req->stylesheet)
2548                 m_s2z_stylesheet =
2549                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2550
2551             // set packing for response records ..
2552             if (srw_req->recordPacking &&
2553                 !strcmp(srw_req->recordPacking, "xml"))
2554                 m_s2z_packing = Z_SRW_recordPacking_XML;
2555             else
2556                 m_s2z_packing = Z_SRW_recordPacking_string;
2557
2558             if (num_diagnostic)
2559             {
2560                 Z_SRW_PDU *srw_pdu =
2561                     yaz_srw_get(odr_encode(),
2562                                 Z_SRW_searchRetrieve_response);
2563                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2564
2565                 srw_res->diagnostics = diagnostic;
2566                 srw_res->num_diagnostics = num_diagnostic;
2567                 send_srw_response(srw_pdu);
2568                 return;
2569             }
2570
2571             // prepare search PDU
2572             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2573                                           Z_APDU_searchRequest);
2574             Z_SearchRequest *z_searchRequest =
2575                 m_s2z_search_apdu->u.searchRequest;
2576
2577             z_searchRequest->num_databaseNames = 1;
2578             z_searchRequest->databaseNames = (char**)
2579                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2580             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2581                                                            backend_db);
2582
2583             // query transformation
2584             Z_Query *query = (Z_Query *)
2585                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2586             z_searchRequest->query = query;
2587
2588             if (srw_req->query_type == Z_SRW_query_type_cql)
2589             {
2590                 Z_External *ext = (Z_External *)
2591                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2592                 ext->direct_reference =
2593                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2594                 ext->indirect_reference = 0;
2595                 ext->descriptor = 0;
2596                 ext->which = Z_External_CQL;
2597                 ext->u.cql = srw_req->query.cql;
2598
2599                 query->which = Z_Query_type_104;
2600                 query->u.type_104 =  ext;
2601             }
2602             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2603             {
2604                 Z_RPNQuery *RPNquery;
2605                 YAZ_PQF_Parser pqf_parser;
2606
2607                 pqf_parser = yaz_pqf_create ();
2608
2609                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2610                                           srw_req->query.pqf);
2611                 if (!RPNquery)
2612                 {
2613                     const char *pqf_msg;
2614                     size_t off;
2615                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2616                     int ioff = off;
2617                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
2618                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2619
2620                     send_to_srw_client_error(10, 0);
2621                     return;
2622                 }
2623                 query->which = Z_Query_type_1;
2624                 query->u.type_1 =  RPNquery;
2625
2626                 yaz_pqf_destroy (pqf_parser);
2627             }
2628             else
2629             {
2630                 send_to_srw_client_error(7, "query");
2631                 return;
2632             }
2633
2634             // present
2635             m_s2z_present_apdu = 0;
2636             int max = 0;
2637             if (srw_req->maximumRecords)
2638                 max = *srw_req->maximumRecords;
2639             int start = 1;
2640             if (srw_req->startRecord)
2641                 start = *srw_req->startRecord;
2642             if (max > 0)
2643             {
2644                 // Some backend, such as Voyager doesn't honor piggyback
2645                 // So we use present always (0 &&).
2646                 if (0 && start <= 1)  // Z39.50 piggyback
2647                 {
2648                     *z_searchRequest->smallSetUpperBound = max;
2649                     *z_searchRequest->mediumSetPresentNumber = max;
2650                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2651
2652                     z_searchRequest->preferredRecordSyntax =
2653                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2654                                                VAL_TEXT_XML);
2655                     if (srw_req->recordSchema)
2656                     {
2657                         z_searchRequest->smallSetElementSetNames =
2658                             z_searchRequest->mediumSetElementSetNames =
2659                             mk_esn_from_schema(m_s2z_odr_search,
2660                                                srw_req->recordSchema);
2661                     }
2662                 }
2663                 else   // Z39.50 present
2664                 {
2665                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2666                                                    Z_APDU_presentRequest);
2667                     Z_PresentRequest *z_presentRequest =
2668                         m_s2z_present_apdu->u.presentRequest;
2669                     *z_presentRequest->resultSetStartPoint = start;
2670                     *z_presentRequest->numberOfRecordsRequested = max;
2671                     z_presentRequest->preferredRecordSyntax =
2672                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2673                                                VAL_TEXT_XML);
2674                     if (srw_req->recordSchema)
2675                     {
2676                         z_presentRequest->recordComposition =
2677                             (Z_RecordComposition *)
2678                             odr_malloc(m_s2z_odr_search,
2679                                        sizeof(Z_RecordComposition));
2680                         z_presentRequest->recordComposition->which =
2681                             Z_RecordComp_simple;
2682                         z_presentRequest->recordComposition->u.simple =
2683                             mk_esn_from_schema(m_s2z_odr_search,
2684                                                srw_req->recordSchema);
2685                     }
2686                 }
2687             }
2688             if (!m_client)
2689             {
2690                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2691                                             Z_APDU_initRequest);
2692
2693                 // prevent m_initRequest_apdu memory from being grabbed
2694                 // in Yaz_Proxy::handle_incoming_Z_PDU
2695                 m_initRequest_apdu = m_s2z_init_apdu;
2696                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2697                 return;
2698             }
2699             else
2700             {
2701                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2702                 return;
2703             }
2704         }
2705         else if (srw_pdu->which == Z_SRW_explain_request)
2706         {
2707             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2708
2709             const char *backend_db = srw_req->database;
2710             srw_get_client(srw_req->database, &backend_db);
2711
2712             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2713
2714             // save stylesheet
2715             if (srw_req->stylesheet)
2716                 m_s2z_stylesheet =
2717                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2718
2719             if (srw_req->recordPacking &&
2720                 !strcmp(srw_req->recordPacking, "xml"))
2721                 m_s2z_packing = Z_SRW_recordPacking_XML;
2722             else
2723                 m_s2z_packing = Z_SRW_recordPacking_string;
2724
2725             if (num_diagnostic)
2726             {
2727                 send_srw_explain_response(diagnostic, num_diagnostic);
2728                 return;
2729             }
2730
2731             if (!m_client)
2732             {
2733                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2734                                             Z_APDU_initRequest);
2735
2736                 // prevent m_initRequest_apdu memory from being grabbed
2737                 // in Yaz_Proxy::handle_incoming_Z_PDU
2738                 m_initRequest_apdu = m_s2z_init_apdu;
2739                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2740             }
2741             else
2742                 send_srw_explain_response(0, 0);
2743             return;
2744         }
2745         else if (srw_pdu->which == Z_SRW_scan_request)
2746         {
2747             m_s2z_database = odr_strdup(m_s2z_odr_init,
2748                                         srw_pdu->u.scan_request->database);
2749
2750             yaz_add_srw_diagnostic(odr_decode(),
2751                                    &diagnostic, &num_diagnostic,
2752                                    4, "scan");
2753             Z_SRW_PDU *srw_pdu =
2754                 yaz_srw_get(odr_encode(),
2755                             Z_SRW_scan_response);
2756             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2757
2758             srw_res->diagnostics = diagnostic;
2759             srw_res->num_diagnostics = num_diagnostic;
2760             send_srw_response(srw_pdu);
2761             return;
2762         }
2763         else
2764         {
2765             m_s2z_database = 0;
2766
2767             send_to_srw_client_error(4, 0);
2768         }
2769     }
2770     send_http_response(400);
2771 }
2772
2773 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2774 {
2775     Z_OtherInformation **oi;
2776     get_otherInfoAPDU(apdu, &oi);
2777
2778     if (apdu->u.initRequest->implementationId)
2779         yaz_log(YLOG_LOG, "%simplementationId: %s",
2780                 m_session_str, apdu->u.initRequest->implementationId);
2781     if (apdu->u.initRequest->implementationName)
2782         yaz_log(YLOG_LOG, "%simplementationName: %s",
2783                 m_session_str, apdu->u.initRequest->implementationName);
2784     if (apdu->u.initRequest->implementationVersion)
2785         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2786                 m_session_str, apdu->u.initRequest->implementationVersion);
2787     if (m_initRequest_apdu == 0)
2788     {
2789         if (m_initRequest_mem)
2790             nmem_destroy(m_initRequest_mem);
2791
2792         m_initRequest_apdu = apdu;
2793         m_initRequest_mem = odr_extract_mem(odr_decode());
2794
2795         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2796             preferredMessageSize;
2797         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2798         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2799             maximumRecordSize;
2800         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2801
2802         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2803             yaz_get_charneg_record(*oi);
2804
2805         // Save proposal charsets and langs.
2806         if (ODR_MASK_GET(apdu->u.initRequest->options,
2807                          Z_Options_negotiationModel)
2808             && charSetandLangRecord)
2809         {
2810
2811             yaz_get_proposal_charneg(m_referenceId_mem,
2812                                      charSetandLangRecord,
2813                                      &m_initRequest_oi_negotiation_charsets,
2814                                      &m_initRequest_oi_negotiation_num_charsets,
2815                                      &m_initRequest_oi_negotiation_langs,
2816                                      &m_initRequest_oi_negotiation_num_langs,
2817                                      &m_initRequest_oi_negotiation_selected);
2818
2819             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2820             {
2821                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2822                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2823                         m_initRequest_oi_negotiation_charsets[i]:"none");
2824             }
2825             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2826             {
2827                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2828                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2829                         m_initRequest_oi_negotiation_langs[i]:"none");
2830             }
2831             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2832                     m_session_str, m_initRequest_oi_negotiation_selected);
2833         }
2834         // save init options for the response..
2835         m_initRequest_options = apdu->u.initRequest->options;
2836
2837         apdu->u.initRequest->options =
2838             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2839                                        sizeof(Odr_bitmask));
2840         ODR_MASK_ZERO(apdu->u.initRequest->options);
2841         int i;
2842         for (i = 0; i<= 24; i++)
2843             ODR_MASK_SET(apdu->u.initRequest->options, i);
2844         // check negotiation option
2845         if (!ODR_MASK_GET(m_initRequest_options,
2846                           Z_Options_negotiationModel))
2847         {
2848             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2849                            Z_Options_negotiationModel);
2850         }
2851         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2852                        Z_Options_concurrentOperations);
2853         // make new version
2854         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2855         apdu->u.initRequest->protocolVersion =
2856             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2857                                        sizeof(Odr_bitmask));
2858         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2859
2860         for (i = 0; i<= 8; i++)
2861             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2862     }
2863     if (m_client->m_init_flag)
2864     {
2865         if (handle_init_response_for_invalid_session(apdu))
2866             return;
2867         if (m_client->m_initResponse)
2868         {
2869             Z_APDU *apdu2 = m_client->m_initResponse;
2870             apdu2->u.initResponse->otherInfo = 0;
2871             if (m_client->m_cookie && *m_client->m_cookie)
2872                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2873                                            m_client->m_cookie);
2874             apdu2->u.initResponse->referenceId =
2875                 apdu->u.initRequest->referenceId;
2876             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2877             apdu2->u.initResponse->protocolVersion =
2878                 m_client->m_initResponse_version;
2879
2880             send_to_client(apdu2);
2881             m_timeout_mode = timeout_normal;
2882             return;
2883         }
2884     }
2885     m_client->m_init_flag = 1;
2886
2887 #if USE_AUTH_MSG
2888     yaz_log(YLOG_LOG, "%suse_auth_msg", m_session_str);
2889     Auth_Msg *m = new Auth_Msg;
2890     m->m_proxy = this;
2891     z_APDU(odr_encode(), &apdu, 0, "encode");
2892     char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
2893     m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
2894     memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
2895     odr_reset(odr_encode());
2896
2897     inc_ref();
2898     m_my_thread->put(m);
2899 #else
2900     int ret = handle_authentication(apdu);
2901     result_authentication(apdu, ret);
2902 #endif
2903 }
2904
2905 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2906 {
2907     Z_ReferenceId **refid = get_referenceIdP(apdu);
2908     nmem_reset(m_referenceId_mem);
2909     if (refid && *refid)
2910     {
2911         m_referenceId = (Z_ReferenceId *)
2912             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2913         m_referenceId->len = m_referenceId->size = (*refid)->len;
2914         m_referenceId->buf = (unsigned char *)
2915             nmem_malloc(m_referenceId_mem, (*refid)->len);
2916         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2917     }
2918     else
2919         m_referenceId = 0;
2920
2921     if (!m_client && m_flag_invalid_session)
2922     {
2923         // Got request for a session that is invalid..
2924         m_apdu_invalid_session = apdu; // save package
2925         m_mem_invalid_session = odr_extract_mem(odr_decode());
2926         apdu = m_initRequest_apdu;     // but throw an init to the target
2927     }
2928
2929     // Determine our client.
2930     Z_OtherInformation **oi;
2931     get_otherInfoAPDU(apdu, &oi);
2932     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2933     if (!m_client)
2934     {
2935         if (m_http_version)
2936         {   // HTTP. Send not found
2937             send_http_response(404);
2938             return;
2939         }
2940         else
2941         {
2942             // Z39.50 just shutdown
2943             delete this;
2944             return;
2945         }
2946     }
2947
2948     m_client->m_server = this;
2949
2950     if (apdu->which == Z_APDU_initRequest)
2951         handle_init(apdu);
2952     else
2953         handle_incoming_Z_PDU_2(apdu);
2954 }
2955
2956 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
2957 {
2958     handle_max_record_retrieve(apdu);
2959
2960     if (apdu)
2961         apdu = handle_syntax_validation(apdu);
2962
2963     if (apdu)
2964         apdu = handle_query_transformation(apdu);
2965
2966     if (apdu)
2967         apdu = handle_target_charset_conversion(apdu);
2968
2969     if (apdu)
2970         apdu = handle_query_validation(apdu);
2971
2972     if (apdu)
2973         apdu = result_set_optimize(apdu);
2974
2975     if (!apdu)
2976     {
2977         m_client->timeout(m_target_idletime);  // mark it active even
2978         recv_GDU_more(true);
2979         // though we didn't use it
2980         return;
2981     }
2982     // Add otherInformation entry in APDU if
2983     // negotiation is in use.
2984     if (apdu)
2985         handle_charset_lang_negotiation(apdu);
2986
2987     // delete other info construct completely if 0 elements
2988     Z_OtherInformation **oi;
2989     get_otherInfoAPDU(apdu, &oi);
2990     if (oi && *oi && (*oi)->num_elements == 0)
2991         *oi = 0;
2992
2993     if (apdu->which == Z_APDU_presentRequest &&
2994         m_client->m_resultSetStartPoint == 0)
2995     {
2996         Z_PresentRequest *pr = apdu->u.presentRequest;
2997         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2998         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2999     } else {
3000         m_client->m_resultSetStartPoint = 0;
3001     }
3002     if (m_client->send_to_target(apdu) < 0)
3003     {
3004         m_client->shutdown();
3005     }
3006     else
3007         m_client->m_waiting = 1;
3008 }
3009
3010 void Yaz_Proxy::connectNotify()
3011 {
3012 }
3013
3014 void Yaz_Proxy::releaseClient()
3015 {
3016     xfree(m_proxyTarget);
3017     m_proxyTarget = 0;
3018     m_flag_invalid_session = 0;
3019     // only keep if keep_alive flag is set...
3020     if (m_client &&
3021         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3022         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3023         m_client->m_waiting == 0)
3024     {
3025         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3026                  m_session_str,
3027                  m_client->get_hostname());
3028         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3029                 m_session_str, m_client->m_pdu_recv,
3030                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3031                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3032         assert (m_client->m_waiting != 2);
3033         // Tell client (if any) that no server connection is there..
3034         m_client->m_server = 0;
3035         m_client = 0;
3036     }
3037     else if (m_client)
3038     {
3039         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
3040                  m_session_str,
3041                  m_client->get_hostname());
3042         assert (m_client->m_waiting != 2);
3043         delete m_client;
3044         m_client = 0;
3045     }
3046     else if (!m_parent)
3047     {
3048         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
3049                  m_session_str);
3050         assert (m_parent);
3051     }
3052     else
3053     {
3054         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
3055                  m_session_str);
3056     }
3057     if (m_parent)
3058         m_parent->pre_init();
3059 }
3060
3061 bool Yaz_Proxy::dec_ref()
3062 {
3063     yaz_log(YLOG_LOG, "%sdec_ref count=%d", m_session_str, m_ref_count);
3064     assert(m_ref_count > 0);
3065
3066     --m_ref_count;
3067     if (m_ref_count > 0)
3068         return false;
3069
3070     releaseClient();
3071
3072     delete this;
3073     return true;
3074 }
3075
3076 const char *Yaz_ProxyClient::get_session_str()
3077 {
3078     if (!m_server)
3079         return "0 ";
3080     return m_server->get_session_str();
3081 }
3082
3083 void Yaz_ProxyClient::shutdown()
3084 {
3085     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3086              get_hostname());
3087
3088     if (m_server)
3089     {
3090         m_waiting = 1;      // ensure it's released from Proxy in releaseClient
3091         m_server->dec_ref();
3092     }
3093     else
3094         delete this;
3095 }
3096
3097 void Yaz_Proxy::failNotify()
3098 {
3099     inc_request_no();
3100     yaz_log (YLOG_LOG, "%sConnection closed by client",
3101              get_session_str());
3102     dec_ref();
3103 }
3104
3105 void Yaz_ProxyClient::failNotify()
3106 {
3107     if (m_server)
3108         m_server->inc_request_no();
3109     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
3110              get_session_str(), get_hostname());
3111     shutdown();
3112 }
3113
3114 void Yaz_ProxyClient::connectNotify()
3115 {
3116     const char *s = get_session_str();
3117     const char *h = get_hostname();
3118     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3119              m_target_idletime);
3120     timeout(m_target_idletime);
3121     if (!m_server)
3122         pre_init_client();
3123 }
3124
3125 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3126                                               *the_PDU_Observable, int fd)
3127 {
3128     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3129 }
3130
3131 Yaz_ProxyClient::~Yaz_ProxyClient()
3132 {
3133     if (m_prev)
3134         *m_prev = m_next;
3135     if (m_next)
3136         m_next->m_prev = m_prev;
3137     m_waiting = 2;     // for debugging purposes only.
3138     odr_destroy(m_init_odr);
3139     delete m_last_query;
3140     xfree (m_last_resultSetId);
3141     xfree (m_cookie);
3142 }
3143
3144 void Yaz_ProxyClient::pre_init_client()
3145 {
3146     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3147     Z_InitRequest *req = apdu->u.initRequest;
3148
3149     int i;
3150     for (i = 0; i<= 24; i++)
3151         ODR_MASK_SET(req->options, i);
3152     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3153                    Z_Options_negotiationModel);
3154     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3155                    Z_Options_concurrentOperations);
3156     for (i = 0; i<= 10; i++)
3157         ODR_MASK_SET(req->protocolVersion, i);
3158
3159     if (send_to_target(apdu) < 0)
3160     {
3161         delete this;
3162     }
3163     else
3164     {
3165         m_waiting = 1;
3166         m_init_flag = 1;
3167     }
3168 }
3169
3170 void Yaz_Proxy::pre_init()
3171 {
3172     int i;
3173     const char *name = 0;
3174     const char *zurl_in_use[MAX_ZURL_PLEX];
3175     int limit_bw, limit_pdu, limit_req, limit_search, limit_connect;
3176     int target_idletime, client_idletime;
3177     int max_clients;
3178     int keepalive_limit_bw, keepalive_limit_pdu;
3179     int pre_init;
3180     const char *cql2rpn = 0;
3181     const char *authentication = 0;
3182     const char *negotiation_charset = 0;
3183     const char *negotiation_lang = 0;
3184
3185     Yaz_ProxyConfig *cfg = check_reconfigure();
3186
3187     zurl_in_use[0] = 0;
3188
3189     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3190         set_APDU_yazlog(1);
3191     else
3192         set_APDU_yazlog(0);
3193
3194     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3195                                           &limit_bw, &limit_pdu, &limit_req,
3196                                           &limit_search, &limit_connect,
3197                                           &target_idletime, &client_idletime,
3198                                           &max_clients,
3199                                           &keepalive_limit_bw,
3200                                           &keepalive_limit_pdu,
3201                                           &pre_init,
3202                                           &cql2rpn,
3203                                           &authentication,
3204                                           &negotiation_charset,
3205                                           &negotiation_lang,
3206                                           0) ; i++)
3207     {
3208         if (pre_init)
3209         {
3210             int j;
3211             for (j = 0; zurl_in_use[j]; j++)
3212             {
3213                 Yaz_ProxyClient *c;
3214                 int spare = 0;
3215                 int spare_waiting = 0;
3216                 int in_use = 0;
3217                 int other = 0;
3218                 for (c = m_clientPool; c; c = c->m_next)
3219                 {
3220                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3221                     {
3222                         if (c->m_cookie == 0)
3223                         {
3224                             if (c->m_server == 0)
3225                                 if (c->m_waiting)
3226                                     spare_waiting++;
3227                                 else
3228                                     spare++;
3229                             else
3230                                 in_use++;
3231                         }
3232                         else
3233                             other++;
3234                     }
3235                 }
3236                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3237                         "sparew=%d preinit=%d",m_session_str,
3238                         name, zurl_in_use[j], in_use, other,
3239                         spare, spare_waiting, pre_init);
3240                 if (spare + spare_waiting < pre_init)
3241                 {
3242                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3243                     c->m_next = m_clientPool;
3244                     if (c->m_next)
3245                         c->m_next->m_prev = &c->m_next;
3246                     m_clientPool = c;
3247                     c->m_prev = &m_clientPool;
3248
3249                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3250                         c->set_APDU_yazlog(1);
3251                     else
3252                         c->set_APDU_yazlog(0);
3253
3254                     if (c->client(zurl_in_use[j]))
3255                     {
3256                         timeout(60);
3257                         delete c;
3258                         return;
3259                     }
3260                     c->timeout(30);
3261                     c->m_waiting = 1;
3262                     c->m_target_idletime = target_idletime;
3263                     c->m_seqno = m_seqno++;
3264                 }
3265             }
3266         }
3267     }
3268 }
3269
3270 void Yaz_Proxy::timeoutNotify()
3271 {
3272     if (m_parent)
3273     {
3274         GDU *gdu;
3275         switch(m_timeout_mode)
3276         {
3277         case timeout_normal:
3278         case timeout_busy:
3279             inc_request_no();
3280             m_in_queue.clear();
3281             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3282             dec_ref();
3283             break;
3284         case timeout_reduce:
3285             timeout(m_client_idletime);
3286             m_timeout_mode = timeout_busy;
3287             gdu = m_timeout_gdu;
3288             m_timeout_gdu = 0;
3289             recv_GDU_normal(gdu);
3290             break;
3291         case timeout_xsl:
3292             assert(m_stylesheet_nprl);
3293             convert_xsl_delay();
3294             recv_GDU_more(true);
3295         }
3296     }
3297     else
3298     {
3299         timeout(600);
3300         pre_init();
3301     }
3302 }
3303
3304 void Yaz_Proxy::markInvalid()
3305 {
3306     m_client = 0;
3307     m_flag_invalid_session = 1;
3308 }
3309
3310 void Yaz_ProxyClient::timeoutNotify()
3311 {
3312     if (m_server)
3313         m_server->inc_request_no();
3314
3315     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3316              get_hostname());
3317     m_waiting = 1;
3318     m_root->pre_init();
3319     if (m_server && m_init_flag)
3320     {
3321         // target timed out in a session that was properly initialized
3322         // server object stay alive but we mark it as invalid so it
3323         // gets initialized again
3324         m_server->markInvalid();
3325         m_server = 0;
3326     }
3327     shutdown();
3328 }
3329
3330 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
3331                                  Yaz_Proxy *parent) :
3332     Z_Assoc (the_PDU_Observable)
3333 {
3334     m_cookie = 0;
3335     m_next = 0;
3336     m_prev = 0;
3337     m_init_flag = 0;
3338     m_last_query = 0;
3339     m_last_resultSetId = 0;
3340     m_last_resultCount = 0;
3341     m_last_ok = 0;
3342     m_sr_transform = 0;
3343     m_waiting = 0;
3344     m_init_odr = odr_createmem (ODR_DECODE);
3345     m_initResponse = 0;
3346     m_initResponse_options = 0;
3347     m_initResponse_version = 0;
3348     m_initResponse_preferredMessageSize = 0;
3349     m_initResponse_maximumRecordSize = 0;
3350     m_resultSetStartPoint = 0;
3351     m_bytes_sent = m_bytes_recv = 0;
3352     m_pdu_recv = 0;
3353     m_server = 0;
3354     m_seqno = 0;
3355     m_target_idletime = 600;
3356     m_root = parent;
3357 }
3358
3359 const char *Yaz_Proxy::option(const char *name, const char *value)
3360 {
3361     if (!strcmp (name, "optimize")) {
3362         if (value) {
3363             xfree (m_optimize);
3364             m_optimize = xstrdup (value);
3365         }
3366         return m_optimize;
3367     }
3368     return 0;
3369 }
3370
3371 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
3372 {
3373
3374 }
3375
3376 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
3377 {
3378     if (apdu->which == Z_GDU_Z3950)
3379         recv_Z_PDU(apdu->u.z3950, len);
3380     else if (apdu->which == Z_GDU_HTTP_Response)
3381         recv_HTTP_response(apdu->u.HTTP_Response, len);
3382     else
3383         shutdown();
3384 }
3385
3386 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
3387 {
3388     if (!m_flag_invalid_session)
3389         return 0;
3390     m_flag_invalid_session = 0;
3391     handle_incoming_Z_PDU(m_apdu_invalid_session);
3392     assert (m_mem_invalid_session);
3393     nmem_destroy(m_mem_invalid_session);
3394     m_mem_invalid_session = 0;
3395     return 1;
3396 }
3397
3398 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
3399 {
3400     m_bytes_recv += len;
3401
3402     m_pdu_recv++;
3403     m_waiting = 0;
3404     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
3405         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
3406                  apdu_name(apdu), get_hostname(), len);
3407     if (apdu->which == Z_APDU_initResponse)
3408     {
3409         if (!m_server)  // if this is a pre init session , check for more
3410             m_root->pre_init();
3411         NMEM nmem = odr_extract_mem (odr_decode());
3412         odr_reset (m_init_odr);
3413         nmem_transfer (m_init_odr->mem, nmem);
3414         m_initResponse = apdu;
3415         m_initResponse_options = apdu->u.initResponse->options;
3416         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3417         m_initResponse_preferredMessageSize =
3418             *apdu->u.initResponse->preferredMessageSize;
3419         m_initResponse_maximumRecordSize =
3420             *apdu->u.initResponse->maximumRecordSize;
3421
3422         Z_InitResponse *ir = apdu->u.initResponse;
3423         char *im0 = ir->implementationName;
3424
3425         char *im1 = (char*)
3426             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3427         *im1 = '\0';
3428         if (im0)
3429         {
3430             strcat(im1, im0);
3431             strcat(im1, " ");
3432         }
3433         strcat(im1, "(YAZ Proxy)");
3434         ir->implementationName = im1;
3435
3436         nmem_destroy (nmem);
3437
3438         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3439             return;
3440     }
3441     if (apdu->which == Z_APDU_searchResponse)
3442     {
3443         Z_SearchResponse *sr = apdu->u.searchResponse;
3444         m_last_resultCount = *sr->resultCount;
3445         int status = *sr->searchStatus;
3446         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3447         {
3448             m_last_ok = 1;
3449
3450             if (sr->records && sr->records->which == Z_Records_DBOSD)
3451             {
3452                 m_cache.add(odr_decode(),
3453                             sr->records->u.databaseOrSurDiagnostics, 1,
3454                             *sr->resultCount);
3455             }
3456         }
3457     }
3458     if (apdu->which == Z_APDU_presentResponse)
3459     {
3460         Z_PresentResponse *pr = apdu->u.presentResponse;
3461         if (m_sr_transform)
3462         {
3463             m_sr_transform = 0;
3464             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3465             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3466             sr->referenceId = pr->referenceId;
3467             *sr->resultCount = m_last_resultCount;
3468             sr->records = pr->records;
3469             sr->nextResultSetPosition = pr->nextResultSetPosition;
3470             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3471             apdu = new_apdu;
3472         }
3473         if (pr->records &&
3474             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3475         {
3476             m_cache.add(odr_decode(),
3477                         pr->records->u.databaseOrSurDiagnostics,
3478                         m_resultSetStartPoint, -1);
3479             m_resultSetStartPoint = 0;
3480         }
3481     }
3482     if (m_cookie)
3483         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3484
3485     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
3486
3487     if (server)
3488         server->send_to_client(apdu);
3489     if (apdu->which == Z_APDU_close)
3490         shutdown();
3491     else if (server)
3492         server->recv_GDU_more(true);
3493 }
3494
3495 void Yaz_Proxy::low_socket_close()
3496 {
3497 #if WIN32
3498 #else
3499     int i;
3500     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3501         if  (m_lo_fd[i] >= 0)
3502             ::close(m_lo_fd[i]);
3503 #endif
3504 }
3505
3506 void Yaz_Proxy::low_socket_open()
3507 {
3508 #if WIN32
3509 #else
3510     int i;
3511     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3512         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3513 #endif
3514 }
3515
3516 int Yaz_Proxy::server(const char *addr)
3517 {
3518     int r = Z_Assoc::server(addr);
3519     if (!r)
3520     {
3521         yaz_log(YLOG_LOG, "%sStarted proxy "
3522 #ifdef VERSION
3523             VERSION
3524 #endif
3525             " on %s", m_session_str, addr);
3526         timeout(1);
3527     }
3528     return r;
3529 }
3530
3531 /*
3532  * Local variables:
3533  * c-basic-offset: 4
3534  * indent-tabs-mode: nil
3535  * End:
3536  * vim: shiftwidth=4 tabstop=8 expandtab
3537  */
3538