src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.45 2006-03-30 10:35:15 adam Exp $
   2    Copyright (c) 1998-2006, Index Data.
   3
   4 This file is part of the yazproxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <stdlib.h>
  42 #include <time.h>
  43 #include <fcntl.h>
  44
  45 #include <yaz/srw.h>
  46 #include <yaz/marcdisp.h>
  47 #include <yaz/yaz-iconv.h>
  48 #include <yaz/log.h>
  49 #include <yaz/diagbib1.h>
  50 #include "proxyp.h"
  51 #include <yaz/pquery.h>
  52 #include <yaz/otherinfo.h>
  53 #include <yaz/charneg.h>
  54 #include "msg-thread.h"
  55
  56 using namespace yazpp_1;
  57
  58 #ifdef WIN32
  59 #define strncasecmp _strnicmp
  60 #endif
  61
  62 #define USE_AUTH_MSG 1
  63
  64 #if USE_AUTH_MSG
  65 class Auth_Msg : public IMsg_Thread {
  66 public:
  67     int m_ret;
  68     IMsg_Thread *handle();
  69     void result();
  70     Yaz_Proxy *m_proxy;
  71     NMEM m_nmem;
  72     char *m_apdu_buf;
  73     int m_apdu_len;
  74     Auth_Msg();
  75     virtual ~Auth_Msg();
  76 };
  77
  78 Auth_Msg::Auth_Msg()
  79 {
  80     m_nmem = nmem_create();
  81 }
  82
  83 Auth_Msg::~Auth_Msg()
  84 {
  85     nmem_destroy(m_nmem);
  86 }
  87
  88 IMsg_Thread *Auth_Msg::handle()
  89 {
  90     ODR decode = odr_createmem(ODR_DECODE);
  91     Z_APDU *apdu;
  92
  93     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
  94     int r = z_APDU(decode, &apdu, 0, 0);
  95     if (!r)
  96     {
  97         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
  98     }
  99     else
 100     {
 101         m_ret = m_proxy->handle_authentication(apdu);
 102     }
 103     odr_destroy(decode);
 104     return this;
 105 }
 106
 107 void Auth_Msg::result()
 108 {
 109     if (m_proxy->dec_ref(false))
 110     {
 111         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
 112     }
 113     else
 114     {
 115         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
 116         Z_APDU *apdu = 0;
 117         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
 118         if (!r)
 119             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
 120         m_proxy->result_authentication(apdu, m_ret);
 121     }
 122     delete this;
 123 }
 124
 125 #endif
 126
 127 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
 128 {
 129     if (apdu == 0 || ret == 0)
 130     {
 131         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
 132         *apdu_reject->u.initResponse->result = 0;
 133         send_to_client(apdu_reject);
 134         dec_ref(false);
 135     }
 136     else
 137     {
 138         if (apdu->which == Z_APDU_initRequest)
 139         {
 140             Yaz_ProxyConfig *cfg = check_reconfigure();
 141             if (cfg)
 142                 cfg->target_authentication(m_default_target, odr_encode(),
 143                                            apdu->u.initRequest);
 144         }
 145         handle_incoming_Z_PDU_2(apdu);
 146     }
 147 }
 148
 149 static const char *apdu_name(Z_APDU *apdu)
 150 {
 151     switch (apdu->which)
 152     {
 153     case Z_APDU_initRequest:
 154         return "initRequest";
 155     case Z_APDU_initResponse:
 156         return "initResponse";
 157     case Z_APDU_searchRequest:
 158         return "searchRequest";
 159     case Z_APDU_searchResponse:
 160         return "searchResponse";
 161     case Z_APDU_presentRequest:
 162         return "presentRequest";
 163     case Z_APDU_presentResponse:
 164         return "presentResponse";
 165     case Z_APDU_deleteResultSetRequest:
 166         return "deleteResultSetRequest";
 167     case Z_APDU_deleteResultSetResponse:
 168         return "deleteResultSetResponse";
 169     case Z_APDU_scanRequest:
 170         return "scanRequest";
 171     case Z_APDU_scanResponse:
 172         return "scanResponse";
 173     case Z_APDU_sortRequest:
 174         return "sortRequest";
 175     case Z_APDU_sortResponse:
 176         return "sortResponse";
 177     case Z_APDU_extendedServicesRequest:
 178         return "extendedServicesRequest";
 179     case Z_APDU_extendedServicesResponse:
 180         return "extendedServicesResponse";
 181     case Z_APDU_close:
 182         return "close";
 183     }
 184     return "other";
 185 }
 186
 187 static const char *gdu_name(Z_GDU *gdu)
 188 {
 189     switch(gdu->which)
 190     {
 191     case Z_GDU_Z3950:
 192         return apdu_name(gdu->u.z3950);
 193     case Z_GDU_HTTP_Request:
 194         return "HTTP Request";
 195     case Z_GDU_HTTP_Response:
 196         return "HTTP Response";
 197     }
 198     return "Unknown request/response";
 199 }
 200
 201 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
 202                      ISocketObservable *the_socket_observable,
 203                      Yaz_Proxy *parent)
 204     :
 205     Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 206 {
 207     m_PDU_Observable = the_PDU_Observable;
 208     m_socket_observable = the_socket_observable;
 209     m_client = 0;
 210     m_parent = parent;
 211     m_clientPool = 0;
 212     m_seqno = 1;
 213     m_keepalive_limit_bw = 500000;
 214     m_keepalive_limit_pdu = 1000;
 215     m_proxyTarget = 0;
 216     m_default_target = 0;
 217     m_proxy_negotiation_charset = 0;
 218     m_proxy_negotiation_lang = 0;
 219     m_proxy_negotiation_default_charset = 0;
 220     m_charset_converter = new Yaz_CharsetConverter;
 221     m_max_clients = 150;
 222     m_log_mask = 0;
 223     m_seed = time(0);
 224     m_client_idletime = 600;
 225     m_target_idletime = 600;
 226     m_optimize = xstrdup ("1");
 227     strcpy(m_session_str, "0 ");
 228     m_session_no = 0;
 229     m_bytes_sent = 0;
 230     m_bytes_recv = 0;
 231     m_bw_max = 0;
 232     m_pdu_max = 0;
 233     m_search_max = 0;
 234     m_connect_max = 0;
 235     m_timeout_mode = timeout_normal;
 236     m_timeout_gdu = 0;
 237     m_max_record_retrieve = 0;
 238     m_reconfig_flag = 0;
 239     m_config_fname = 0;
 240     m_request_no = 0;
 241     m_flag_invalid_session = 0;
 242     m_referenceId = 0;
 243     m_referenceId_mem = nmem_create();
 244     m_config = 0;
 245     m_marcxml_mode = none;
 246     m_stylesheet_xsp = 0;
 247     m_stylesheet_nprl = 0;
 248     m_stylesheet_apdu = 0;
 249     m_s2z_stylesheet = 0;
 250     m_s2z_database = 0;
 251     m_schema = 0;
 252     m_backend_type = 0;
 253     m_backend_charset = 0;
 254     m_frontend_type = 0;
 255     m_initRequest_apdu = 0;
 256     m_initRequest_mem = 0;
 257     m_initRequest_preferredMessageSize = 0;
 258     m_initRequest_maximumRecordSize = 0;
 259     m_initRequest_options = 0;
 260     m_initRequest_version = 0;
 261     m_initRequest_oi_negotiation_charsets = 0;
 262     m_initRequest_oi_negotiation_num_charsets = 0;
 263     m_initRequest_oi_negotiation_langs = 0;
 264     m_initRequest_oi_negotiation_num_langs = 0;
 265     m_initRequest_oi_negotiation_selected = 0;
 266     m_apdu_invalid_session = 0;
 267     m_mem_invalid_session = 0;
 268     m_s2z_odr_init = 0;
 269     m_s2z_odr_search = 0;
 270     m_s2z_init_apdu = 0;
 271     m_s2z_search_apdu = 0;
 272     m_s2z_present_apdu = 0;
 273     m_http_keepalive = 0;
 274     m_http_version = 0;
 275     m_soap_ns = 0;
 276     m_s2z_packing = Z_SRW_recordPacking_string;
 277 #if HAVE_GETTIMEOFDAY
 278     m_time_tv = xmalloc(sizeof(struct timeval));
 279     struct timeval *tv = (struct timeval *) m_time_tv;
 280     tv->tv_sec = 0;
 281     tv->tv_usec = 0;
 282 #else
 283     m_time_tv = 0;
 284 #endif
 285     m_usemarcon_ini_stage1 = 0;
 286     m_usemarcon_ini_stage2 = 0;
 287     m_usemarcon = new Yaz_usemarcon();
 288     if (!m_parent)
 289         low_socket_open();
 290     m_my_thread = 0;
 291     m_ref_count = 1;
 292     m_main_ptr_dec = false;
 293     m_peername = 0;
 294 }
 295
 296 void Yaz_Proxy::inc_ref()
 297 {
 298     m_ref_count++;
 299 }
 300
 301 Yaz_Proxy::~Yaz_Proxy()
 302 {
 303     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 304             m_bytes_sent, m_bytes_recv);
 305     nmem_destroy(m_initRequest_mem);
 306     nmem_destroy(m_mem_invalid_session);
 307     nmem_destroy(m_referenceId_mem);
 308
 309     xfree(m_proxyTarget);
 310     xfree(m_default_target);
 311     xfree(m_proxy_negotiation_charset);
 312     xfree(m_proxy_negotiation_lang);
 313     xfree(m_proxy_negotiation_default_charset);
 314     delete m_charset_converter;
 315     xfree(m_optimize);
 316
 317 #if HAVE_XSLT
 318     if (m_stylesheet_xsp)
 319         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 320 #endif
 321     xfree (m_time_tv);
 322
 323     xfree (m_peername);
 324     xfree (m_schema);
 325     xfree (m_backend_type);
 326     xfree (m_backend_charset);
 327     xfree (m_usemarcon_ini_stage1);
 328     xfree (m_usemarcon_ini_stage2);
 329     delete m_usemarcon;
 330     if (m_s2z_odr_init)
 331         odr_destroy(m_s2z_odr_init);
 332     if (m_s2z_odr_search)
 333         odr_destroy(m_s2z_odr_search);
 334     if (!m_parent)
 335         low_socket_close();
 336     if (!m_parent)
 337         delete m_my_thread;
 338     delete m_config;
 339 }
 340
 341 void Yaz_Proxy::set_debug_mode(int mode)
 342 {
 343     m_debug_mode = mode;
 344 }
 345
 346 int Yaz_Proxy::set_config(const char *config)
 347 {
 348     delete m_config;
 349     m_config = new Yaz_ProxyConfig();
 350     xfree(m_config_fname);
 351     m_config_fname = xstrdup(config);
 352     int r = m_config->read_xml(config);
 353     if (!r)
 354         m_config->get_generic_info(&m_log_mask, &m_max_clients,
 355                                    &m_connect_max);
 356     return r;
 357 }
 358
 359 void Yaz_Proxy::set_default_target(const char *target)
 360 {
 361     xfree (m_default_target);
 362     m_default_target = 0;
 363     if (target)
 364         m_default_target = (char *) xstrdup (target);
 365 }
 366
 367 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang,
 368                                        const char *default_charset)
 369 {
 370     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
 371         "default charset to '%s', language to '%s'", m_session_str,
 372         charset?charset:"none",
 373         default_charset?default_charset:"none",
 374         lang?lang:"none");
 375     xfree (m_proxy_negotiation_charset);
 376     xfree (m_proxy_negotiation_lang);
 377     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 378     if (charset)
 379         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 380     if (lang)
 381         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 382     if (default_charset)
 383         m_proxy_negotiation_default_charset =
 384             (char *) xstrdup (default_charset);
 385 }
 386
 387 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 388 {
 389     if (m_parent)
 390         return m_parent->check_reconfigure();
 391
 392     Yaz_ProxyConfig *cfg = m_config;
 393     if (m_reconfig_flag)
 394     {
 395         yaz_log(YLOG_LOG, "reconfigure");
 396         yaz_log_reopen();
 397         if (m_config_fname && cfg)
 398         {
 399             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 400             int r = cfg->read_xml(m_config_fname);
 401             if (r)
 402                 yaz_log(YLOG_WARN, "reconfigure failed");
 403             else
 404             {
 405                 m_log_mask = 0;
 406                 cfg->get_generic_info(&m_log_mask, &m_max_clients,
 407                                       &m_connect_max);
 408             }
 409         }
 410         else
 411             yaz_log(YLOG_LOG, "reconfigure");
 412         m_reconfig_flag = 0;
 413     }
 414     return cfg;
 415 }
 416
 417 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 418                                         *the_PDU_Observable, int fd)
 419 {
 420     check_reconfigure();
 421     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 422                                          m_socket_observable, this);
 423     new_proxy->m_config = 0;
 424     new_proxy->m_config_fname = 0;
 425     new_proxy->timeout(m_client_idletime);
 426     new_proxy->m_target_idletime = m_target_idletime;
 427     new_proxy->set_default_target(m_default_target);
 428     new_proxy->m_max_clients = m_max_clients;
 429     new_proxy->m_log_mask = m_log_mask;
 430     new_proxy->set_APDU_log(get_APDU_log());
 431     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 432         new_proxy->set_APDU_yazlog(1);
 433     else
 434         new_proxy->set_APDU_yazlog(0);
 435     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 436     m_session_no++;
 437     new_proxy->m_peername = xstrdup(the_PDU_Observable->getpeername());
 438     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 439              new_proxy->m_peername);
 440     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 441         m_proxy_negotiation_lang, m_proxy_negotiation_default_charset);
 442     // create thread object the first time we get an incoming connection
 443     if (!m_my_thread)
 444         m_my_thread = new Msg_Thread(m_socket_observable, 1);
 445     new_proxy->m_my_thread = m_my_thread;
 446     return new_proxy;
 447 }
 448
 449 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 450 {
 451     int oid[OID_SIZE];
 452     Z_OtherInformationUnit *oi;
 453     struct oident ent;
 454     ent.proto = PROTO_Z3950;
 455     ent.oclass = CLASS_USERINFO;
 456     ent.value = (oid_value) VAL_COOKIE;
 457     assert (oid_ent_to_oid (&ent, oid));
 458
 459     if (oid_ent_to_oid (&ent, oid) &&
 460         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 461         oi->which == Z_OtherInfo_characterInfo)
 462         return oi->information.characterInfo;
 463     return 0;
 464 }
 465 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 466 {
 467     int oid[OID_SIZE];
 468     Z_OtherInformationUnit *oi;
 469     struct oident ent;
 470     ent.proto = PROTO_Z3950;
 471     ent.oclass = CLASS_USERINFO;
 472     ent.value = (oid_value) VAL_PROXY;
 473     if (oid_ent_to_oid (&ent, oid) &&
 474         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 475         oi->which == Z_OtherInfo_characterInfo)
 476         return oi->information.characterInfo;
 477     return 0;
 478 }
 479 const char *Yaz_Proxy::load_balance(const char **url)
 480 {
 481     int zurl_in_use[MAX_ZURL_PLEX];
 482     int zurl_in_spare[MAX_ZURL_PLEX];
 483     Yaz_ProxyClient *c;
 484     int i;
 485
 486     for (i = 0; i<MAX_ZURL_PLEX; i++)
 487     {
 488         zurl_in_use[i] = 0;
 489         zurl_in_spare[i] = 0;
 490     }
 491     for (c = m_parent->m_clientPool; c; c = c->m_next)
 492     {
 493         for (i = 0; url[i]; i++)
 494             if (!strcmp(url[i], c->get_hostname()))
 495             {
 496                 zurl_in_use[i]++;
 497                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 498                     zurl_in_spare[i]++;
 499             }
 500     }
 501     int min_use = 100000;
 502     int spare_for_min = 0;
 503     int max_spare = 0;
 504     const char *ret_min = 0;
 505     const char *ret_spare = 0;
 506     for (i = 0; url[i]; i++)
 507     {
 508         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 509                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 510         if (min_use > zurl_in_use[i])
 511         {
 512             ret_min = url[i];
 513             min_use = zurl_in_use[i];
 514             spare_for_min = zurl_in_spare[i];
 515         }
 516         if (max_spare < zurl_in_spare[i])
 517         {
 518             ret_spare = url[i];
 519             max_spare = zurl_in_spare[i];
 520         }
 521     }
 522     return ret_min;
 523 }
 524
 525 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 526                                        const char *proxy_host)
 527 {
 528     assert (m_parent);
 529     Yaz_Proxy *parent = m_parent;
 530     Yaz_ProxyClient *c = m_client;
 531
 532     if (!m_proxyTarget)
 533     {
 534         const char *url[MAX_ZURL_PLEX];
 535         Yaz_ProxyConfig *cfg = check_reconfigure();
 536         if (proxy_host)
 537         {
 538             if (parent && parent->m_debug_mode)
 539             {
 540                 // only to be enabled for debugging...
 541                 if (!strcmp(proxy_host, "stop"))
 542                     exit(0);
 543             }
 544             xfree(m_default_target);
 545             m_default_target = xstrdup(proxy_host);
 546         }
 547         proxy_host = m_default_target;
 548         int client_idletime = -1;
 549         const char *cql2rpn_fname = 0;
 550         const char *negotiation_charset = 0;
 551         const char *negotiation_lang = 0;
 552         const char *query_charset = 0;
 553         const char *default_client_query_charset = 0;
 554         url[0] = m_default_target;
 555         url[1] = 0;
 556         if (cfg)
 557         {
 558             int pre_init = 0;
 559             cfg->get_target_info(proxy_host, url, &m_bw_max,
 560                                  &m_pdu_max, &m_max_record_retrieve,
 561                                  &m_search_max,
 562                                  &m_target_idletime, &client_idletime,
 563                                  &parent->m_max_clients,
 564                                  &m_keepalive_limit_bw,
 565                                  &m_keepalive_limit_pdu,
 566                                  &pre_init,
 567                                  &cql2rpn_fname,
 568                                  &negotiation_charset,
 569                                  &negotiation_lang,
 570                                  &query_charset,
 571                                  &default_client_query_charset);
 572         }
 573         if (client_idletime != -1)
 574         {
 575             m_client_idletime = client_idletime;
 576             timeout(m_client_idletime);
 577         }
 578         if (cql2rpn_fname)
 579             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 580         if (negotiation_charset || negotiation_lang || default_client_query_charset)
 581         {
 582             set_proxy_negotiation(negotiation_charset,
 583                 negotiation_lang, default_client_query_charset);
 584         }
 585         m_charset_converter->set_target_query_charset(query_charset);
 586         if (!url[0])
 587         {
 588             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 589             return 0;
 590         }
 591         // we don't handle multiplexing for cookie session, so we just
 592         // pick the first one in this case (anonymous users will be able
 593         // to use any backend)
 594         if (cookie && *cookie)
 595             m_proxyTarget = (char*) xstrdup(url[0]);
 596         else
 597             m_proxyTarget = (char*) xstrdup(load_balance(url));
 598     }
 599     if (cookie && *cookie)
 600     {   // search in sessions with a cookie
 601         for (c = parent->m_clientPool; c; c = c->m_next)
 602         {
 603             assert (c->m_prev);
 604             assert (*c->m_prev == c);
 605             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 606                 !strcmp(m_proxyTarget, c->get_hostname()))
 607             {
 608                 // Found it in cache
 609                 // The following handles "cancel"
 610                 // If connection is busy (waiting for PDU) and
 611                 // we have an initRequest we can safely do re-open
 612                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 613                 {
 614                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 615                              c->get_hostname());
 616                     c->close();
 617                     c->m_init_flag = 0;
 618
 619                     c->m_last_ok = 0;
 620                     c->m_cache.clear();
 621                     c->m_last_resultCount = 0;
 622                     c->m_sr_transform = 0;
 623                     c->m_waiting = 0;
 624                     c->m_resultSetStartPoint = 0;
 625                     c->m_target_idletime = m_target_idletime;
 626                     if (c->client(m_proxyTarget))
 627                     {
 628                         delete c;
 629                         return 0;
 630                     }
 631                     c->timeout(30);
 632                 }
 633                 c->m_seqno = parent->m_seqno;
 634                 if (c->m_server && c->m_server != this)
 635                     c->m_server->m_client = 0;
 636                 c->m_server = this;
 637                 (parent->m_seqno)++;
 638                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 639                 return c;
 640             }
 641         }
 642     }
 643     else if (!c && apdu->which == Z_APDU_initRequest )
 644     {
 645         // anonymous sessions without cookie.
 646         // if authentication is set it is NOT anonymous se we can't share them.
 647         // If charset and lang negotiation is use it is NOT anonymous session too.
 648         for (c = parent->m_clientPool; c; c = c->m_next)
 649         {
 650             assert(c->m_prev);
 651             assert(*c->m_prev == c);
 652             if (c->m_server == 0 && c->m_cookie == 0 &&  c->m_waiting == 0
 653                 && c->compare_idAuthentication(apdu)
 654                 && c->compare_charset(apdu)
 655                 && !strcmp(m_proxyTarget, c->get_hostname()))
 656             {
 657                 // found it in cache
 658                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 659                          m_session_str, parent->m_seqno, c->get_hostname());
 660
 661                 c->m_seqno = parent->m_seqno;
 662                 assert(c->m_server == 0);
 663                 c->m_server = this;
 664
 665                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 666                     c->set_APDU_yazlog(1);
 667                 else
 668                     c->set_APDU_yazlog(0);
 669
 670                 (parent->m_seqno)++;
 671
 672                 parent->pre_init();
 673
 674                 return c;
 675             }
 676         }
 677     }
 678     if (!m_client)
 679     {
 680         if (apdu->which != Z_APDU_initRequest)
 681         {
 682             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 683             return 0;
 684         }
 685         // go through list of clients - and find the lowest/oldest one.
 686         Yaz_ProxyClient *c_min = 0;
 687         int min_seq = -1;
 688         int no_of_clients = 0;
 689         if (parent->m_clientPool)
 690             yaz_log (YLOG_DEBUG, "Existing sessions");
 691         for (c = parent->m_clientPool; c; c = c->m_next)
 692         {
 693             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 694                                c->m_waiting, c->get_hostname(),
 695                                c->m_cookie ? c->m_cookie : "");
 696             no_of_clients++;
 697             if (min_seq < 0 || c->m_seqno < min_seq)
 698             {
 699                 min_seq = c->m_seqno;
 700                 c_min = c;
 701             }
 702         }
 703         if (no_of_clients >= parent->m_max_clients)
 704         {
 705             c = c_min;
 706             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 707             {
 708                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 709                          m_session_str, parent->m_max_clients, c->m_seqno);
 710                 if (c->m_server && c->m_server != this)
 711                     delete c->m_server;   // PROBLEM: m_ref_count!
 712                 c->m_server = 0;
 713             }
 714             else
 715             {
 716                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 717                          m_session_str, parent->m_max_clients,
 718                          c->m_seqno, parent->m_seqno, c->get_hostname());
 719                 xfree (c->m_cookie);
 720                 c->m_cookie = 0;
 721                 if (cookie)
 722                     c->m_cookie = xstrdup(cookie);
 723                 c->m_seqno = parent->m_seqno;
 724                 if (c->m_server && c->m_server != this)
 725                 {
 726                     c->m_server->m_client = 0;
 727                     delete c->m_server;   // PROBLEM: m_ref_count!
 728                 }
 729                 (parent->m_seqno)++;
 730                 c->m_target_idletime = m_target_idletime;
 731                 c->timeout(m_target_idletime);
 732
 733                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 734                     c->set_APDU_yazlog(1);
 735                 else
 736                     c->set_APDU_yazlog(0);
 737
 738                 return c;
 739             }
 740         }
 741         else
 742         {
 743             yaz_log (YLOG_LOG, "%sNEW %d %s",
 744                      m_session_str, parent->m_seqno, m_proxyTarget);
 745             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 746             c->m_next = parent->m_clientPool;
 747             if (c->m_next)
 748                 c->m_next->m_prev = &c->m_next;
 749             parent->m_clientPool = c;
 750             c->m_prev = &parent->m_clientPool;
 751         }
 752
 753         xfree (c->m_cookie);
 754         c->m_cookie = 0;
 755         if (cookie)
 756             c->m_cookie = xstrdup(cookie);
 757
 758         c->m_seqno = parent->m_seqno;
 759         c->m_init_flag = 0;
 760         c->m_last_resultCount = 0;
 761         c->m_last_ok = 0;
 762         c->m_cache.clear();
 763         c->m_sr_transform = 0;
 764         c->m_waiting = 0;
 765         c->m_resultSetStartPoint = 0;
 766         (parent->m_seqno)++;
 767         if (c->client(m_proxyTarget))
 768         {
 769             delete c;
 770             return 0;
 771         }
 772         c->m_target_idletime = m_target_idletime;
 773         c->timeout(30);
 774
 775         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 776             c->set_APDU_yazlog(1);
 777         else
 778             c->set_APDU_yazlog(0);
 779
 780         c->set_idAuthentication(apdu);
 781     }
 782     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 783     return c;
 784 }
 785
 786 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 787 {
 788     int i;
 789     for (i = 0; i<num; i++)
 790     {
 791         oident *ent;
 792         Z_DefaultDiagFormat *r;
 793         Z_DiagRec *p = pp[i];
 794         if (p->which != Z_DiagRec_defaultFormat)
 795         {
 796             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 797             return;
 798         }
 799         else
 800             r = p->u.defaultFormat;
 801         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 802             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 803             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 804         switch (r->which)
 805         {
 806         case Z_DefaultDiagFormat_v2Addinfo:
 807             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 808                     m_session_str,
 809                     *r->condition, diagbib1_str(*r->condition),
 810                     r->u.v2Addinfo);
 811             break;
 812         case Z_DefaultDiagFormat_v3Addinfo:
 813             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 814                     m_session_str,
 815                     *r->condition, diagbib1_str(*r->condition),
 816                     r->u.v3Addinfo);
 817             break;
 818         }
 819     }
 820 }
 821
 822 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 823 {
 824     if (!m_stylesheet_xsp || p->num_records <= 0)
 825     {
 826         return 0;  /* no XSLT to be done ... */
 827     }
 828
 829     m_stylesheet_offset = 0;
 830     m_stylesheet_nprl = p;
 831     m_stylesheet_apdu = apdu;
 832     m_timeout_mode = timeout_xsl;
 833
 834     timeout(0);
 835     return 1;
 836 }
 837
 838 void Yaz_Proxy::convert_xsl_delay()
 839 {
 840 #if HAVE_XSLT
 841     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 842     if (npr->which == Z_NamePlusRecord_databaseRecord)
 843     {
 844         Z_External *r = npr->u.databaseRecord;
 845         if (r->which == Z_External_octet)
 846         {
 847 #if 0
 848             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 849 #endif
 850             xmlDocPtr res, doc = xmlParseMemory(
 851                 (char*) r->u.octet_aligned->buf,
 852                 r->u.octet_aligned->len);
 853
 854
 855             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 856                     m_session_str, m_stylesheet_offset);
 857             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 858                                       doc, 0);
 859
 860             if (res)
 861             {
 862                 xmlChar *out_buf;
 863                 int out_len;
 864                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 865
 866                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 867                     u.databaseRecord =
 868                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 869                                  (char*) out_buf, out_len);
 870                 xmlFree(out_buf);
 871                 xmlFreeDoc(res);
 872             }
 873
 874             xmlFreeDoc(doc);
 875         }
 876     }
 877 #endif
 878     m_stylesheet_offset++;
 879     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 880     {
 881         m_timeout_mode = timeout_normal;
 882         m_stylesheet_nprl = 0;
 883 #if HAVE_XSLT
 884         if (m_stylesheet_xsp)
 885             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 886 #endif
 887         m_stylesheet_xsp = 0;
 888         timeout(m_client_idletime);
 889         send_PDU_convert(m_stylesheet_apdu);
 890     }
 891     else
 892         timeout(0);
 893 }
 894
 895 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 896 {
 897     if (m_frontend_type != VAL_NONE)
 898     {
 899         int i;
 900         for (i = 0; i < p->num_records; i++)
 901         {
 902             Z_NamePlusRecord *npr = p->records[i];
 903             if (npr->which == Z_NamePlusRecord_databaseRecord)
 904             {
 905                 Z_External *r = npr->u.databaseRecord;
 906                 if (r->which == Z_External_octet)
 907                 {
 908 #if HAVE_USEMARCON
 909                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 910                     {
 911                         if (!m_usemarcon->m_stage1)
 912                         {
 913                             m_usemarcon->m_stage1 = new CDetails();
 914                         }
 915                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 916                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 917                         int res = m_usemarcon->m_stage1->Start();
 918                         if (res == 0)
 919                         {
 920                             char *converted;
 921                             int convlen;
 922                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 923                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 924                             {
 925                                 if (!m_usemarcon->m_stage2)
 926                                 {
 927                                     m_usemarcon->m_stage2 = new CDetails();
 928                                 }
 929                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 930                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 931                                 res = m_usemarcon->m_stage2->Start();
 932                                 if (res == 0)
 933                                 {
 934                                     free(converted);
 935                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 936                                 }
 937                                 else
 938                                 {
 939                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 940                                 }
 941                             }
 942                             npr->u.databaseRecord =
 943                                 z_ext_record(odr_encode(),
 944                                              m_frontend_type,
 945                                              converted,
 946                                              strlen(converted));
 947                             free(converted);
 948                         }
 949                         else
 950                         {
 951                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 952                         }
 953                         continue;
 954                     }
 955 #endif
 956 /* HAVE_USEMARCON */
 957                     npr->u.databaseRecord =
 958                         z_ext_record(odr_encode(),
 959                                      m_frontend_type,
 960                                      (char*) r->u.octet_aligned->buf,
 961                                      r->u.octet_aligned->len);
 962                 }
 963             }
 964         }
 965     }
 966 }
 967
 968 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
 969                                         const char *backend_charset)
 970 {
 971     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
 972     int sel =   m_charset_converter->get_client_charset_selected();
 973     const char *client_record_charset =
 974         m_charset_converter->get_client_query_charset();
 975     if (sel && backend_charset && client_record_charset &&
 976         strcmp(backend_charset, client_record_charset))
 977     {
 978         int i;
 979         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
 980                                         backend_charset);
 981         yaz_marc_t mt = yaz_marc_create();
 982         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
 983         yaz_marc_iconv(mt, cd);
 984         for (i = 0; i < p->num_records; i++)
 985         {
 986             Z_NamePlusRecord *npr = p->records[i];
 987             if (npr->which == Z_NamePlusRecord_databaseRecord)
 988             {
 989                 Z_External *r = npr->u.databaseRecord;
 990                 oident *ent = oid_getentbyoid(r->direct_reference);
 991                 if (!ent || ent->value == VAL_NONE)
 992                     continue;
 993
 994                 if (ent->value == VAL_SUTRS)
 995                 {
 996                     WRBUF w = wrbuf_alloc();
 997
 998                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
 999                                       r->u.octet_aligned->len);
1000                     npr->u.databaseRecord =
1001                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
1002                                      wrbuf_len(w));
1003                     wrbuf_free(w, 1);
1004                 }
1005                 else if (ent->value == VAL_TEXT_XML)
1006                 {
1007                     ;
1008                 }
1009                 else if (r->which == Z_External_octet)
1010                 {
1011                     int rlen;
1012                     char *result;
1013                     if (yaz_marc_decode_buf(mt,
1014                                             (char*) r->u.octet_aligned->buf,
1015                                             r->u.octet_aligned->len,
1016                                             &result, &rlen))
1017                     {
1018                         npr->u.databaseRecord =
1019                             z_ext_record(odr_encode(), ent->value, result, rlen);
1020                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1021                                 m_session_str);
1022                     }
1023                 }
1024             }
1025         }
1026         if (cd)
1027             yaz_iconv_close(cd);
1028         yaz_marc_destroy(mt);
1029     }
1030     else
1031     {
1032         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
1033     }
1034 }
1035
1036 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1037                                    const char *backend_charset)
1038 {
1039     int i;
1040     if (!backend_charset)
1041         backend_charset = "MARC-8";
1042     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1043     yaz_marc_t mt = yaz_marc_create();
1044     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1045     yaz_marc_iconv(mt, cd);
1046     for (i = 0; i < p->num_records; i++)
1047     {
1048         Z_NamePlusRecord *npr = p->records[i];
1049         if (npr->which == Z_NamePlusRecord_databaseRecord)
1050         {
1051             Z_External *r = npr->u.databaseRecord;
1052             if (r->which == Z_External_OPAC)
1053             {
1054                 WRBUF w = wrbuf_alloc();
1055
1056                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
1057                 npr->u.databaseRecord = z_ext_record(
1058                     odr_encode(), VAL_TEXT_XML,
1059                     wrbuf_buf(w), wrbuf_len(w)
1060                     );
1061                 wrbuf_free(w, 1);
1062             }
1063             else if (r->which == Z_External_octet)
1064             {
1065                 int rlen;
1066                 char *result;
1067                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1068                                         r->u.octet_aligned->len,
1069                                         &result, &rlen))
1070                 {
1071                     npr->u.databaseRecord =
1072                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
1073                 }
1074             }
1075         }
1076     }
1077     if (cd)
1078         yaz_iconv_close(cd);
1079     yaz_marc_destroy(mt);
1080 }
1081
1082 void Yaz_Proxy::logtime()
1083 {
1084 #if HAVE_GETTIMEOFDAY
1085     struct timeval *tv = (struct timeval*) m_time_tv;
1086     if (tv->tv_sec)
1087     {
1088         struct timeval tv1;
1089         gettimeofday(&tv1, 0);
1090         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1091             (tv1.tv_usec - tv->tv_usec);
1092         if (diff >= 0)
1093             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1094                     diff/1000000, (diff/1000)%1000);
1095     }
1096     tv->tv_sec = 0;
1097     tv->tv_usec = 0;
1098 #endif
1099 }
1100
1101 int Yaz_Proxy::send_http_response(int code)
1102 {
1103     ODR o = odr_encode();
1104     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1105     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1106     if (m_http_version)
1107         hres->version = odr_strdup(o, m_http_version);
1108     if (m_http_keepalive)
1109         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1110     else
1111         timeout(0);
1112
1113     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1114     {
1115         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1116                  gdu_name(gdu));
1117     }
1118     int len;
1119     int r = send_GDU(gdu, &len);
1120     m_bytes_sent += len;
1121     m_bw_stat.add_bytes(len);
1122     logtime();
1123
1124     recv_GDU_more(true);
1125
1126     return r;
1127 }
1128
1129 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */)
1130 {
1131     ODR o = odr_encode();
1132     const char *ctype = "text/xml";
1133     Z_GDU *gdu = z_get_HTTP_Response(o, http_code);
1134     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1135     if (m_http_version)
1136         hres->version = odr_strdup(o, m_http_version);
1137     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1138     if (m_http_keepalive)
1139         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1140     else
1141         timeout(0);
1142     if (http_code == 401)
1143         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\"");
1144
1145     static Z_SOAP_Handler soap_handlers[2] = {
1146 #if HAVE_XSLT
1147         {"http://www.loc.gov/zing/srw/", 0,
1148          (Z_SOAP_fun) yaz_srw_codec},
1149 #endif
1150         {0, 0, 0}
1151     };
1152
1153     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1154     soap_package->which = Z_SOAP_generic;
1155     soap_package->u.generic =
1156         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1157     soap_package->u.generic->no = 0;
1158     soap_package->u.generic->ns = soap_handlers[0].ns;
1159     soap_package->u.generic->p = (void *) srw_pdu;
1160     soap_package->ns = m_soap_ns;
1161     z_soap_codec_enc_xsl(o, &soap_package,
1162                          &hres->content_buf, &hres->content_len,
1163                          soap_handlers, 0, m_s2z_stylesheet);
1164     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1165     {
1166         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1167                  gdu_name(gdu));
1168     }
1169     int len;
1170     int r = send_GDU(gdu, &len);
1171     m_bytes_sent += len;
1172     m_bw_stat.add_bytes(len);
1173     logtime();
1174
1175     recv_GDU_more(true);
1176
1177     return r;
1178 }
1179
1180 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1181 {
1182     ODR o = odr_encode();
1183     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1184         odr_malloc(o, sizeof(*diagnostic));
1185     int num_diagnostic = 1;
1186     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1187     return send_srw_search_response(diagnostic, num_diagnostic, srw_error == 3 ? 401 : 200);
1188 }
1189
1190 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1191                              Z_DefaultDiagFormat *ddf)
1192 {
1193     int bib1_code = *ddf->condition;
1194     if (bib1_code == 109)
1195         return 404;
1196     srw_res->num_diagnostics = 1;
1197     srw_res->diagnostics = (Z_SRW_diagnostic *)
1198         odr_malloc(o, sizeof(*srw_res->diagnostics));
1199     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1200                           yaz_diag_bib1_to_srw(*ddf->condition),
1201                           ddf->u.v2Addinfo);
1202     return 0;
1203 }
1204
1205 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1206 {
1207     ODR o = odr_encode();
1208     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1209     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1210
1211     srw_res->numberOfRecords = odr_intdup (o, hits);
1212     if (records && records->which == Z_Records_DBOSD)
1213     {
1214         srw_res->num_records =
1215             records->u.databaseOrSurDiagnostics->num_records;
1216         int i;
1217         srw_res->records = (Z_SRW_record *)
1218             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1219         for (i = 0; i < srw_res->num_records; i++)
1220         {
1221             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1222             if (npr->which != Z_NamePlusRecord_databaseRecord)
1223             {
1224                 srw_res->records[i].recordSchema = "diagnostic";
1225                 srw_res->records[i].recordPacking = m_s2z_packing;
1226                 srw_res->records[i].recordData_buf = "67";
1227                 srw_res->records[i].recordData_len = 2;
1228                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1229                 continue;
1230             }
1231             Z_External *r = npr->u.databaseRecord;
1232             oident *ent = oid_getentbyoid(r->direct_reference);
1233             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1234             {
1235                 srw_res->records[i].recordSchema = m_schema;
1236                 srw_res->records[i].recordPacking = m_s2z_packing;
1237                 srw_res->records[i].recordData_buf = (char*)
1238                     r->u.octet_aligned->buf;
1239                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1240                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1241             }
1242             else
1243             {
1244                 srw_res->records[i].recordSchema = "diagnostic";
1245                 srw_res->records[i].recordPacking = m_s2z_packing;
1246                 srw_res->records[i].recordData_buf = "67";
1247                 srw_res->records[i].recordData_len = 2;
1248                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1249             }
1250         }
1251     }
1252     if (records && records->which == Z_Records_NSD)
1253     {
1254         int http_code;
1255         http_code = z_to_srw_diag(odr_encode(), srw_res,
1256                                    records->u.nonSurrogateDiagnostic);
1257         if (http_code)
1258             return send_http_response(http_code);
1259     }
1260     return send_srw_response(srw_pdu);
1261
1262 }
1263
1264 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1265                                         int num_diagnostics, int http_code /* = 200 */)
1266 {
1267     ODR o = odr_encode();
1268     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1269     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1270
1271     srw_res->num_diagnostics = num_diagnostics;
1272     srw_res->diagnostics = diagnostics;
1273     return send_srw_response(srw_pdu, http_code);
1274 }
1275
1276 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1277                                         int num_diagnostics)
1278 {
1279     Yaz_ProxyConfig *cfg = check_reconfigure();
1280     if (cfg)
1281     {
1282         int len;
1283         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1284                                        m_s2z_database, &len);
1285         if (b)
1286         {
1287             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1288             Z_SRW_explainResponse *er = res->u.explain_response;
1289
1290             er->record.recordData_buf = b;
1291             er->record.recordData_len = len;
1292             er->record.recordPacking = m_s2z_packing;
1293             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1294
1295             er->diagnostics = diagnostics;
1296             er->num_diagnostics = num_diagnostics;
1297             return send_srw_response(res);
1298         }
1299     }
1300     return send_http_response(404);
1301 }
1302
1303 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1304 {
1305     if (m_http_version)
1306     {
1307         if (apdu->which == Z_APDU_initResponse)
1308         {
1309             Z_InitResponse *res = apdu->u.initResponse;
1310             if (*res->result == 0)
1311             {
1312                 send_to_srw_client_error(3, 0);
1313             }
1314             else if (!m_s2z_search_apdu)
1315             {
1316                 send_srw_explain_response(0, 0);
1317             }
1318             else
1319             {
1320                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1321             }
1322         }
1323         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1324         {
1325             m_s2z_search_apdu = 0;
1326             Z_SearchResponse *res = apdu->u.searchResponse;
1327             m_s2z_hit_count = *res->resultCount;
1328             if (res->records && res->records->which == Z_Records_NSD)
1329             {
1330                 send_to_srw_client_ok(0, res->records, 1);
1331             }
1332             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1333             {
1334                 // adjust
1335                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1336
1337                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1338                 {
1339                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1340                         > m_s2z_hit_count)
1341                         *pr->numberOfRecordsRequested =
1342                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1343                 }
1344                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1345             }
1346             else
1347             {
1348                 m_s2z_present_apdu = 0;
1349                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1350             }
1351         }
1352         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1353         {
1354             int start =
1355                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1356
1357             m_s2z_present_apdu = 0;
1358             Z_PresentResponse *res = apdu->u.presentResponse;
1359             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1360         }
1361     }
1362     else
1363     {
1364         int len = 0;
1365         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1366             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1367                      apdu_name(apdu));
1368         int r = send_Z_PDU(apdu, &len);
1369         m_bytes_sent += len;
1370         m_bw_stat.add_bytes(len);
1371         logtime();
1372         return r;
1373     }
1374     return 0;
1375 }
1376
1377 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1378 {
1379     int kill_session = 0;
1380     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1381
1382     if (new_id)
1383         *new_id = m_referenceId;
1384
1385     if (apdu->which == Z_APDU_searchResponse)
1386     {
1387         Z_SearchResponse *sr = apdu->u.searchResponse;
1388         Z_Records *p = sr->records;
1389         if (p && p->which == Z_Records_NSD)
1390         {
1391             Z_DiagRec dr, *dr_p = &dr;
1392             dr.which = Z_DiagRec_defaultFormat;
1393             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1394
1395             *sr->searchStatus = 0;
1396             display_diagrecs(&dr_p, 1);
1397         }
1398         else
1399         {
1400             if (p && p->which == Z_Records_DBOSD)
1401             {
1402                 if (m_backend_type
1403 #if HAVE_USEMARCON
1404                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1405 #endif
1406                     )
1407                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1408                 if (m_marcxml_mode == marcxml)
1409                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1410                                        m_backend_charset);
1411                 else
1412                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1413                                             m_backend_charset);
1414                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1415                     return 0;
1416
1417             }
1418             if (sr->resultCount)
1419             {
1420                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1421                         *sr->resultCount);
1422                 if (*sr->resultCount < 0)
1423                 {
1424                     m_flag_invalid_session = 1;
1425                     kill_session = 1;
1426
1427                     *sr->searchStatus = 0;
1428                     sr->records =
1429                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1430                     *sr->resultCount = 0;
1431                 }
1432             }
1433         }
1434     }
1435     else if (apdu->which == Z_APDU_presentResponse)
1436     {
1437         Z_PresentResponse *sr = apdu->u.presentResponse;
1438         Z_Records *p = sr->records;
1439         if (p && p->which == Z_Records_NSD)
1440         {
1441             Z_DiagRec dr, *dr_p = &dr;
1442             dr.which = Z_DiagRec_defaultFormat;
1443             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1444             if (*sr->presentStatus == Z_PresentStatus_success)
1445                 *sr->presentStatus = Z_PresentStatus_failure;
1446             display_diagrecs(&dr_p, 1);
1447         }
1448         if (p && p->which == Z_Records_DBOSD)
1449         {
1450             if (m_backend_type
1451 #if HAVE_USEMARCON
1452                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1453 #endif
1454                 )
1455                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1456             if (m_marcxml_mode == marcxml)
1457                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1458                                    m_backend_charset);
1459             else
1460                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1461                                         m_backend_charset);
1462             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1463                 return 0;
1464         }
1465     }
1466     else if (apdu->which == Z_APDU_initResponse)
1467     {
1468         //Get and check negotiation record
1469         //from init response.
1470         handle_charset_lang_negotiation(apdu);
1471
1472         if (m_initRequest_options)
1473         {
1474             Z_Options *nopt =
1475                 (Odr_bitmask *)odr_malloc(odr_encode(),
1476                                           sizeof(Odr_bitmask));
1477             ODR_MASK_ZERO(nopt);
1478
1479             int i;
1480             for (i = 0; i<24; i++)
1481                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1482                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1483                     ODR_MASK_SET(nopt, i);
1484             apdu->u.initResponse->options = nopt;
1485         }
1486         if (m_initRequest_version)
1487         {
1488             Z_ProtocolVersion *nopt =
1489                 (Odr_bitmask *)odr_malloc(odr_encode(),
1490                                           sizeof(Odr_bitmask));
1491             ODR_MASK_ZERO(nopt);
1492
1493             int i;
1494             for (i = 0; i<8; i++)
1495                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1496                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1497                     ODR_MASK_SET(nopt, i);
1498             apdu->u.initResponse->protocolVersion = nopt;
1499         }
1500         apdu->u.initResponse->preferredMessageSize =
1501             odr_intdup(odr_encode(),
1502                        m_client->m_initResponse_preferredMessageSize >
1503                        m_initRequest_preferredMessageSize ?
1504                        m_initRequest_preferredMessageSize :
1505                        m_client->m_initResponse_preferredMessageSize);
1506         apdu->u.initResponse->maximumRecordSize =
1507             odr_intdup(odr_encode(),
1508                        m_client->m_initResponse_maximumRecordSize >
1509                        m_initRequest_maximumRecordSize ?
1510                        m_initRequest_maximumRecordSize :
1511                        m_client->m_initResponse_maximumRecordSize);
1512     }
1513
1514     int r = send_PDU_convert(apdu);
1515     if (r)
1516         return r;
1517     if (kill_session)
1518     {
1519         delete m_client;
1520         m_client = 0;
1521         m_parent->pre_init();
1522     }
1523     return r;
1524 }
1525
1526 void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu)
1527 {
1528     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1529
1530     odr_reset(m_idAuthentication_odr);
1531     z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0);
1532     m_idAuthentication_ber_buf =
1533         odr_getbuf(m_idAuthentication_odr,
1534                    &m_idAuthentication_ber_size, 0);
1535 }
1536
1537 bool Yaz_ProxyClient::compare_charset(Z_APDU *apdu)
1538 {
1539     return true;
1540 }
1541
1542 bool Yaz_ProxyClient::compare_idAuthentication(Z_APDU *apdu)
1543 {
1544     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1545     ODR odr = odr_createmem(ODR_ENCODE);
1546
1547     z_IdAuthentication(odr, &t, 1, 0);
1548     int sz;
1549     char *buf = odr_getbuf(odr, &sz, 0);
1550     if (buf && m_idAuthentication_ber_buf
1551         && sz == m_idAuthentication_ber_size
1552         && !memcmp(m_idAuthentication_ber_buf, buf, sz))
1553     {
1554         odr_destroy(odr);
1555         return true;
1556     }
1557     odr_destroy(odr);
1558     if (!buf && !m_idAuthentication_ber_buf)
1559         return true;
1560     return false;
1561 }
1562
1563 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1564 {
1565     int len = 0;
1566     const char *apdu_name_tmp = apdu_name(apdu);
1567     int r = send_Z_PDU(apdu, &len);
1568     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1569         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1570                  get_session_str(),
1571                  apdu_name_tmp, get_hostname(), len);
1572     m_bytes_sent += len;
1573     return r;
1574 }
1575
1576 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1577 {
1578     if (apdu->which == Z_APDU_presentRequest)
1579     {
1580         Z_PresentRequest *pr = apdu->u.presentRequest;
1581         int toget = *pr->numberOfRecordsRequested;
1582         int start = *pr->resultSetStartPoint;
1583
1584         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1585                 pr->resultSetId, start, toget);
1586
1587         if (*m_parent->m_optimize == '0')
1588             return apdu;
1589
1590         if (!m_client->m_last_resultSetId)
1591         {
1592             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1593             new_apdu->u.presentResponse->records =
1594                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1595                                                pr->resultSetId);
1596             send_to_client(new_apdu);
1597             return 0;
1598         }
1599         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1600         {
1601             if (start+toget-1 > m_client->m_last_resultCount)
1602             {
1603                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1604                 new_apdu->u.presentResponse->records =
1605                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1606                 send_to_client(new_apdu);
1607                 return 0;
1608             }
1609             Z_NamePlusRecordList *npr;
1610 #if 0
1611             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1612                     m_session_str, start, toget, yaz_z3950oid_to_str(
1613                         pr->preferredRecordSyntax, &oclass));
1614 #endif
1615             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1616                                           pr->preferredRecordSyntax,
1617                                           pr->recordComposition))
1618             {
1619                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1620                          m_session_str);
1621                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1622                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1623
1624                 new_apdu->u.presentResponse->numberOfRecordsReturned
1625                     = odr_intdup(odr_encode(), toget);
1626
1627                 new_apdu->u.presentResponse->records = (Z_Records*)
1628                     odr_malloc(odr_encode(), sizeof(Z_Records));
1629                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1630                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1631                 new_apdu->u.presentResponse->nextResultSetPosition =
1632                     odr_intdup(odr_encode(), start+toget);
1633
1634                 send_to_client(new_apdu);
1635                 return 0;
1636             }
1637         }
1638     }
1639
1640     if (apdu->which != Z_APDU_searchRequest)
1641         return apdu;
1642     Z_SearchRequest *sr = apdu->u.searchRequest;
1643     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1644     Yaz_Z_Databases this_databases;
1645
1646     this_databases.set(sr->num_databaseNames, (const char **)
1647                        sr->databaseNames);
1648
1649     this_query->set_Z_Query(sr->query);
1650
1651     char query_str[120];
1652     this_query->print(query_str, sizeof(query_str)-1);
1653     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1654
1655     if (*m_parent->m_optimize != '0' &&
1656         m_client->m_last_ok && m_client->m_last_query &&
1657         m_client->m_last_query->match(this_query) &&
1658         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1659         m_client->m_last_databases.match(this_databases))
1660     {
1661         delete this_query;
1662         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1663             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1664         {
1665             Z_NamePlusRecordList *npr;
1666             int toget = *sr->mediumSetPresentNumber;
1667             Z_RecordComposition *comp = 0;
1668
1669             if (toget > m_client->m_last_resultCount)
1670                 toget = m_client->m_last_resultCount;
1671
1672             if (sr->mediumSetElementSetNames)
1673             {
1674                 comp = (Z_RecordComposition *)
1675                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1676                 comp->which = Z_RecordComp_simple;
1677                 comp->u.simple = sr->mediumSetElementSetNames;
1678             }
1679
1680             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1681                                           sr->preferredRecordSyntax, comp))
1682             {
1683                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1684                          m_session_str);
1685                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1686                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1687                 new_apdu->u.searchResponse->resultCount =
1688                     &m_client->m_last_resultCount;
1689
1690                 new_apdu->u.searchResponse->numberOfRecordsReturned
1691                     = odr_intdup(odr_encode(), toget);
1692
1693                 new_apdu->u.searchResponse->presentStatus =
1694                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1695                 new_apdu->u.searchResponse->records = (Z_Records*)
1696                     odr_malloc(odr_encode(), sizeof(Z_Records));
1697                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1698                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1699                 new_apdu->u.searchResponse->nextResultSetPosition =
1700                     odr_intdup(odr_encode(), toget+1);
1701                 send_to_client(new_apdu);
1702                 return 0;
1703             }
1704             else
1705             {
1706                 // medium Set
1707                 // send present request (medium size)
1708                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1709                          m_session_str);
1710
1711                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1712                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1713                 pr->referenceId = sr->referenceId;
1714                 pr->resultSetId = sr->resultSetName;
1715                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1716                 *pr->numberOfRecordsRequested = toget;
1717                 pr->recordComposition = comp;
1718                 m_client->m_sr_transform = 1;
1719                 return new_apdu;
1720             }
1721         }
1722         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1723             m_client->m_last_resultCount <= 0)
1724         {
1725             // large set. Return pseudo-search response immediately
1726             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1727                      m_session_str);
1728             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1729             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1730             new_apdu->u.searchResponse->resultCount =
1731                 &m_client->m_last_resultCount;
1732             send_to_client(new_apdu);
1733             return 0;
1734         }
1735         else
1736         {
1737             Z_NamePlusRecordList *npr;
1738             int toget = m_client->m_last_resultCount;
1739             Z_RecordComposition *comp = 0;
1740             // small set
1741             // send a present request (small set)
1742
1743             if (sr->smallSetElementSetNames)
1744             {
1745                 comp = (Z_RecordComposition *)
1746                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1747                 comp->which = Z_RecordComp_simple;
1748                 comp->u.simple = sr->smallSetElementSetNames;
1749             }
1750
1751             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1752                                           sr->preferredRecordSyntax, comp))
1753             {
1754                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1755                          m_session_str);
1756                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1757                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1758                 new_apdu->u.searchResponse->resultCount =
1759                     &m_client->m_last_resultCount;
1760
1761                 new_apdu->u.searchResponse->numberOfRecordsReturned
1762                     = odr_intdup(odr_encode(), toget);
1763
1764                 new_apdu->u.searchResponse->presentStatus =
1765                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1766                 new_apdu->u.searchResponse->records = (Z_Records*)
1767                     odr_malloc(odr_encode(), sizeof(Z_Records));
1768                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1769                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1770                 new_apdu->u.searchResponse->nextResultSetPosition =
1771                     odr_intdup(odr_encode(), toget+1);
1772                 send_to_client(new_apdu);
1773                 return 0;
1774             }
1775             else
1776             {
1777                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1778                          m_session_str);
1779                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1780                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1781                 pr->referenceId = sr->referenceId;
1782                 pr->resultSetId = sr->resultSetName;
1783                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1784                 *pr->numberOfRecordsRequested = toget;
1785                 pr->recordComposition = comp;
1786                 m_client->m_sr_transform = 1;
1787                 return new_apdu;
1788             }
1789         }
1790     }
1791     else  // query doesn't match
1792     {
1793         delete m_client->m_last_query;
1794         m_client->m_last_query = this_query;
1795         m_client->m_last_ok = 0;
1796         m_client->m_cache.clear();
1797         m_client->m_resultSetStartPoint = 0;
1798
1799         xfree (m_client->m_last_resultSetId);
1800         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1801
1802         m_client->m_last_databases.set(sr->num_databaseNames,
1803                                        (const char **) sr->databaseNames);
1804     }
1805     return apdu;
1806 }
1807
1808
1809 void Yaz_Proxy::inc_request_no()
1810 {
1811     char *cp = strchr(m_session_str, ' ');
1812     m_request_no++;
1813     if (cp)
1814         sprintf(cp+1, "%d ", m_request_no);
1815 }
1816
1817 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1818 {
1819     inc_request_no();
1820
1821     m_bytes_recv += len;
1822
1823     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1824         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1825                  m_session_str, gdu_name(apdu), len);
1826
1827 #if HAVE_GETTIMEOFDAY
1828     gettimeofday((struct timeval *) m_time_tv, 0);
1829 #endif
1830     m_bw_stat.add_bytes(len);
1831     m_pdu_stat.add_bytes(1);
1832
1833     GDU *gdu = new GDU(apdu);
1834     m_in_queue.enqueue(gdu);
1835
1836     recv_GDU_more(false);
1837 }
1838
1839 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
1840 {
1841     int reduce = 0;
1842
1843     if (1)
1844     {
1845         m_parent->m_connect.add_connect(m_peername);
1846         int connect_total = m_parent->m_connect.get_total(m_peername);
1847         int connect_max = m_parent->m_connect_max;
1848
1849         if (connect_max && connect_total > connect_max)
1850         {
1851             yaz_log(YLOG_LOG, "%sconnect delay total=%d max=%d",
1852                     m_session_str, connect_total, connect_max);
1853             reduce = connect_total / connect_max;
1854         }
1855         else
1856             yaz_log(YLOG_LOG, "%sconnect OK total=%d", m_session_str,
1857                     connect_total);
1858         m_parent->m_connect.cleanup(false);
1859     }
1860
1861
1862     int bw_total = m_bw_stat.get_total();
1863     int pdu_total = m_pdu_stat.get_total();
1864
1865     assert(m_timeout_mode == timeout_busy);
1866     assert(m_timeout_gdu == 0);
1867
1868     if (m_bw_max)
1869     {
1870         if (bw_total > m_bw_max)
1871         {
1872             reduce = (bw_total/m_bw_max);
1873         }
1874     }
1875     if (m_pdu_max)
1876     {
1877         if (pdu_total > m_pdu_max)
1878         {
1879             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1880             reduce = (reduce > nreduce) ? reduce : nreduce;
1881         }
1882     }
1883     m_http_version = 0;
1884
1885 #if 0
1886     /* uncomment to force a big reduce */
1887     m_timeout_mode = timeout_reduce;
1888     m_timeout_gdu = gdu;
1889     timeout(3);       // call us reduce seconds later
1890     return;
1891 #endif
1892     if (reduce)
1893     {
1894         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1895                 m_session_str, reduce, bw_total, pdu_total,
1896                 m_bw_max, m_pdu_max);
1897
1898         m_timeout_mode = timeout_reduce;
1899         m_timeout_gdu = gdu;
1900         timeout(reduce);       // call us reduce seconds later
1901     }
1902     else
1903         recv_GDU_normal(gdu);
1904 }
1905
1906 void Yaz_Proxy::recv_GDU_more(bool normal)
1907 {
1908     GDU *g;
1909     if (normal && m_timeout_mode == timeout_busy)
1910         m_timeout_mode = timeout_normal;
1911     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
1912     {
1913         m_timeout_mode = timeout_busy;
1914         recv_GDU_reduce(g);
1915     }
1916 }
1917
1918 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
1919 {
1920     Z_GDU *apdu = 0;
1921     gdu->move_away_gdu(odr_decode(), &apdu);
1922     delete gdu;
1923
1924     if (apdu->which == Z_GDU_Z3950)
1925         handle_incoming_Z_PDU(apdu->u.z3950);
1926     else if (apdu->which == Z_GDU_HTTP_Request)
1927         handle_incoming_HTTP(apdu->u.HTTP_Request);
1928 }
1929
1930 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1931 {
1932     if (m_max_record_retrieve)
1933     {
1934         if (apdu->which == Z_APDU_presentRequest)
1935         {
1936             Z_PresentRequest *pr = apdu->u.presentRequest;
1937             if (pr->numberOfRecordsRequested &&
1938                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1939                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1940         }
1941     }
1942 }
1943
1944 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1945 {
1946     if (apdu->which == Z_APDU_initRequest)
1947     {
1948         if (m_initRequest_options &&
1949             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1950             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1951         {
1952             // There is no negotiation proposal from
1953             // client's side. OK. The proxy negotiation
1954             // in use, only.
1955             Z_InitRequest *initRequest = apdu->u.initRequest;
1956             Z_OtherInformation **otherInfo;
1957             Z_OtherInformationUnit *oi;
1958             get_otherInfoAPDU(apdu, &otherInfo);
1959             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1960             if (oi)
1961             {
1962                 ODR_MASK_SET(initRequest->options,
1963                     Z_Options_negotiationModel);
1964                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1965                 oi->information.externallyDefinedInfo =
1966                 yaz_set_proposal_charneg(odr_encode(),
1967                     (const char**)&m_proxy_negotiation_charset,
1968                     m_proxy_negotiation_charset ? 1:0,
1969                     (const char**)&m_proxy_negotiation_lang,
1970                     m_proxy_negotiation_lang ? 1:0,
1971                     1);
1972             }
1973         }
1974         else if (m_initRequest_options &&
1975                  ODR_MASK_GET(m_initRequest_options,
1976                               Z_Options_negotiationModel) &&
1977                  m_charset_converter->get_target_query_charset())
1978         {
1979             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1980                     m_session_str,
1981                     m_charset_converter->get_target_query_charset());
1982             Z_InitRequest *initRequest = apdu->u.initRequest;
1983             Z_CharSetandLanguageNegotiation *negotiation =
1984                 yaz_get_charneg_record (initRequest->otherInfo);
1985             if (negotiation &&
1986                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1987             {
1988                 NMEM nmem = nmem_create();
1989                 char **charsets = 0;
1990                 int num_charsets = 0;
1991                 char **langs = 0;
1992                 int num_langs = 0;
1993                 int selected = 0;
1994                 yaz_get_proposal_charneg (nmem, negotiation,
1995                                           &charsets, &num_charsets,
1996                                           &langs, &num_langs, &selected);
1997                 int i;
1998                 for (i = 0; i<num_charsets; i++)
1999                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
2000                             charsets[i]);
2001                 for (i = 0; i<num_langs; i++)
2002                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
2003                             langs[i]);
2004
2005                 const char *t_charset =
2006                     m_charset_converter->get_target_query_charset();
2007                 // sweep through charsets and pick the first supported
2008                 // conversion
2009                 for (i = 0; i<num_charsets; i++)
2010                 {
2011                     const char *c_charset = charsets[i];
2012                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
2013                         break;
2014                 }
2015                 if (i != num_charsets)
2016                 {
2017                     // got one .. set up ODR for reverse direction
2018                     const char *c_charset = charsets[i];
2019                     odr_set_charset(odr_encode(), c_charset, t_charset);
2020                     m_charset_converter->set_client_query_charset(c_charset);
2021                     m_charset_converter->set_client_charset_selected(selected);
2022                 }
2023                 nmem_destroy(nmem);
2024                 ODR_MASK_CLEAR(m_initRequest_options,
2025                                Z_Options_negotiationModel);
2026                 yaz_del_charneg_record(&initRequest->otherInfo);
2027             }
2028             else
2029             {
2030                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
2031                         m_session_str);
2032             }
2033         }
2034         else if (m_charset_converter->get_target_query_charset() &&
2035             m_proxy_negotiation_default_charset)
2036         {
2037             m_charset_converter->
2038                 set_client_query_charset(m_proxy_negotiation_default_charset);
2039         }
2040     }
2041     else if (apdu->which == Z_APDU_initResponse)
2042     {
2043         Z_InitResponse *initResponse = apdu->u.initResponse;
2044         Z_OtherInformation **otherInfo;
2045         get_otherInfoAPDU(apdu, &otherInfo);
2046
2047         Z_CharSetandLanguageNegotiation *charneg = 0;
2048
2049         if (otherInfo && *otherInfo &&
2050             ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel)
2051             && (charneg = yaz_get_charneg_record(*otherInfo)))
2052         {
2053             char *charset = 0;
2054             char *lang = 0;
2055             int selected = 0;
2056
2057             yaz_get_response_charneg(m_referenceId_mem, charneg,
2058                 &charset, &lang, &selected);
2059
2060             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
2061                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
2062
2063             if (m_initRequest_options &&
2064                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
2065             {
2066                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
2067                     m_session_str);
2068             }
2069             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2070             {
2071                 // negotiation-charset, negotiation-lang
2072                 // elements of config file in use.
2073
2074                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2075                     m_session_str);
2076
2077                 // clear negotiation option.
2078                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2079
2080                 // Delete negotiation (charneg-3) entry.
2081                 yaz_del_charneg_record(otherInfo);
2082             }
2083         }
2084         else
2085         {
2086             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2087             {
2088                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2089                         m_session_str);
2090             }
2091             else if (m_charset_converter->get_client_query_charset())
2092             {
2093                 Z_OtherInformation **otherInfo;
2094                 Z_OtherInformationUnit *oi;
2095                 get_otherInfoAPDU(apdu, &otherInfo);
2096                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2097                 if (oi)
2098                 {
2099                     ODR_MASK_SET(initResponse->options,
2100                                  Z_Options_negotiationModel);
2101                     if (m_initRequest_options)
2102                         ODR_MASK_SET(m_initRequest_options,
2103                                      Z_Options_negotiationModel);
2104
2105                     oi->which = Z_OtherInfo_externallyDefinedInfo;
2106                     oi->information.externallyDefinedInfo =
2107                         yaz_set_response_charneg(
2108                             odr_encode(),
2109                             m_charset_converter->get_client_query_charset(),
2110                             0 /* no lang */,
2111                             m_charset_converter->get_client_charset_selected());
2112                 }
2113             }
2114         }
2115     }
2116 }
2117
2118 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2119                                                      int error,
2120                                                      const char *addinfo)
2121 {
2122     Z_Records *rec = (Z_Records *)
2123         odr_malloc (odr, sizeof(*rec));
2124     int *err = (int *)
2125         odr_malloc (odr, sizeof(*err));
2126     Z_DiagRec *drec = (Z_DiagRec *)
2127         odr_malloc (odr, sizeof(*drec));
2128     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2129         odr_malloc (odr, sizeof(*dr));
2130     *err = error;
2131     rec->which = Z_Records_NSD;
2132     rec->u.nonSurrogateDiagnostic = dr;
2133     dr->diagnosticSetId =
2134         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
2135     dr->condition = err;
2136     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2137     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2138     return rec;
2139 }
2140
2141 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2142 {
2143     if (apdu->which == Z_APDU_searchRequest &&
2144         apdu->u.searchRequest->query &&
2145         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2146         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2147     {
2148         Z_RPNQuery *rpnquery = 0;
2149         Z_SearchRequest *sr = apdu->u.searchRequest;
2150         char *addinfo = 0;
2151
2152         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2153                 sr->query->u.type_104->u.cql);
2154
2155         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2156                                           &rpnquery, odr_encode(),
2157                                           &addinfo);
2158         if (r == -3)
2159             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2160         else if (r)
2161         {
2162             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2163             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2164
2165             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2166             new_apdu->u.searchResponse->records =
2167                 create_nonSurrogateDiagnostics(odr_encode(),
2168                                                yaz_diag_srw_to_bib1(r),
2169                                                addinfo);
2170             *new_apdu->u.searchResponse->searchStatus = 0;
2171
2172             send_to_client(new_apdu);
2173
2174             return 0;
2175         }
2176         else
2177         {
2178             sr->query->which = Z_Query_type_1;
2179             sr->query->u.type_1 = rpnquery;
2180         }
2181         return apdu;
2182     }
2183     return apdu;
2184 }
2185
2186 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2187 {
2188     if (apdu->which == Z_APDU_searchRequest &&
2189         apdu->u.searchRequest->query)
2190     {
2191         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2192             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2193         {
2194             if (m_http_version)
2195                 m_charset_converter->set_client_query_charset("UTF-8");
2196             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2197             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2198         }
2199     }
2200     return apdu;
2201 }
2202
2203
2204 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2205 {
2206     if (apdu->which == Z_APDU_searchRequest)
2207     {
2208         Z_SearchRequest *sr = apdu->u.searchRequest;
2209         int err = 0;
2210         char *addinfo = 0;
2211
2212         Yaz_ProxyConfig *cfg = check_reconfigure();
2213         if (cfg)
2214             err = cfg->check_query(odr_encode(), m_default_target,
2215                                    sr->query, &addinfo);
2216         if (err)
2217         {
2218             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2219
2220             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2221             new_apdu->u.searchResponse->records =
2222                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2223             *new_apdu->u.searchResponse->searchStatus = 0;
2224
2225             send_to_client(new_apdu);
2226
2227             return 0;
2228         }
2229     }
2230     return apdu;
2231 }
2232
2233 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2234 {
2235     if (apdu->which != Z_APDU_initRequest)
2236         return 1;  // pass if no init request
2237     Z_InitRequest *req = apdu->u.initRequest;
2238
2239     Yaz_ProxyConfig *cfg = check_reconfigure();
2240     if (!cfg)
2241         return 1;  // pass if no config
2242
2243     int ret;
2244     if (req->idAuthentication == 0)
2245     {
2246         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2247                                          m_peername);
2248     }
2249     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2250     {
2251         ret = cfg->client_authentication(
2252             m_default_target,
2253             req->idAuthentication->u.idPass->userId,
2254             req->idAuthentication->u.idPass->groupId,
2255             req->idAuthentication->u.idPass->password,
2256             m_peername);
2257     }
2258     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2259     {
2260         char user[64], pass[64];
2261         *user = '\0';
2262         *pass = '\0';
2263         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2264         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2265                                          m_peername);
2266     }
2267     else
2268         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2269                                          m_peername);
2270     return ret;
2271 }
2272
2273 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2274 {
2275     m_marcxml_mode = none;
2276     if (apdu->which == Z_APDU_searchRequest)
2277     {
2278         Z_SearchRequest *sr = apdu->u.searchRequest;
2279         int err = 0;
2280         char *addinfo = 0;
2281         Yaz_ProxyConfig *cfg = check_reconfigure();
2282
2283         Z_RecordComposition rc_temp, *rc = 0;
2284         if (sr->smallSetElementSetNames)
2285         {
2286             rc_temp.which = Z_RecordComp_simple;
2287             rc_temp.u.simple = sr->smallSetElementSetNames;
2288             rc = &rc_temp;
2289         }
2290
2291         if (sr->preferredRecordSyntax)
2292         {
2293             struct oident *ent;
2294             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2295             m_frontend_type = ent->value;
2296         }
2297         else
2298             m_frontend_type = VAL_NONE;
2299
2300         char *stylesheet_name = 0;
2301         if (cfg)
2302             err = cfg->check_syntax(odr_encode(),
2303                                     m_default_target,
2304                                     sr->preferredRecordSyntax, rc,
2305                                     &addinfo, &stylesheet_name, &m_schema,
2306                                     &m_backend_type, &m_backend_charset,
2307                                     &m_usemarcon_ini_stage1,
2308                                     &m_usemarcon_ini_stage2);
2309         if (stylesheet_name)
2310         {
2311             m_parent->low_socket_close();
2312
2313 #if HAVE_XSLT
2314             if (m_stylesheet_xsp)
2315                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2316             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2317                                                        stylesheet_name);
2318 #endif
2319             m_stylesheet_offset = 0;
2320             xfree(stylesheet_name);
2321
2322             m_parent->low_socket_open();
2323         }
2324         if (err == -1)
2325         {
2326             sr->smallSetElementSetNames = 0;
2327             sr->mediumSetElementSetNames = 0;
2328             m_marcxml_mode = marcxml;
2329             if (m_backend_type)
2330             {
2331
2332                 sr->preferredRecordSyntax =
2333                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2334                                         m_backend_type);
2335             }
2336             else
2337                 sr->preferredRecordSyntax =
2338                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2339                                            VAL_USMARC);
2340         }
2341         else if (err)
2342         {
2343             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2344
2345             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2346             new_apdu->u.searchResponse->records =
2347                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2348             *new_apdu->u.searchResponse->searchStatus = 0;
2349
2350             send_to_client(new_apdu);
2351
2352             return 0;
2353         }
2354         else if (m_backend_type)
2355         {
2356             sr->preferredRecordSyntax =
2357                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2358         }
2359     }
2360     else if (apdu->which == Z_APDU_presentRequest)
2361     {
2362         Z_PresentRequest *pr = apdu->u.presentRequest;
2363         int err = 0;
2364         char *addinfo = 0;
2365         Yaz_ProxyConfig *cfg = check_reconfigure();
2366
2367         if (pr->preferredRecordSyntax)
2368         {
2369             struct oident *ent;
2370             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2371             m_frontend_type = ent->value;
2372         }
2373         else
2374             m_frontend_type = VAL_NONE;
2375
2376         char *stylesheet_name = 0;
2377         if (cfg)
2378             err = cfg->check_syntax(odr_encode(), m_default_target,
2379                                     pr->preferredRecordSyntax,
2380                                     pr->recordComposition,
2381                                     &addinfo, &stylesheet_name, &m_schema,
2382                                     &m_backend_type, &m_backend_charset,
2383                                     &m_usemarcon_ini_stage1,
2384                                     &m_usemarcon_ini_stage2
2385                                     );
2386         if (stylesheet_name)
2387         {
2388             m_parent->low_socket_close();
2389
2390 #if HAVE_XSLT
2391             if (m_stylesheet_xsp)
2392                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2393             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2394                                                        stylesheet_name);
2395 #endif
2396             m_stylesheet_offset = 0;
2397             xfree(stylesheet_name);
2398
2399             m_parent->low_socket_open();
2400         }
2401         if (err == -1)
2402         {
2403             pr->recordComposition = 0;
2404             m_marcxml_mode = marcxml;
2405             if (m_backend_type)
2406             {
2407
2408                 pr->preferredRecordSyntax =
2409                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2410                                         m_backend_type);
2411             }
2412             else
2413                 pr->preferredRecordSyntax =
2414                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2415                                            VAL_USMARC);
2416         }
2417         else if (err)
2418         {
2419             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2420
2421             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2422             new_apdu->u.presentResponse->records =
2423                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2424             *new_apdu->u.presentResponse->presentStatus =
2425                 Z_PresentStatus_failure;
2426
2427             send_to_client(new_apdu);
2428
2429             return 0;
2430         }
2431         else if (m_backend_type)
2432         {
2433             pr->preferredRecordSyntax =
2434                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2435         }
2436     }
2437     return apdu;
2438 }
2439
2440 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2441 {
2442     if (!schema)
2443         return 0;
2444     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2445         odr_malloc(o, sizeof(Z_ElementSetNames));
2446     esn->which = Z_ElementSetNames_generic;
2447     esn->u.generic = odr_strdup(o, schema);
2448     return esn;
2449 }
2450
2451 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2452 {
2453     const char *t = 0;
2454     Yaz_ProxyConfig *cfg = check_reconfigure();
2455     if (cfg)
2456         t = cfg->get_explain_name(db, backend_db);
2457
2458     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2459     {
2460         releaseClient();
2461     }
2462
2463     if (t)
2464     {
2465         xfree(m_default_target);
2466         m_default_target = xstrdup(t);
2467     }
2468 }
2469
2470 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2471 {
2472     struct stat sbuf;
2473     if (strcmp(hreq->method, "GET"))
2474         return 0;
2475     if (hreq->path[0] != '/')
2476     {
2477         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2478         return 0;
2479     }
2480     const char *cp = hreq->path;
2481     while (*cp)
2482     {
2483         if (*cp == '/' && strchr("/.", cp[1]))
2484         {
2485             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2486             return 0;
2487         }
2488         cp++;
2489     }
2490     const char *fname = hreq->path+1;
2491     if (stat(fname, &sbuf))
2492     {
2493         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2494         return 0;
2495     }
2496     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2497     {
2498         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2499         return 0;
2500     }
2501     if (sbuf.st_size > (off_t) 1000000)
2502     {
2503         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2504         return 0;
2505     }
2506
2507     ODR o = odr_encode();
2508     Yaz_ProxyConfig *cfg = check_reconfigure();
2509     const char *ctype = cfg->check_mime_type(fname);
2510     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2511     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2512     if (m_http_version)
2513         hres->version = odr_strdup(o, m_http_version);
2514     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2515     if (m_http_keepalive)
2516         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2517     else
2518         timeout(0);
2519
2520     hres->content_len = sbuf.st_size;
2521     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2522     FILE *f = fopen(fname, "rb");
2523     if (f)
2524     {
2525         fread(hres->content_buf, 1, hres->content_len, f);
2526         fclose(f);
2527     }
2528     else
2529     {
2530         return 0;
2531     }
2532     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2533     {
2534         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2535                  fname);
2536     }
2537     int len;
2538     send_GDU(gdu, &len);
2539     recv_GDU_more(true);
2540     return 1;
2541 }
2542
2543 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2544 {
2545     if (m_s2z_odr_init)
2546     {
2547         odr_destroy(m_s2z_odr_init);
2548         m_s2z_odr_init = 0;
2549     }
2550     if (m_s2z_odr_search)
2551     {
2552         odr_destroy(m_s2z_odr_search);
2553         m_s2z_odr_search = 0;
2554     }
2555
2556     m_http_keepalive = 0;
2557     m_http_version = 0;
2558     if (!strcmp(hreq->version, "1.0"))
2559     {
2560         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2561         if (v && !strcmp(v, "Keep-Alive"))
2562             m_http_keepalive = 1;
2563         else
2564             m_http_keepalive = 0;
2565         m_http_version = "1.0";
2566     }
2567     else
2568     {
2569         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2570         if (v && !strcmp(v, "close"))
2571             m_http_keepalive = 0;
2572         else
2573             m_http_keepalive = 1;
2574         m_http_version = "1.1";
2575     }
2576
2577     const char *a = z_HTTP_header_lookup(hreq->headers, "Authorization");
2578     char authorization_str[255];
2579     *authorization_str = '\0';
2580     if (a && strncasecmp(a, "Basic ", 6) == 0)
2581         base64_decode(a + 6, authorization_str, 254);
2582
2583     Z_SRW_PDU *srw_pdu = 0;
2584     Z_SOAP *soap_package = 0;
2585     char *charset = 0;
2586     Z_SRW_diagnostic *diagnostic = 0;
2587     int num_diagnostic = 0;
2588
2589     if (file_access(hreq))
2590     {
2591         return;
2592     }
2593     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2594                             &charset) == 0
2595              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2596                                &charset, &diagnostic, &num_diagnostic) == 0)
2597     {
2598         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2599         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2600         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2601         m_s2z_init_apdu = 0;
2602         m_s2z_search_apdu = 0;
2603         m_s2z_present_apdu = 0;
2604
2605         m_s2z_stylesheet = 0;
2606
2607         Z_IdAuthentication *auth = NULL;
2608         if (*authorization_str)
2609         {
2610             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
2611             auth->which = Z_IdAuthentication_idPass;
2612             auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
2613             auth->u.idPass->groupId = NULL;
2614             char *p = strchr(authorization_str, ':');
2615             if (p)
2616             {
2617                 *p = '\0';
2618                 p++;
2619                 auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
2620             }
2621             auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
2622         }
2623
2624         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2625         {
2626             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2627
2628             const char *backend_db = srw_req->database;
2629             srw_get_client(srw_req->database, &backend_db);
2630
2631             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2632             // recordXPath unsupported.
2633             if (srw_req->recordXPath)
2634             {
2635                 yaz_add_srw_diagnostic(odr_decode(),
2636                                        &diagnostic, &num_diagnostic,
2637                                        72, 0);
2638             }
2639             // sort unsupported
2640             if (srw_req->sort_type != Z_SRW_sort_type_none)
2641             {
2642                 yaz_add_srw_diagnostic(odr_decode(),
2643                                        &diagnostic, &num_diagnostic,
2644                                        80, 0);
2645             }
2646             // save stylesheet
2647             if (srw_req->stylesheet)
2648                 m_s2z_stylesheet =
2649                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2650
2651             // set packing for response records ..
2652             if (srw_req->recordPacking &&
2653                 !strcmp(srw_req->recordPacking, "xml"))
2654                 m_s2z_packing = Z_SRW_recordPacking_XML;
2655             else
2656                 m_s2z_packing = Z_SRW_recordPacking_string;
2657
2658             if (num_diagnostic)
2659             {
2660                 Z_SRW_PDU *srw_pdu =
2661                     yaz_srw_get(odr_encode(),
2662                                 Z_SRW_searchRetrieve_response);
2663                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2664
2665                 srw_res->diagnostics = diagnostic;
2666                 srw_res->num_diagnostics = num_diagnostic;
2667                 send_srw_response(srw_pdu);
2668                 return;
2669             }
2670
2671             // prepare search PDU
2672             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2673                                           Z_APDU_searchRequest);
2674             Z_SearchRequest *z_searchRequest =
2675                 m_s2z_search_apdu->u.searchRequest;
2676
2677             z_searchRequest->num_databaseNames = 1;
2678             z_searchRequest->databaseNames = (char**)
2679                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2680             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2681                                                            backend_db);
2682
2683             // query transformation
2684             Z_Query *query = (Z_Query *)
2685                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2686             z_searchRequest->query = query;
2687
2688             if (srw_req->query_type == Z_SRW_query_type_cql)
2689             {
2690                 Z_External *ext = (Z_External *)
2691                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2692                 ext->direct_reference =
2693                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2694                 ext->indirect_reference = 0;
2695                 ext->descriptor = 0;
2696                 ext->which = Z_External_CQL;
2697                 ext->u.cql = srw_req->query.cql;
2698
2699                 query->which = Z_Query_type_104;
2700                 query->u.type_104 =  ext;
2701             }
2702             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2703             {
2704                 Z_RPNQuery *RPNquery;
2705                 YAZ_PQF_Parser pqf_parser;
2706
2707                 pqf_parser = yaz_pqf_create ();
2708
2709                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2710                                           srw_req->query.pqf);
2711                 if (!RPNquery)
2712                 {
2713                     const char *pqf_msg;
2714                     size_t off;
2715                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2716                     int ioff = off;
2717                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
2718                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2719
2720                     send_to_srw_client_error(10, 0);
2721                     return;
2722                 }
2723                 query->which = Z_Query_type_1;
2724                 query->u.type_1 =  RPNquery;
2725
2726                 yaz_pqf_destroy (pqf_parser);
2727             }
2728             else
2729             {
2730                 send_to_srw_client_error(7, "query");
2731                 return;
2732             }
2733
2734             // present
2735             m_s2z_present_apdu = 0;
2736             int max = 0;
2737             if (srw_req->maximumRecords)
2738                 max = *srw_req->maximumRecords;
2739             int start = 1;
2740             if (srw_req->startRecord)
2741                 start = *srw_req->startRecord;
2742             if (max > 0)
2743             {
2744                 // Some backend, such as Voyager doesn't honor piggyback
2745                 // So we use present always (0 &&).
2746                 if (0 && start <= 1)  // Z39.50 piggyback
2747                 {
2748                     *z_searchRequest->smallSetUpperBound = max;
2749                     *z_searchRequest->mediumSetPresentNumber = max;
2750                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2751
2752                     z_searchRequest->preferredRecordSyntax =
2753                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2754                                                VAL_TEXT_XML);
2755                     if (srw_req->recordSchema)
2756                     {
2757                         z_searchRequest->smallSetElementSetNames =
2758                             z_searchRequest->mediumSetElementSetNames =
2759                             mk_esn_from_schema(m_s2z_odr_search,
2760                                                srw_req->recordSchema);
2761                     }
2762                 }
2763                 else   // Z39.50 present
2764                 {
2765                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2766                                                    Z_APDU_presentRequest);
2767                     Z_PresentRequest *z_presentRequest =
2768                         m_s2z_present_apdu->u.presentRequest;
2769                     *z_presentRequest->resultSetStartPoint = start;
2770                     *z_presentRequest->numberOfRecordsRequested = max;
2771                     z_presentRequest->preferredRecordSyntax =
2772                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2773                                                VAL_TEXT_XML);
2774                     if (srw_req->recordSchema)
2775                     {
2776                         z_presentRequest->recordComposition =
2777                             (Z_RecordComposition *)
2778                             odr_malloc(m_s2z_odr_search,
2779                                        sizeof(Z_RecordComposition));
2780                         z_presentRequest->recordComposition->which =
2781                             Z_RecordComp_simple;
2782                         z_presentRequest->recordComposition->u.simple =
2783                             mk_esn_from_schema(m_s2z_odr_search,
2784                                                srw_req->recordSchema);
2785                     }
2786                 }
2787             }
2788             if (!m_client)
2789             {
2790                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2791                                             Z_APDU_initRequest);
2792
2793                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2794
2795                 // prevent m_initRequest_apdu memory from being grabbed
2796                 // in Yaz_Proxy::handle_incoming_Z_PDU
2797                 m_initRequest_apdu = m_s2z_init_apdu;
2798                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2799                 return;
2800             }
2801             else
2802             {
2803                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2804                 return;
2805             }
2806         }
2807         else if (srw_pdu->which == Z_SRW_explain_request)
2808         {
2809             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2810
2811             const char *backend_db = srw_req->database;
2812             srw_get_client(srw_req->database, &backend_db);
2813
2814             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2815
2816             // save stylesheet
2817             if (srw_req->stylesheet)
2818                 m_s2z_stylesheet =
2819                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2820
2821             if (srw_req->recordPacking &&
2822                 !strcmp(srw_req->recordPacking, "xml"))
2823                 m_s2z_packing = Z_SRW_recordPacking_XML;
2824             else
2825                 m_s2z_packing = Z_SRW_recordPacking_string;
2826
2827             if (num_diagnostic)
2828             {
2829                 send_srw_explain_response(diagnostic, num_diagnostic);
2830                 return;
2831             }
2832
2833             if (!m_client)
2834             {
2835                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2836                                             Z_APDU_initRequest);
2837
2838                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2839
2840                 // prevent m_initRequest_apdu memory from being grabbed
2841                 // in Yaz_Proxy::handle_incoming_Z_PDU
2842                 m_initRequest_apdu = m_s2z_init_apdu;
2843                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2844             }
2845             else
2846                 send_srw_explain_response(0, 0);
2847             return;
2848         }
2849         else if (srw_pdu->which == Z_SRW_scan_request)
2850         {
2851             m_s2z_database = odr_strdup(m_s2z_odr_init,
2852                                         srw_pdu->u.scan_request->database);
2853
2854             yaz_add_srw_diagnostic(odr_decode(),
2855                                    &diagnostic, &num_diagnostic,
2856                                    4, "scan");
2857             Z_SRW_PDU *srw_pdu =
2858                 yaz_srw_get(odr_encode(),
2859                             Z_SRW_scan_response);
2860             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2861
2862             srw_res->diagnostics = diagnostic;
2863             srw_res->num_diagnostics = num_diagnostic;
2864             send_srw_response(srw_pdu);
2865             return;
2866         }
2867         else
2868         {
2869             m_s2z_database = 0;
2870
2871             send_to_srw_client_error(4, 0);
2872         }
2873     }
2874     send_http_response(400);
2875 }
2876
2877 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2878 {
2879
2880     Z_OtherInformation **oi;
2881     get_otherInfoAPDU(apdu, &oi);
2882
2883     if (apdu->u.initRequest->implementationId)
2884         yaz_log(YLOG_LOG, "%simplementationId: %s",
2885                 m_session_str, apdu->u.initRequest->implementationId);
2886     if (apdu->u.initRequest->implementationName)
2887         yaz_log(YLOG_LOG, "%simplementationName: %s",
2888                 m_session_str, apdu->u.initRequest->implementationName);
2889     if (apdu->u.initRequest->implementationVersion)
2890         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2891                 m_session_str, apdu->u.initRequest->implementationVersion);
2892     if (m_initRequest_apdu == 0)
2893     {
2894         if (m_initRequest_mem)
2895             nmem_destroy(m_initRequest_mem);
2896
2897         m_initRequest_apdu = apdu;
2898         m_initRequest_mem = odr_extract_mem(odr_decode());
2899
2900         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2901             preferredMessageSize;
2902         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2903         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2904             maximumRecordSize;
2905         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2906
2907         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2908             yaz_get_charneg_record(*oi);
2909
2910         // Save proposal charsets and langs.
2911         if (ODR_MASK_GET(apdu->u.initRequest->options,
2912                          Z_Options_negotiationModel)
2913             && charSetandLangRecord)
2914         {
2915
2916             yaz_get_proposal_charneg(m_referenceId_mem,
2917                                      charSetandLangRecord,
2918                                      &m_initRequest_oi_negotiation_charsets,
2919                                      &m_initRequest_oi_negotiation_num_charsets,
2920                                      &m_initRequest_oi_negotiation_langs,
2921                                      &m_initRequest_oi_negotiation_num_langs,
2922                                      &m_initRequest_oi_negotiation_selected);
2923
2924             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2925             {
2926                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2927                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2928                         m_initRequest_oi_negotiation_charsets[i]:"none");
2929             }
2930             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2931             {
2932                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2933                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2934                         m_initRequest_oi_negotiation_langs[i]:"none");
2935             }
2936             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2937                     m_session_str, m_initRequest_oi_negotiation_selected);
2938         }
2939         // save init options for the response..
2940         m_initRequest_options = apdu->u.initRequest->options;
2941
2942         apdu->u.initRequest->options =
2943             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2944                                        sizeof(Odr_bitmask));
2945         ODR_MASK_ZERO(apdu->u.initRequest->options);
2946         int i;
2947         for (i = 0; i<= 24; i++)
2948             ODR_MASK_SET(apdu->u.initRequest->options, i);
2949         // check negotiation option
2950         if (!ODR_MASK_GET(m_initRequest_options,
2951                           Z_Options_negotiationModel))
2952         {
2953             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2954                            Z_Options_negotiationModel);
2955         }
2956         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2957                        Z_Options_concurrentOperations);
2958         // make new version
2959         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2960         apdu->u.initRequest->protocolVersion =
2961             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2962                                        sizeof(Odr_bitmask));
2963         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2964
2965         for (i = 0; i<= 8; i++)
2966             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2967     }
2968     handle_charset_lang_negotiation(apdu);
2969     if (m_client->m_init_flag)
2970     {
2971         if (handle_init_response_for_invalid_session(apdu))
2972             return;
2973         if (m_client->m_initResponse)
2974         {
2975             Z_APDU *apdu2 = m_client->m_initResponse;
2976             apdu2->u.initResponse->otherInfo = 0;
2977             if (m_client->m_cookie && *m_client->m_cookie)
2978                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2979                                            m_client->m_cookie);
2980             apdu2->u.initResponse->referenceId =
2981                 apdu->u.initRequest->referenceId;
2982             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2983             apdu2->u.initResponse->protocolVersion =
2984                 m_client->m_initResponse_version;
2985
2986             handle_charset_lang_negotiation(apdu2);
2987
2988             send_to_client(apdu2);
2989             m_timeout_mode = timeout_normal;
2990             return;
2991         }
2992     }
2993     m_client->m_init_flag = 1;
2994
2995 #if USE_AUTH_MSG
2996     Auth_Msg *m = new Auth_Msg;
2997     m->m_proxy = this;
2998     z_APDU(odr_encode(), &apdu, 0, "encode");
2999     char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
3000     m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
3001     memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
3002     odr_reset(odr_encode());
3003
3004     inc_ref();
3005     m_my_thread->put(m);
3006 #else
3007     int ret = handle_authentication(apdu);
3008     result_authentication(apdu, ret);
3009 #endif
3010 }
3011
3012 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
3013 {
3014     Z_ReferenceId **refid = get_referenceIdP(apdu);
3015     nmem_reset(m_referenceId_mem);
3016     if (refid && *refid)
3017     {
3018         m_referenceId = (Z_ReferenceId *)
3019             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
3020         m_referenceId->len = m_referenceId->size = (*refid)->len;
3021         m_referenceId->buf = (unsigned char *)
3022             nmem_malloc(m_referenceId_mem, (*refid)->len);
3023         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
3024     }
3025     else
3026         m_referenceId = 0;
3027
3028     if (!m_client && m_flag_invalid_session)
3029     {
3030         // Got request for a session that is invalid..
3031         m_apdu_invalid_session = apdu; // save package
3032         m_mem_invalid_session = odr_extract_mem(odr_decode());
3033         apdu = m_initRequest_apdu;     // but throw an init to the target
3034     }
3035
3036     // Determine our client.
3037     Z_OtherInformation **oi;
3038     get_otherInfoAPDU(apdu, &oi);
3039     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
3040     if (!m_client)
3041     {
3042         if (m_http_version)
3043         {   // HTTP. Send not found
3044             send_http_response(404);
3045             return;
3046         }
3047         else
3048         {
3049             // Z39.50 just shutdown
3050             delete this;
3051             return;
3052         }
3053     }
3054
3055     m_client->m_server = this;
3056
3057     if (apdu->which == Z_APDU_initRequest)
3058         handle_init(apdu);
3059     else
3060         handle_incoming_Z_PDU_2(apdu);
3061 }
3062
3063 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
3064 {
3065     handle_max_record_retrieve(apdu);
3066
3067     if (apdu)
3068         apdu = handle_syntax_validation(apdu);
3069
3070     if (apdu)
3071         apdu = handle_query_transformation(apdu);
3072
3073     if (apdu)
3074         apdu = handle_target_charset_conversion(apdu);
3075
3076     if (apdu)
3077         apdu = handle_query_validation(apdu);
3078
3079     if (apdu)
3080         apdu = result_set_optimize(apdu);
3081
3082     if (!apdu)
3083     {
3084         m_client->timeout(m_target_idletime);  // mark it active even
3085         recv_GDU_more(true);
3086         // though we didn't use it
3087         return;
3088     }
3089
3090     // delete other info construct completely if 0 elements
3091     Z_OtherInformation **oi;
3092     get_otherInfoAPDU(apdu, &oi);
3093     if (oi && *oi && (*oi)->num_elements == 0)
3094         *oi = 0;
3095
3096     if (apdu->which == Z_APDU_presentRequest &&
3097         m_client->m_resultSetStartPoint == 0)
3098     {
3099         Z_PresentRequest *pr = apdu->u.presentRequest;
3100         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3101         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3102     } else {
3103         m_client->m_resultSetStartPoint = 0;
3104     }
3105     if (m_client->send_to_target(apdu) < 0)
3106     {
3107         m_client->shutdown();
3108     }
3109     else
3110         m_client->m_waiting = 1;
3111 }
3112
3113 void Yaz_Proxy::connectNotify()
3114 {
3115 }
3116
3117 void Yaz_Proxy::releaseClient()
3118 {
3119     xfree(m_proxyTarget);
3120     m_proxyTarget = 0;
3121     m_flag_invalid_session = 0;
3122     // only keep if keep_alive flag is set...
3123     if (m_client &&
3124         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3125         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3126         m_client->m_waiting == 0)
3127     {
3128         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3129                  m_session_str,
3130                  m_client->get_hostname());
3131         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3132                 m_session_str, m_client->m_pdu_recv,
3133                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3134                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3135         assert (m_client->m_waiting != 2);
3136         // Tell client (if any) that no server connection is there..
3137         m_client->m_server = 0;
3138         m_client = 0;
3139     }
3140     else if (m_client)
3141     {
3142         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
3143                  m_session_str,
3144                  m_client->get_hostname());
3145         assert (m_client->m_waiting != 2);
3146         delete m_client;
3147         m_client = 0;
3148     }
3149     else if (!m_parent)
3150     {
3151         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
3152                  m_session_str);
3153         assert (m_parent);
3154     }
3155     else
3156     {
3157         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
3158                  m_session_str);
3159     }
3160     if (m_parent)
3161         m_parent->pre_init();
3162 }
3163
3164 bool Yaz_Proxy::dec_ref(bool main_ptr)
3165 {
3166     assert(m_ref_count > 0);
3167     if (main_ptr)
3168     {
3169         if (m_main_ptr_dec)
3170             return false;
3171         m_main_ptr_dec = true;
3172     }
3173
3174     m_http_keepalive = 0;
3175
3176     --m_ref_count;
3177     if (m_ref_count > 0)
3178         return false;
3179
3180     releaseClient();
3181
3182     delete this;
3183     return true;
3184 }
3185
3186 const char *Yaz_ProxyClient::get_session_str()
3187 {
3188     if (!m_server)
3189         return "0 ";
3190     return m_server->get_session_str();
3191 }
3192
3193 void Yaz_ProxyClient::shutdown()
3194 {
3195     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3196              get_hostname());
3197
3198     if (m_server)
3199     {
3200         m_waiting = 1;      // ensure it's released from Proxy in releaseClient
3201         m_server->dec_ref(true);
3202     }
3203     else
3204         delete this;
3205 }
3206
3207 void Yaz_Proxy::failNotify()
3208 {
3209     inc_request_no();
3210     yaz_log (YLOG_LOG, "%sConnection closed by client",
3211              get_session_str());
3212     dec_ref(true);
3213 }
3214
3215 void Yaz_Proxy::send_response_fail_client(const char *addr)
3216 {
3217     if (m_http_version)
3218     {
3219         Z_SRW_diagnostic *diagnostic = 0;
3220         int num_diagnostic = 0;
3221
3222         yaz_add_srw_diagnostic(odr_encode(),
3223                                &diagnostic, &num_diagnostic,
3224                                YAZ_SRW_SYSTEM_TEMPORARILY_UNAVAILABLE, addr);
3225         if (m_s2z_search_apdu)
3226             send_srw_search_response(diagnostic, num_diagnostic);
3227         else
3228             send_srw_explain_response(diagnostic, num_diagnostic);
3229     }
3230 }
3231 void Yaz_ProxyClient::failNotify()
3232 {
3233     if (m_server)
3234         m_server->inc_request_no();
3235     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
3236              get_session_str(), get_hostname());
3237
3238     if (m_server)
3239         m_server->send_response_fail_client(get_hostname());
3240     shutdown();
3241 }
3242
3243 void Yaz_ProxyClient::connectNotify()
3244 {
3245     const char *s = get_session_str();
3246     const char *h = get_hostname();
3247     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3248              m_target_idletime);
3249     timeout(m_target_idletime);
3250     if (!m_server)
3251         pre_init_client();
3252 }
3253
3254 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3255                                               *the_PDU_Observable, int fd)
3256 {
3257     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3258 }
3259
3260 Yaz_ProxyClient::~Yaz_ProxyClient()
3261 {
3262     if (m_prev)
3263         *m_prev = m_next;
3264     if (m_next)
3265         m_next->m_prev = m_prev;
3266     m_waiting = 2;     // for debugging purposes only.
3267     odr_destroy(m_init_odr);
3268     odr_destroy(m_idAuthentication_odr);
3269     delete m_last_query;
3270     xfree (m_last_resultSetId);
3271     xfree (m_cookie);
3272 }
3273
3274 void Yaz_ProxyClient::pre_init_client()
3275 {
3276     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3277     Z_InitRequest *req = apdu->u.initRequest;
3278
3279     int i;
3280     for (i = 0; i<= 24; i++)
3281         ODR_MASK_SET(req->options, i);
3282     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3283                    Z_Options_negotiationModel);
3284     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3285                    Z_Options_concurrentOperations);
3286     for (i = 0; i<= 10; i++)
3287         ODR_MASK_SET(req->protocolVersion, i);
3288
3289     if (send_to_target(apdu) < 0)
3290     {
3291         delete this;
3292     }
3293     else
3294     {
3295         m_waiting = 1;
3296         m_init_flag = 1;
3297     }
3298 }
3299
3300 void Yaz_Proxy::pre_init()
3301 {
3302     int i;
3303     const char *name = 0;
3304     const char *zurl_in_use[MAX_ZURL_PLEX];
3305     int limit_bw, limit_pdu, limit_req, limit_search;
3306     int target_idletime, client_idletime;
3307     int max_clients;
3308     int keepalive_limit_bw, keepalive_limit_pdu;
3309     int pre_init;
3310     const char *cql2rpn = 0;
3311     const char *authentication = 0;
3312     const char *negotiation_charset = 0;
3313     const char *negotiation_lang = 0;
3314
3315     Yaz_ProxyConfig *cfg = check_reconfigure();
3316
3317     zurl_in_use[0] = 0;
3318
3319     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3320         set_APDU_yazlog(1);
3321     else
3322         set_APDU_yazlog(0);
3323
3324     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3325                                           &limit_bw, &limit_pdu, &limit_req,
3326                                           &limit_search,
3327                                           &target_idletime, &client_idletime,
3328                                           &max_clients,
3329                                           &keepalive_limit_bw,
3330                                           &keepalive_limit_pdu,
3331                                           &pre_init,
3332                                           &cql2rpn,
3333                                           &authentication,
3334                                           &negotiation_charset,
3335                                           &negotiation_lang,
3336                                           0,
3337                                           0) ; i++)
3338     {
3339         if (pre_init)
3340         {
3341             int j;
3342             for (j = 0; zurl_in_use[j]; j++)
3343             {
3344                 Yaz_ProxyClient *c;
3345                 int spare = 0;
3346                 int spare_waiting = 0;
3347                 int in_use = 0;
3348                 int other = 0;
3349                 for (c = m_clientPool; c; c = c->m_next)
3350                 {
3351                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3352                     {
3353                         if (c->m_cookie == 0)
3354                         {
3355                             if (c->m_server == 0)
3356                                 if (c->m_waiting)
3357                                     spare_waiting++;
3358                                 else
3359                                     spare++;
3360                             else
3361                                 in_use++;
3362                         }
3363                         else
3364                             other++;
3365                     }
3366                 }
3367                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3368                         "sparew=%d preinit=%d",m_session_str,
3369                         name, zurl_in_use[j], in_use, other,
3370                         spare, spare_waiting, pre_init);
3371                 if (spare + spare_waiting < pre_init)
3372                 {
3373                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3374                     c->m_next = m_clientPool;
3375                     if (c->m_next)
3376                         c->m_next->m_prev = &c->m_next;
3377                     m_clientPool = c;
3378                     c->m_prev = &m_clientPool;
3379
3380                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3381                         c->set_APDU_yazlog(1);
3382                     else
3383                         c->set_APDU_yazlog(0);
3384
3385                     if (c->client(zurl_in_use[j]))
3386                     {
3387                         timeout(60);
3388                         delete c;
3389                         return;
3390                     }
3391                     c->timeout(30);
3392                     c->m_waiting = 1;
3393                     c->m_target_idletime = target_idletime;
3394                     c->m_seqno = m_seqno++;
3395                 }
3396             }
3397         }
3398     }
3399 }
3400
3401 void Yaz_Proxy::timeoutNotify()
3402 {
3403     if (m_parent)
3404     {
3405         GDU *gdu;
3406         switch(m_timeout_mode)
3407         {
3408         case timeout_normal:
3409         case timeout_busy:
3410             inc_request_no();
3411             m_in_queue.clear();
3412             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3413             dec_ref(true);
3414             break;
3415         case timeout_reduce:
3416             timeout(m_client_idletime);
3417             m_timeout_mode = timeout_busy;
3418             gdu = m_timeout_gdu;
3419             m_timeout_gdu = 0;
3420             recv_GDU_normal(gdu);
3421             break;
3422         case timeout_xsl:
3423             assert(m_stylesheet_nprl);
3424             convert_xsl_delay();
3425             recv_GDU_more(true);
3426         }
3427     }
3428     else
3429     {
3430         timeout(600);
3431         pre_init();
3432     }
3433 }
3434
3435 void Yaz_Proxy::markInvalid()
3436 {
3437     m_client = 0;
3438     m_flag_invalid_session = 1;
3439 }
3440
3441 void Yaz_ProxyClient::timeoutNotify()
3442 {
3443     if (m_server)
3444         m_server->inc_request_no();
3445
3446     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3447              get_hostname());
3448
3449     if (m_server)
3450         m_server->send_response_fail_client(get_hostname());
3451     shutdown();
3452
3453     m_root->pre_init();
3454 }
3455
3456 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
3457                                  Yaz_Proxy *parent) :
3458     Z_Assoc (the_PDU_Observable)
3459 {
3460     m_cookie = 0;
3461     m_next = 0;
3462     m_prev = 0;
3463     m_init_flag = 0;
3464     m_last_query = 0;
3465     m_last_resultSetId = 0;
3466     m_last_resultCount = 0;
3467     m_last_ok = 0;
3468     m_sr_transform = 0;
3469     m_waiting = 0;
3470     m_init_odr = odr_createmem (ODR_DECODE);
3471     m_initResponse = 0;
3472     m_initResponse_options = 0;
3473     m_initResponse_version = 0;
3474     m_initResponse_preferredMessageSize = 0;
3475     m_initResponse_maximumRecordSize = 0;
3476     m_resultSetStartPoint = 0;
3477     m_bytes_sent = m_bytes_recv = 0;
3478     m_pdu_recv = 0;
3479     m_server = 0;
3480     m_seqno = 0;
3481     m_target_idletime = 600;
3482     m_root = parent;
3483     m_idAuthentication_odr = odr_createmem(ODR_ENCODE);
3484     m_idAuthentication_ber_buf = 0;
3485     m_idAuthentication_ber_size = 0;
3486 }
3487
3488 const char *Yaz_Proxy::option(const char *name, const char *value)
3489 {
3490     if (!strcmp (name, "optimize")) {
3491         if (value) {
3492             xfree (m_optimize);
3493             m_optimize = xstrdup (value);
3494         }
3495         return m_optimize;
3496     }
3497     return 0;
3498 }
3499
3500 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
3501 {
3502
3503 }
3504
3505 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
3506 {
3507     if (apdu->which == Z_GDU_Z3950)
3508         recv_Z_PDU(apdu->u.z3950, len);
3509     else if (apdu->which == Z_GDU_HTTP_Response)
3510         recv_HTTP_response(apdu->u.HTTP_Response, len);
3511     else
3512         shutdown();
3513 }
3514
3515 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
3516 {
3517     if (!m_flag_invalid_session)
3518         return 0;
3519     m_flag_invalid_session = 0;
3520     handle_incoming_Z_PDU(m_apdu_invalid_session);
3521     assert (m_mem_invalid_session);
3522     nmem_destroy(m_mem_invalid_session);
3523     m_mem_invalid_session = 0;
3524     return 1;
3525 }
3526
3527 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
3528 {
3529     m_bytes_recv += len;
3530
3531     m_pdu_recv++;
3532     m_waiting = 0;
3533     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
3534         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
3535                  apdu_name(apdu), get_hostname(), len);
3536     if (apdu->which == Z_APDU_initResponse)
3537     {
3538         if (!m_server)  // if this is a pre init session , check for more
3539             m_root->pre_init();
3540         NMEM nmem = odr_extract_mem (odr_decode());
3541         odr_reset (m_init_odr);
3542         nmem_transfer (m_init_odr->mem, nmem);
3543         m_initResponse = apdu;
3544         m_initResponse_options = apdu->u.initResponse->options;
3545         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3546         m_initResponse_preferredMessageSize =
3547             *apdu->u.initResponse->preferredMessageSize;
3548         m_initResponse_maximumRecordSize =
3549             *apdu->u.initResponse->maximumRecordSize;
3550
3551         Z_InitResponse *ir = apdu->u.initResponse;
3552         char *im0 = ir->implementationName;
3553
3554         char *im1 = (char*)
3555             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3556         *im1 = '\0';
3557         if (im0)
3558         {
3559             strcat(im1, im0);
3560             strcat(im1, " ");
3561         }
3562         strcat(im1, "(YAZ Proxy)");
3563         ir->implementationName = im1;
3564
3565         nmem_destroy (nmem);
3566
3567         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3568             return;
3569     }
3570     if (apdu->which == Z_APDU_searchResponse)
3571     {
3572         Z_SearchResponse *sr = apdu->u.searchResponse;
3573         m_last_resultCount = *sr->resultCount;
3574         int status = *sr->searchStatus;
3575         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3576         {
3577             m_last_ok = 1;
3578
3579             if (sr->records && sr->records->which == Z_Records_DBOSD)
3580             {
3581                 m_cache.add(odr_decode(),
3582                             sr->records->u.databaseOrSurDiagnostics, 1,
3583                             *sr->resultCount);
3584             }
3585         }
3586     }
3587     if (apdu->which == Z_APDU_presentResponse)
3588     {
3589         Z_PresentResponse *pr = apdu->u.presentResponse;
3590         if (m_sr_transform)
3591         {
3592             m_sr_transform = 0;
3593             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3594             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3595             sr->referenceId = pr->referenceId;
3596             *sr->resultCount = m_last_resultCount;
3597             sr->records = pr->records;
3598             sr->nextResultSetPosition = pr->nextResultSetPosition;
3599             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3600             apdu = new_apdu;
3601         }
3602         if (pr->records &&
3603             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3604         {
3605             m_cache.add(odr_decode(),
3606                         pr->records->u.databaseOrSurDiagnostics,
3607                         m_resultSetStartPoint, -1);
3608             m_resultSetStartPoint = 0;
3609         }
3610     }
3611     if (m_cookie)
3612         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3613
3614     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
3615
3616     if (server)
3617         server->send_to_client(apdu);
3618     if (apdu->which == Z_APDU_close)
3619         shutdown();
3620     else if (server)
3621         server->recv_GDU_more(true);
3622 }
3623
3624 void Yaz_Proxy::low_socket_close()
3625 {
3626 #if WIN32
3627 #else
3628     int i;
3629     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3630         if  (m_lo_fd[i] >= 0)
3631             ::close(m_lo_fd[i]);
3632 #endif
3633 }
3634
3635 void Yaz_Proxy::low_socket_open()
3636 {
3637 #if WIN32
3638 #else
3639     int i;
3640     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3641         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3642 #endif
3643 }
3644
3645 int Yaz_Proxy::server(const char *addr)
3646 {
3647     int r = Z_Assoc::server(addr);
3648     if (!r)
3649     {
3650         yaz_log(YLOG_LOG, "%sStarted proxy "
3651 #ifdef VERSION
3652             VERSION
3653 #endif
3654             " on %s", m_session_str, addr);
3655         timeout(1);
3656     }
3657     return r;
3658 }
3659
3660 void Yaz_Proxy::base64_decode(const char *base64, char *buf, int buf_len)
3661 {
3662     const char *base64_chars =
3663         "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
3664     int len = strlen(base64);
3665     int buf_pos = 0;
3666     int index = 1;
3667
3668     for (int pos = 0; pos <= len; pos++)
3669     {
3670         if (base64[pos] == '=' || buf_pos + 1 >= buf_len)
3671             break;
3672
3673         const char *ch_ptr = strchr(base64_chars, base64[pos]);
3674         if (!ch_ptr)
3675             break;
3676         char ch = (char) (ch_ptr - base64_chars);
3677         switch (index)
3678         {
3679             case 1:
3680                 buf[buf_pos] = ch << 2;
3681                 break;
3682             case 2:
3683                 buf[buf_pos++] += (ch & 0x30) >> 4;
3684                 buf[buf_pos] = (ch & 0x0f) << 4;
3685                 break;
3686             case 3:
3687                 buf[buf_pos++] += (ch & 0x3c) >> 2;
3688                 buf[buf_pos] = (ch & 0x03) << 6;
3689                 break;
3690             case 4:
3691                 buf[buf_pos++] += ch;
3692         }
3693         if (index < 4)
3694             index++;
3695         else
3696             index = 1;
3697     }
3698     buf[buf_pos] = '\0';
3699 }
3700
3701 /*
3702  * Local variables:
3703  * c-basic-offset: 4
3704  * indent-tabs-mode: nil
3705  * End:
3706  * vim: shiftwidth=4 tabstop=8 expandtab
3707  */
3708