src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.50 2006-04-06 12:04:20 adam Exp $
   2    Copyright (c) 1998-2006, Index Data.
   3
   4 This file is part of the yazproxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <stdlib.h>
  42 #include <time.h>
  43 #include <fcntl.h>
  44
  45 #include <yaz/srw.h>
  46 #include <yaz/marcdisp.h>
  47 #include <yaz/yaz-iconv.h>
  48 #include <yaz/log.h>
  49 #include <yaz/diagbib1.h>
  50 #include "proxyp.h"
  51 #include <yaz/pquery.h>
  52 #include <yaz/otherinfo.h>
  53 #include <yaz/charneg.h>
  54 #include "msg-thread.h"
  55
  56 using namespace yazpp_1;
  57
  58 #ifdef WIN32
  59 #define strncasecmp _strnicmp
  60 #endif
  61
  62 #define USE_AUTH_MSG 1
  63
  64 #if USE_AUTH_MSG
  65 class YAZ_EXPORT Auth_Msg : public IMsg_Thread {
  66 public:
  67     int m_ret;
  68     IMsg_Thread *handle();
  69     void result();
  70     Yaz_Proxy *m_proxy;
  71     NMEM m_nmem;
  72     char *m_apdu_buf;
  73     int m_apdu_len;
  74     Auth_Msg();
  75     virtual ~Auth_Msg();
  76 };
  77
  78 Auth_Msg::Auth_Msg()
  79 {
  80     m_nmem = nmem_create();
  81 }
  82
  83 Auth_Msg::~Auth_Msg()
  84 {
  85     nmem_destroy(m_nmem);
  86 }
  87
  88 IMsg_Thread *Auth_Msg::handle()
  89 {
  90     ODR decode = odr_createmem(ODR_DECODE);
  91     Z_APDU *apdu;
  92
  93     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
  94     int r = z_APDU(decode, &apdu, 0, 0);
  95     if (!r)
  96     {
  97         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
  98     }
  99     else
 100     {
 101         m_ret = m_proxy->handle_authentication(apdu);
 102     }
 103     odr_destroy(decode);
 104     return this;
 105 }
 106
 107 void Auth_Msg::result()
 108 {
 109     if (m_proxy->dec_ref(false))
 110     {
 111         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
 112     }
 113     else
 114     {
 115         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
 116         Z_APDU *apdu = 0;
 117         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
 118         if (!r)
 119             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
 120         m_proxy->result_authentication(apdu, m_ret);
 121     }
 122     delete this;
 123 }
 124
 125 #endif
 126
 127 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
 128 {
 129     if (apdu == 0 || ret == 0)
 130     {
 131         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
 132         *apdu_reject->u.initResponse->result = 0;
 133         send_to_client(apdu_reject);
 134         dec_ref(false);
 135     }
 136     else
 137     {
 138         if (apdu->which == Z_APDU_initRequest)
 139         {
 140             Yaz_ProxyConfig *cfg = check_reconfigure();
 141             if (cfg)
 142                 cfg->target_authentication(m_default_target, odr_encode(),
 143                                            apdu->u.initRequest);
 144         }
 145         handle_incoming_Z_PDU_2(apdu);
 146     }
 147 }
 148
 149 static const char *apdu_name(Z_APDU *apdu)
 150 {
 151     switch (apdu->which)
 152     {
 153     case Z_APDU_initRequest:
 154         return "initRequest";
 155     case Z_APDU_initResponse:
 156         return "initResponse";
 157     case Z_APDU_searchRequest:
 158         return "searchRequest";
 159     case Z_APDU_searchResponse:
 160         return "searchResponse";
 161     case Z_APDU_presentRequest:
 162         return "presentRequest";
 163     case Z_APDU_presentResponse:
 164         return "presentResponse";
 165     case Z_APDU_deleteResultSetRequest:
 166         return "deleteResultSetRequest";
 167     case Z_APDU_deleteResultSetResponse:
 168         return "deleteResultSetResponse";
 169     case Z_APDU_scanRequest:
 170         return "scanRequest";
 171     case Z_APDU_scanResponse:
 172         return "scanResponse";
 173     case Z_APDU_sortRequest:
 174         return "sortRequest";
 175     case Z_APDU_sortResponse:
 176         return "sortResponse";
 177     case Z_APDU_extendedServicesRequest:
 178         return "extendedServicesRequest";
 179     case Z_APDU_extendedServicesResponse:
 180         return "extendedServicesResponse";
 181     case Z_APDU_close:
 182         return "close";
 183     }
 184     return "other";
 185 }
 186
 187 static const char *gdu_name(Z_GDU *gdu)
 188 {
 189     switch(gdu->which)
 190     {
 191     case Z_GDU_Z3950:
 192         return apdu_name(gdu->u.z3950);
 193     case Z_GDU_HTTP_Request:
 194         return "HTTP Request";
 195     case Z_GDU_HTTP_Response:
 196         return "HTTP Response";
 197     }
 198     return "Unknown request/response";
 199 }
 200
 201 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
 202                      ISocketObservable *the_socket_observable,
 203                      Yaz_Proxy *parent)
 204     :
 205     Z_Assoc(the_PDU_Observable),
 206     m_bw_stat(60), m_pdu_stat(60), m_search_stat(60)
 207 {
 208     m_PDU_Observable = the_PDU_Observable;
 209     m_socket_observable = the_socket_observable;
 210     m_client = 0;
 211     m_parent = parent;
 212     m_clientPool = 0;
 213     m_seqno = 1;
 214     m_keepalive_limit_bw = 500000;
 215     m_keepalive_limit_pdu = 1000;
 216     m_proxyTarget = 0;
 217     m_default_target = 0;
 218     m_proxy_negotiation_charset = 0;
 219     m_proxy_negotiation_lang = 0;
 220     m_proxy_negotiation_default_charset = 0;
 221     m_charset_converter = new Yaz_CharsetConverter;
 222     m_max_clients = 150;
 223     m_log_mask = 0;
 224     m_seed = time(0);
 225     m_client_idletime = 600;
 226     m_target_idletime = 600;
 227     m_optimize = xstrdup ("1");
 228     strcpy(m_session_str, "0 ");
 229     m_session_no = 0;
 230     m_bytes_sent = 0;
 231     m_bytes_recv = 0;
 232     m_bw_max = 0;
 233     m_pdu_max = 0;
 234     m_search_max = 0;
 235     m_connect_max = 0;
 236     m_limit_connect = 0;
 237     m_timeout_mode = timeout_normal;
 238     m_timeout_gdu = 0;
 239     m_max_record_retrieve = 0;
 240     m_reconfig_flag = 0;
 241     m_config_fname = 0;
 242     m_request_no = 0;
 243     m_flag_invalid_session = 0;
 244     m_referenceId = 0;
 245     m_referenceId_mem = nmem_create();
 246     m_config = 0;
 247     m_marcxml_mode = none;
 248     m_stylesheet_xsp = 0;
 249     m_stylesheet_nprl = 0;
 250     m_stylesheet_apdu = 0;
 251     m_s2z_stylesheet = 0;
 252     m_s2z_database = 0;
 253     m_schema = 0;
 254     m_backend_type = 0;
 255     m_backend_charset = 0;
 256     m_frontend_type = 0;
 257     m_initRequest_apdu = 0;
 258     m_initRequest_mem = 0;
 259     m_initRequest_preferredMessageSize = 0;
 260     m_initRequest_maximumRecordSize = 0;
 261     m_initRequest_options = 0;
 262     m_initRequest_version = 0;
 263     m_initRequest_oi_negotiation_charsets = 0;
 264     m_initRequest_oi_negotiation_num_charsets = 0;
 265     m_initRequest_oi_negotiation_langs = 0;
 266     m_initRequest_oi_negotiation_num_langs = 0;
 267     m_initRequest_oi_negotiation_selected = 0;
 268     m_apdu_invalid_session = 0;
 269     m_mem_invalid_session = 0;
 270     m_s2z_odr_init = 0;
 271     m_s2z_odr_search = 0;
 272     m_s2z_init_apdu = 0;
 273     m_s2z_search_apdu = 0;
 274     m_s2z_present_apdu = 0;
 275     m_http_keepalive = 0;
 276     m_http_version = 0;
 277     m_soap_ns = 0;
 278     m_s2z_packing = Z_SRW_recordPacking_string;
 279 #if HAVE_GETTIMEOFDAY
 280     m_time_tv = xmalloc(sizeof(struct timeval));
 281     struct timeval *tv = (struct timeval *) m_time_tv;
 282     tv->tv_sec = 0;
 283     tv->tv_usec = 0;
 284 #else
 285     m_time_tv = 0;
 286 #endif
 287     m_usemarcon_ini_stage1 = 0;
 288     m_usemarcon_ini_stage2 = 0;
 289     m_usemarcon = new Yaz_usemarcon();
 290     if (!m_parent)
 291         low_socket_open();
 292     m_my_thread = 0;
 293     m_ref_count = 1;
 294     m_main_ptr_dec = false;
 295     m_peername = 0;
 296     m_initial_reduce = 0;
 297 }
 298
 299 void Yaz_Proxy::inc_ref()
 300 {
 301     m_ref_count++;
 302 }
 303
 304 Yaz_Proxy::~Yaz_Proxy()
 305 {
 306     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 307             m_bytes_sent, m_bytes_recv);
 308     nmem_destroy(m_initRequest_mem);
 309     nmem_destroy(m_mem_invalid_session);
 310     nmem_destroy(m_referenceId_mem);
 311
 312     xfree(m_proxyTarget);
 313     xfree(m_default_target);
 314     xfree(m_proxy_negotiation_charset);
 315     xfree(m_proxy_negotiation_lang);
 316     xfree(m_proxy_negotiation_default_charset);
 317     delete m_charset_converter;
 318     xfree(m_optimize);
 319
 320 #if HAVE_XSLT
 321     if (m_stylesheet_xsp)
 322         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 323 #endif
 324     xfree (m_time_tv);
 325
 326     xfree (m_peername);
 327     xfree (m_schema);
 328     xfree (m_backend_type);
 329     xfree (m_backend_charset);
 330     xfree (m_usemarcon_ini_stage1);
 331     xfree (m_usemarcon_ini_stage2);
 332     delete m_usemarcon;
 333     if (m_s2z_odr_init)
 334         odr_destroy(m_s2z_odr_init);
 335     if (m_s2z_odr_search)
 336         odr_destroy(m_s2z_odr_search);
 337     if (!m_parent)
 338         low_socket_close();
 339     if (!m_parent)
 340         delete m_my_thread;
 341     delete m_config;
 342 }
 343
 344 void Yaz_Proxy::set_debug_mode(int mode)
 345 {
 346     m_debug_mode = mode;
 347 }
 348
 349 int Yaz_Proxy::set_config(const char *config)
 350 {
 351     delete m_config;
 352     m_config = new Yaz_ProxyConfig();
 353     xfree(m_config_fname);
 354     m_config_fname = xstrdup(config);
 355     int r = m_config->read_xml(config);
 356     if (!r)
 357         m_config->get_generic_info(&m_log_mask, &m_max_clients,
 358                                    &m_connect_max, &m_limit_connect);
 359     return r;
 360 }
 361
 362 void Yaz_Proxy::set_default_target(const char *target)
 363 {
 364     xfree (m_default_target);
 365     m_default_target = 0;
 366     if (target)
 367         m_default_target = (char *) xstrdup (target);
 368 }
 369
 370 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang,
 371                                        const char *default_charset)
 372 {
 373     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
 374         "default charset to '%s', language to '%s'", m_session_str,
 375         charset?charset:"none",
 376         default_charset?default_charset:"none",
 377         lang?lang:"none");
 378     xfree (m_proxy_negotiation_charset);
 379     xfree (m_proxy_negotiation_lang);
 380     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 381     if (charset)
 382         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 383     if (lang)
 384         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 385     if (default_charset)
 386         m_proxy_negotiation_default_charset =
 387             (char *) xstrdup (default_charset);
 388 }
 389
 390 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 391 {
 392     if (m_parent)
 393         return m_parent->check_reconfigure();
 394
 395     Yaz_ProxyConfig *cfg = m_config;
 396     if (m_reconfig_flag)
 397     {
 398         yaz_log(YLOG_LOG, "reconfigure");
 399         yaz_log_reopen();
 400         if (m_config_fname && cfg)
 401         {
 402             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 403             int r = cfg->read_xml(m_config_fname);
 404             if (r)
 405                 yaz_log(YLOG_WARN, "reconfigure failed");
 406             else
 407             {
 408                 m_log_mask = 0;
 409                 cfg->get_generic_info(&m_log_mask, &m_max_clients,
 410                                       &m_connect_max, &m_limit_connect);
 411             }
 412         }
 413         else
 414             yaz_log(YLOG_LOG, "reconfigure");
 415         m_reconfig_flag = 0;
 416     }
 417     return cfg;
 418 }
 419
 420 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 421                                         *the_PDU_Observable, int fd)
 422 {
 423     check_reconfigure();
 424
 425     char session_str[200];
 426     const char *peername = the_PDU_Observable->getpeername();
 427     if (m_log_mask & PROXY_LOG_IP_CLIENT)
 428         sprintf(session_str, "%ld:%d %s 0 ",
 429                 (long) time(0), m_session_no, peername);
 430     else
 431         sprintf(session_str, "%ld:%d 0 ",
 432                 (long) time(0), m_session_no);
 433     m_session_no++;
 434
 435     yaz_log (YLOG_LOG, "%sNew session %s", session_str, peername);
 436
 437     m_connect.cleanup(false);
 438     m_connect.add_connect(peername);
 439
 440     int connect_total = m_connect.get_total(peername);
 441     int connect_max = m_connect_max;
 442     if (connect_max && connect_total > connect_max)
 443     {
 444         yaz_log(YLOG_LOG, "%sconnect not accepted total=%d max=%d",
 445                 session_str, connect_total, connect_max);
 446         return 0;
 447     }
 448     yaz_log(YLOG_LOG, "%sconnect accepted total=%d", session_str,
 449             connect_total);
 450
 451     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 452                                          m_socket_observable, this);
 453
 454     if (m_limit_connect)
 455         new_proxy->m_initial_reduce = connect_total / m_limit_connect;
 456
 457     new_proxy->m_config = 0;
 458     new_proxy->m_config_fname = 0;
 459     new_proxy->timeout(m_client_idletime);
 460     new_proxy->m_target_idletime = m_target_idletime;
 461     new_proxy->set_default_target(m_default_target);
 462     new_proxy->m_max_clients = m_max_clients;
 463     new_proxy->m_log_mask = m_log_mask;
 464     new_proxy->set_APDU_log(get_APDU_log());
 465     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 466         new_proxy->set_APDU_yazlog(1);
 467     else
 468         new_proxy->set_APDU_yazlog(0);
 469     strcpy(new_proxy->m_session_str, session_str);
 470     new_proxy->m_peername = xstrdup(peername);
 471     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 472         m_proxy_negotiation_lang, m_proxy_negotiation_default_charset);
 473     // create thread object the first time we get an incoming connection
 474     if (!m_my_thread)
 475         m_my_thread = new Msg_Thread(m_socket_observable, 1);
 476     new_proxy->m_my_thread = m_my_thread;
 477     return new_proxy;
 478 }
 479
 480 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 481 {
 482     int oid[OID_SIZE];
 483     Z_OtherInformationUnit *oi;
 484     struct oident ent;
 485     ent.proto = PROTO_Z3950;
 486     ent.oclass = CLASS_USERINFO;
 487     ent.value = (oid_value) VAL_COOKIE;
 488     assert (oid_ent_to_oid (&ent, oid));
 489
 490     if (oid_ent_to_oid (&ent, oid) &&
 491         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 492         oi->which == Z_OtherInfo_characterInfo)
 493         return oi->information.characterInfo;
 494     return 0;
 495 }
 496 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 497 {
 498     int oid[OID_SIZE];
 499     Z_OtherInformationUnit *oi;
 500     struct oident ent;
 501     ent.proto = PROTO_Z3950;
 502     ent.oclass = CLASS_USERINFO;
 503     ent.value = (oid_value) VAL_PROXY;
 504     if (oid_ent_to_oid (&ent, oid) &&
 505         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 506         oi->which == Z_OtherInfo_characterInfo)
 507         return oi->information.characterInfo;
 508     return 0;
 509 }
 510 const char *Yaz_Proxy::load_balance(const char **url)
 511 {
 512     int zurl_in_use[MAX_ZURL_PLEX];
 513     int zurl_in_spare[MAX_ZURL_PLEX];
 514     Yaz_ProxyClient *c;
 515     int i;
 516
 517     for (i = 0; i<MAX_ZURL_PLEX; i++)
 518     {
 519         zurl_in_use[i] = 0;
 520         zurl_in_spare[i] = 0;
 521     }
 522     for (c = m_parent->m_clientPool; c; c = c->m_next)
 523     {
 524         for (i = 0; url[i]; i++)
 525             if (!strcmp(url[i], c->get_hostname()))
 526             {
 527                 zurl_in_use[i]++;
 528                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 529                     zurl_in_spare[i]++;
 530             }
 531     }
 532     int min_use = 100000;
 533     int spare_for_min = 0;
 534     int max_spare = 0;
 535     const char *ret_min = 0;
 536     const char *ret_spare = 0;
 537     for (i = 0; url[i]; i++)
 538     {
 539         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 540                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 541         if (min_use > zurl_in_use[i])
 542         {
 543             ret_min = url[i];
 544             min_use = zurl_in_use[i];
 545             spare_for_min = zurl_in_spare[i];
 546         }
 547         if (max_spare < zurl_in_spare[i])
 548         {
 549             ret_spare = url[i];
 550             max_spare = zurl_in_spare[i];
 551         }
 552     }
 553     return ret_min;
 554 }
 555
 556 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 557                                        const char *proxy_host)
 558 {
 559     assert (m_parent);
 560     Yaz_Proxy *parent = m_parent;
 561     Yaz_ProxyClient *c = m_client;
 562
 563     if (!m_proxyTarget)
 564     {
 565         const char *url[MAX_ZURL_PLEX];
 566         Yaz_ProxyConfig *cfg = check_reconfigure();
 567         if (proxy_host)
 568         {
 569             if (parent && parent->m_debug_mode)
 570             {
 571                 // only to be enabled for debugging...
 572                 if (!strcmp(proxy_host, "stop"))
 573                     exit(0);
 574             }
 575             xfree(m_default_target);
 576             m_default_target = xstrdup(proxy_host);
 577         }
 578         proxy_host = m_default_target;
 579         int client_idletime = -1;
 580         const char *cql2rpn_fname = 0;
 581         const char *negotiation_charset = 0;
 582         const char *negotiation_lang = 0;
 583         const char *query_charset = 0;
 584         const char *default_client_query_charset = 0;
 585         url[0] = m_default_target;
 586         url[1] = 0;
 587         if (cfg)
 588         {
 589             int pre_init = 0;
 590             cfg->get_target_info(proxy_host, url, &m_bw_max,
 591                                  &m_pdu_max, &m_max_record_retrieve,
 592                                  &m_search_max,
 593                                  &m_target_idletime, &client_idletime,
 594                                  &parent->m_max_clients,
 595                                  &m_keepalive_limit_bw,
 596                                  &m_keepalive_limit_pdu,
 597                                  &pre_init,
 598                                  &cql2rpn_fname,
 599                                  &negotiation_charset,
 600                                  &negotiation_lang,
 601                                  &query_charset,
 602                                  &default_client_query_charset);
 603         }
 604         if (client_idletime != -1)
 605         {
 606             m_client_idletime = client_idletime;
 607             timeout(m_client_idletime);
 608         }
 609         if (cql2rpn_fname)
 610             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 611         if (negotiation_charset || negotiation_lang || default_client_query_charset)
 612         {
 613             set_proxy_negotiation(negotiation_charset,
 614                 negotiation_lang, default_client_query_charset);
 615         }
 616         m_charset_converter->set_target_query_charset(query_charset);
 617         if (!url[0])
 618         {
 619             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 620             return 0;
 621         }
 622         // we don't handle multiplexing for cookie session, so we just
 623         // pick the first one in this case (anonymous users will be able
 624         // to use any backend)
 625         if (cookie && *cookie)
 626             m_proxyTarget = (char*) xstrdup(url[0]);
 627         else
 628             m_proxyTarget = (char*) xstrdup(load_balance(url));
 629     }
 630     if (cookie && *cookie)
 631     {   // search in sessions with a cookie
 632         for (c = parent->m_clientPool; c; c = c->m_next)
 633         {
 634             assert (c->m_prev);
 635             assert (*c->m_prev == c);
 636             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 637                 !strcmp(m_proxyTarget, c->get_hostname()))
 638             {
 639                 // Found it in cache
 640                 // The following handles "cancel"
 641                 // If connection is busy (waiting for PDU) and
 642                 // we have an initRequest we can safely do re-open
 643                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 644                 {
 645                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 646                              c->get_hostname());
 647                     c->close();
 648                     c->m_init_flag = 0;
 649
 650                     c->m_last_ok = 0;
 651                     c->m_cache.clear();
 652                     c->m_last_resultCount = 0;
 653                     c->m_sr_transform = 0;
 654                     c->m_waiting = 0;
 655                     c->m_resultSetStartPoint = 0;
 656                     c->m_target_idletime = m_target_idletime;
 657                     if (c->client(m_proxyTarget))
 658                     {
 659                         delete c;
 660                         return 0;
 661                     }
 662                     c->timeout(30);
 663                 }
 664                 c->m_seqno = parent->m_seqno;
 665                 if (c->m_server && c->m_server != this)
 666                     c->m_server->m_client = 0;
 667                 c->m_server = this;
 668                 (parent->m_seqno)++;
 669                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 670                 return c;
 671             }
 672         }
 673     }
 674     else if (!c && apdu->which == Z_APDU_initRequest )
 675     {
 676         // anonymous sessions without cookie.
 677         // if authentication is set it is NOT anonymous se we can't share them.
 678         // If charset and lang negotiation is use it is NOT anonymous session too.
 679         for (c = parent->m_clientPool; c; c = c->m_next)
 680         {
 681             assert(c->m_prev);
 682             assert(*c->m_prev == c);
 683             if (c->m_server == 0 && c->m_cookie == 0 &&  c->m_waiting == 0
 684                 && c->compare_idAuthentication(apdu)
 685                 && c->compare_charset(apdu)
 686                 && !strcmp(m_proxyTarget, c->get_hostname()))
 687             {
 688                 // found it in cache
 689                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 690                          m_session_str, parent->m_seqno, c->get_hostname());
 691
 692                 c->m_seqno = parent->m_seqno;
 693                 assert(c->m_server == 0);
 694                 c->m_server = this;
 695
 696                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 697                     c->set_APDU_yazlog(1);
 698                 else
 699                     c->set_APDU_yazlog(0);
 700
 701                 (parent->m_seqno)++;
 702
 703                 parent->pre_init();
 704
 705                 return c;
 706             }
 707         }
 708     }
 709     if (!m_client)
 710     {
 711         if (apdu->which != Z_APDU_initRequest)
 712         {
 713             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 714             return 0;
 715         }
 716         // go through list of clients - and find the lowest/oldest one.
 717         Yaz_ProxyClient *c_min = 0;
 718         int min_seq = -1;
 719         int no_of_clients = 0;
 720         if (parent->m_clientPool)
 721             yaz_log (YLOG_DEBUG, "Existing sessions");
 722         for (c = parent->m_clientPool; c; c = c->m_next)
 723         {
 724             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 725                                c->m_waiting, c->get_hostname(),
 726                                c->m_cookie ? c->m_cookie : "");
 727             no_of_clients++;
 728             if (min_seq < 0 || c->m_seqno < min_seq)
 729             {
 730                 min_seq = c->m_seqno;
 731                 c_min = c;
 732             }
 733         }
 734         if (no_of_clients >= parent->m_max_clients)
 735         {
 736             c = c_min;
 737             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 738             {
 739                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 740                          m_session_str, parent->m_max_clients, c->m_seqno);
 741                 if (c->m_server && c->m_server != this)
 742                     delete c->m_server;   // PROBLEM: m_ref_count!
 743                 c->m_server = 0;
 744             }
 745             else
 746             {
 747                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 748                          m_session_str, parent->m_max_clients,
 749                          c->m_seqno, parent->m_seqno, c->get_hostname());
 750                 xfree (c->m_cookie);
 751                 c->m_cookie = 0;
 752                 if (cookie)
 753                     c->m_cookie = xstrdup(cookie);
 754                 c->m_seqno = parent->m_seqno;
 755                 if (c->m_server && c->m_server != this)
 756                 {
 757                     c->m_server->m_client = 0;
 758                     delete c->m_server;   // PROBLEM: m_ref_count!
 759                 }
 760                 (parent->m_seqno)++;
 761                 c->m_target_idletime = m_target_idletime;
 762                 c->timeout(m_target_idletime);
 763
 764                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 765                     c->set_APDU_yazlog(1);
 766                 else
 767                     c->set_APDU_yazlog(0);
 768
 769                 return c;
 770             }
 771         }
 772         else
 773         {
 774             yaz_log (YLOG_LOG, "%sNEW %d %s",
 775                      m_session_str, parent->m_seqno, m_proxyTarget);
 776             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 777             c->m_next = parent->m_clientPool;
 778             if (c->m_next)
 779                 c->m_next->m_prev = &c->m_next;
 780             parent->m_clientPool = c;
 781             c->m_prev = &parent->m_clientPool;
 782         }
 783
 784         xfree (c->m_cookie);
 785         c->m_cookie = 0;
 786         if (cookie)
 787             c->m_cookie = xstrdup(cookie);
 788
 789         c->m_seqno = parent->m_seqno;
 790         c->m_init_flag = 0;
 791         c->m_last_resultCount = 0;
 792         c->m_last_ok = 0;
 793         c->m_cache.clear();
 794         c->m_sr_transform = 0;
 795         c->m_waiting = 0;
 796         c->m_resultSetStartPoint = 0;
 797         (parent->m_seqno)++;
 798         if (c->client(m_proxyTarget))
 799         {
 800             delete c;
 801             return 0;
 802         }
 803         c->m_target_idletime = m_target_idletime;
 804         c->timeout(30);
 805
 806         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 807             c->set_APDU_yazlog(1);
 808         else
 809             c->set_APDU_yazlog(0);
 810
 811         c->set_idAuthentication(apdu);
 812     }
 813     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 814     return c;
 815 }
 816
 817 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 818 {
 819     int i;
 820     for (i = 0; i<num; i++)
 821     {
 822         oident *ent;
 823         Z_DefaultDiagFormat *r;
 824         Z_DiagRec *p = pp[i];
 825         if (p->which != Z_DiagRec_defaultFormat)
 826         {
 827             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 828             return;
 829         }
 830         else
 831             r = p->u.defaultFormat;
 832         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 833             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 834             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 835         switch (r->which)
 836         {
 837         case Z_DefaultDiagFormat_v2Addinfo:
 838             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 839                     m_session_str,
 840                     *r->condition, diagbib1_str(*r->condition),
 841                     r->u.v2Addinfo);
 842             break;
 843         case Z_DefaultDiagFormat_v3Addinfo:
 844             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 845                     m_session_str,
 846                     *r->condition, diagbib1_str(*r->condition),
 847                     r->u.v3Addinfo);
 848             break;
 849         }
 850     }
 851 }
 852
 853 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 854 {
 855     if (!m_stylesheet_xsp || p->num_records <= 0)
 856     {
 857         return 0;  /* no XSLT to be done ... */
 858     }
 859
 860     m_stylesheet_offset = 0;
 861     m_stylesheet_nprl = p;
 862     m_stylesheet_apdu = apdu;
 863     m_timeout_mode = timeout_xsl;
 864
 865     timeout(0);
 866     return 1;
 867 }
 868
 869 void Yaz_Proxy::convert_xsl_delay()
 870 {
 871 #if HAVE_XSLT
 872     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 873     if (npr->which == Z_NamePlusRecord_databaseRecord)
 874     {
 875         Z_External *r = npr->u.databaseRecord;
 876         if (r->which == Z_External_octet)
 877         {
 878 #if 0
 879             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 880 #endif
 881             xmlDocPtr res, doc = xmlParseMemory(
 882                 (char*) r->u.octet_aligned->buf,
 883                 r->u.octet_aligned->len);
 884
 885
 886             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 887                     m_session_str, m_stylesheet_offset);
 888             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 889                                       doc, 0);
 890
 891             if (res)
 892             {
 893                 xmlChar *out_buf;
 894                 int out_len;
 895                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 896
 897                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 898                     u.databaseRecord =
 899                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 900                                  (char*) out_buf, out_len);
 901                 xmlFree(out_buf);
 902                 xmlFreeDoc(res);
 903             }
 904
 905             xmlFreeDoc(doc);
 906         }
 907     }
 908 #endif
 909     m_stylesheet_offset++;
 910     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 911     {
 912         m_timeout_mode = timeout_normal;
 913         m_stylesheet_nprl = 0;
 914 #if HAVE_XSLT
 915         if (m_stylesheet_xsp)
 916             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 917 #endif
 918         m_stylesheet_xsp = 0;
 919         timeout(m_client_idletime);
 920         send_PDU_convert(m_stylesheet_apdu);
 921     }
 922     else
 923         timeout(0);
 924 }
 925
 926 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 927 {
 928     if (m_frontend_type != VAL_NONE)
 929     {
 930         int i;
 931         for (i = 0; i < p->num_records; i++)
 932         {
 933             Z_NamePlusRecord *npr = p->records[i];
 934             if (npr->which == Z_NamePlusRecord_databaseRecord)
 935             {
 936                 Z_External *r = npr->u.databaseRecord;
 937                 if (r->which == Z_External_octet)
 938                 {
 939 #if HAVE_USEMARCON
 940                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 941                     {
 942                         if (!m_usemarcon->m_stage1)
 943                         {
 944                             m_usemarcon->m_stage1 = new CDetails();
 945                         }
 946                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 947                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 948                         int res = m_usemarcon->m_stage1->Start();
 949                         if (res == 0)
 950                         {
 951                             char *converted;
 952                             int convlen;
 953                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 954                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 955                             {
 956                                 if (!m_usemarcon->m_stage2)
 957                                 {
 958                                     m_usemarcon->m_stage2 = new CDetails();
 959                                 }
 960                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 961                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 962                                 res = m_usemarcon->m_stage2->Start();
 963                                 if (res == 0)
 964                                 {
 965                                     free(converted);
 966                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 967                                 }
 968                                 else
 969                                 {
 970                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 971                                 }
 972                             }
 973                             npr->u.databaseRecord =
 974                                 z_ext_record(odr_encode(),
 975                                              m_frontend_type,
 976                                              converted,
 977                                              strlen(converted));
 978                             free(converted);
 979                         }
 980                         else
 981                         {
 982                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 983                         }
 984                         continue;
 985                     }
 986 #endif
 987 /* HAVE_USEMARCON */
 988                     npr->u.databaseRecord =
 989                         z_ext_record(odr_encode(),
 990                                      m_frontend_type,
 991                                      (char*) r->u.octet_aligned->buf,
 992                                      r->u.octet_aligned->len);
 993                 }
 994             }
 995         }
 996     }
 997 }
 998
 999 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
1000                                         const char *backend_charset)
1001 {
1002     int sel =   m_charset_converter->get_client_charset_selected();
1003     const char *client_record_charset =
1004         m_charset_converter->get_client_query_charset();
1005     if (sel && backend_charset && client_record_charset &&
1006         strcmp(backend_charset, client_record_charset))
1007     {
1008         int i;
1009         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
1010                                         backend_charset);
1011         yaz_marc_t mt = yaz_marc_create();
1012         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
1013         yaz_marc_iconv(mt, cd);
1014         for (i = 0; i < p->num_records; i++)
1015         {
1016             Z_NamePlusRecord *npr = p->records[i];
1017             if (npr->which == Z_NamePlusRecord_databaseRecord)
1018             {
1019                 Z_External *r = npr->u.databaseRecord;
1020                 oident *ent = oid_getentbyoid(r->direct_reference);
1021                 if (!ent || ent->value == VAL_NONE)
1022                     continue;
1023
1024                 if (ent->value == VAL_SUTRS)
1025                 {
1026                     WRBUF w = wrbuf_alloc();
1027
1028                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
1029                                       r->u.octet_aligned->len);
1030                     npr->u.databaseRecord =
1031                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
1032                                      wrbuf_len(w));
1033                     wrbuf_free(w, 1);
1034                 }
1035                 else if (ent->value == VAL_TEXT_XML)
1036                 {
1037                     ;
1038                 }
1039                 else if (r->which == Z_External_octet)
1040                 {
1041                     int rlen;
1042                     char *result;
1043                     if (yaz_marc_decode_buf(mt,
1044                                             (char*) r->u.octet_aligned->buf,
1045                                             r->u.octet_aligned->len,
1046                                             &result, &rlen))
1047                     {
1048                         npr->u.databaseRecord =
1049                             z_ext_record(odr_encode(), ent->value, result, rlen);
1050                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1051                                 m_session_str);
1052                     }
1053                 }
1054             }
1055         }
1056         if (cd)
1057             yaz_iconv_close(cd);
1058         yaz_marc_destroy(mt);
1059     }
1060 }
1061
1062 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1063                                    const char *backend_charset)
1064 {
1065     int i;
1066     if (!backend_charset)
1067         backend_charset = "MARC-8";
1068     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1069     yaz_marc_t mt = yaz_marc_create();
1070     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1071     yaz_marc_iconv(mt, cd);
1072     for (i = 0; i < p->num_records; i++)
1073     {
1074         Z_NamePlusRecord *npr = p->records[i];
1075         if (npr->which == Z_NamePlusRecord_databaseRecord)
1076         {
1077             Z_External *r = npr->u.databaseRecord;
1078             if (r->which == Z_External_OPAC)
1079             {
1080                 WRBUF w = wrbuf_alloc();
1081
1082                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
1083                 npr->u.databaseRecord = z_ext_record(
1084                     odr_encode(), VAL_TEXT_XML,
1085                     wrbuf_buf(w), wrbuf_len(w)
1086                     );
1087                 wrbuf_free(w, 1);
1088             }
1089             else if (r->which == Z_External_octet)
1090             {
1091                 int rlen;
1092                 char *result;
1093                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1094                                         r->u.octet_aligned->len,
1095                                         &result, &rlen))
1096                 {
1097                     npr->u.databaseRecord =
1098                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
1099                 }
1100             }
1101         }
1102     }
1103     if (cd)
1104         yaz_iconv_close(cd);
1105     yaz_marc_destroy(mt);
1106 }
1107
1108 void Yaz_Proxy::logtime()
1109 {
1110 #if HAVE_GETTIMEOFDAY
1111     struct timeval *tv = (struct timeval*) m_time_tv;
1112     if (tv->tv_sec)
1113     {
1114         struct timeval tv1;
1115         gettimeofday(&tv1, 0);
1116         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1117             (tv1.tv_usec - tv->tv_usec);
1118         if (diff >= 0)
1119             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1120                     diff/1000000, (diff/1000)%1000);
1121     }
1122     tv->tv_sec = 0;
1123     tv->tv_usec = 0;
1124 #endif
1125 }
1126
1127 int Yaz_Proxy::send_http_response(int code)
1128 {
1129     ODR o = odr_encode();
1130     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1131     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1132     if (m_http_version)
1133         hres->version = odr_strdup(o, m_http_version);
1134     if (m_http_keepalive)
1135         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1136     else
1137         timeout(0);
1138
1139     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1140     {
1141         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1142                  gdu_name(gdu));
1143     }
1144     int len;
1145     int r = send_GDU(gdu, &len);
1146     m_bytes_sent += len;
1147     m_bw_stat.add_bytes(len);
1148     logtime();
1149
1150     recv_GDU_more(true);
1151
1152     return r;
1153 }
1154
1155 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */)
1156 {
1157     ODR o = odr_encode();
1158     const char *ctype = "text/xml";
1159     Z_GDU *gdu = z_get_HTTP_Response(o, http_code);
1160     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1161     if (m_http_version)
1162         hres->version = odr_strdup(o, m_http_version);
1163     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1164     if (m_http_keepalive)
1165         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1166     else
1167         timeout(0);
1168     if (http_code == 401)
1169         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\"");
1170
1171     static Z_SOAP_Handler soap_handlers[2] = {
1172 #if HAVE_XSLT
1173         {"http://www.loc.gov/zing/srw/", 0,
1174          (Z_SOAP_fun) yaz_srw_codec},
1175 #endif
1176         {0, 0, 0}
1177     };
1178
1179     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1180     soap_package->which = Z_SOAP_generic;
1181     soap_package->u.generic =
1182         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1183     soap_package->u.generic->no = 0;
1184     soap_package->u.generic->ns = soap_handlers[0].ns;
1185     soap_package->u.generic->p = (void *) srw_pdu;
1186     soap_package->ns = m_soap_ns;
1187     z_soap_codec_enc_xsl(o, &soap_package,
1188                          &hres->content_buf, &hres->content_len,
1189                          soap_handlers, 0, m_s2z_stylesheet);
1190     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1191     {
1192         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1193                  gdu_name(gdu));
1194     }
1195     int len;
1196     int r = send_GDU(gdu, &len);
1197     m_bytes_sent += len;
1198     m_bw_stat.add_bytes(len);
1199     logtime();
1200
1201     recv_GDU_more(true);
1202
1203     return r;
1204 }
1205
1206 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1207 {
1208     ODR o = odr_encode();
1209     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1210         odr_malloc(o, sizeof(*diagnostic));
1211     int num_diagnostic = 1;
1212     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1213     return send_srw_search_response(diagnostic, num_diagnostic, srw_error == 3 ? 401 : 200);
1214 }
1215
1216 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1217                              Z_DefaultDiagFormat *ddf)
1218 {
1219     int bib1_code = *ddf->condition;
1220     if (bib1_code == 109)
1221         return 404;
1222     srw_res->num_diagnostics = 1;
1223     srw_res->diagnostics = (Z_SRW_diagnostic *)
1224         odr_malloc(o, sizeof(*srw_res->diagnostics));
1225     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1226                           yaz_diag_bib1_to_srw(*ddf->condition),
1227                           ddf->u.v2Addinfo);
1228     return 0;
1229 }
1230
1231 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1232 {
1233     ODR o = odr_encode();
1234     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1235     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1236
1237     srw_res->numberOfRecords = odr_intdup (o, hits);
1238     if (records && records->which == Z_Records_DBOSD)
1239     {
1240         srw_res->num_records =
1241             records->u.databaseOrSurDiagnostics->num_records;
1242         int i;
1243         srw_res->records = (Z_SRW_record *)
1244             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1245         for (i = 0; i < srw_res->num_records; i++)
1246         {
1247             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1248             if (npr->which != Z_NamePlusRecord_databaseRecord)
1249             {
1250                 srw_res->records[i].recordSchema = "diagnostic";
1251                 srw_res->records[i].recordPacking = m_s2z_packing;
1252                 srw_res->records[i].recordData_buf = "67";
1253                 srw_res->records[i].recordData_len = 2;
1254                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1255                 continue;
1256             }
1257             Z_External *r = npr->u.databaseRecord;
1258             oident *ent = oid_getentbyoid(r->direct_reference);
1259             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1260             {
1261                 srw_res->records[i].recordSchema = m_schema;
1262                 srw_res->records[i].recordPacking = m_s2z_packing;
1263                 srw_res->records[i].recordData_buf = (char*)
1264                     r->u.octet_aligned->buf;
1265                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1266                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1267             }
1268             else
1269             {
1270                 srw_res->records[i].recordSchema = "diagnostic";
1271                 srw_res->records[i].recordPacking = m_s2z_packing;
1272                 srw_res->records[i].recordData_buf = "67";
1273                 srw_res->records[i].recordData_len = 2;
1274                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1275             }
1276         }
1277     }
1278     if (records && records->which == Z_Records_NSD)
1279     {
1280         int http_code;
1281         http_code = z_to_srw_diag(odr_encode(), srw_res,
1282                                    records->u.nonSurrogateDiagnostic);
1283         if (http_code)
1284             return send_http_response(http_code);
1285     }
1286     return send_srw_response(srw_pdu);
1287
1288 }
1289
1290 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1291                                         int num_diagnostics, int http_code /* = 200 */)
1292 {
1293     ODR o = odr_encode();
1294     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1295     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1296
1297     srw_res->num_diagnostics = num_diagnostics;
1298     srw_res->diagnostics = diagnostics;
1299     return send_srw_response(srw_pdu, http_code);
1300 }
1301
1302 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1303                                         int num_diagnostics)
1304 {
1305     Yaz_ProxyConfig *cfg = check_reconfigure();
1306     if (cfg)
1307     {
1308         int len;
1309         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1310                                        m_s2z_database, &len);
1311         if (b)
1312         {
1313             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1314             Z_SRW_explainResponse *er = res->u.explain_response;
1315
1316             er->record.recordData_buf = b;
1317             er->record.recordData_len = len;
1318             er->record.recordPacking = m_s2z_packing;
1319             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1320
1321             er->diagnostics = diagnostics;
1322             er->num_diagnostics = num_diagnostics;
1323             return send_srw_response(res);
1324         }
1325     }
1326     return send_http_response(404);
1327 }
1328
1329 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1330 {
1331     if (m_http_version)
1332     {
1333         if (apdu->which == Z_APDU_initResponse)
1334         {
1335             Z_InitResponse *res = apdu->u.initResponse;
1336             if (*res->result == 0)
1337             {
1338                 send_to_srw_client_error(3, 0);
1339             }
1340             else if (!m_s2z_search_apdu)
1341             {
1342                 send_srw_explain_response(0, 0);
1343             }
1344             else
1345             {
1346                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1347             }
1348         }
1349         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1350         {
1351             m_s2z_search_apdu = 0;
1352             Z_SearchResponse *res = apdu->u.searchResponse;
1353             m_s2z_hit_count = *res->resultCount;
1354             if (res->records && res->records->which == Z_Records_NSD)
1355             {
1356                 send_to_srw_client_ok(0, res->records, 1);
1357             }
1358             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1359             {
1360                 // adjust
1361                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1362
1363                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1364                 {
1365                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1366                         > m_s2z_hit_count)
1367                         *pr->numberOfRecordsRequested =
1368                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1369                 }
1370                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1371             }
1372             else
1373             {
1374                 m_s2z_present_apdu = 0;
1375                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1376             }
1377         }
1378         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1379         {
1380             int start =
1381                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1382
1383             m_s2z_present_apdu = 0;
1384             Z_PresentResponse *res = apdu->u.presentResponse;
1385             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1386         }
1387     }
1388     else
1389     {
1390         int len = 0;
1391         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1392             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1393                      apdu_name(apdu));
1394         int r = send_Z_PDU(apdu, &len);
1395         m_bytes_sent += len;
1396         m_bw_stat.add_bytes(len);
1397         logtime();
1398         return r;
1399     }
1400     return 0;
1401 }
1402
1403 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1404 {
1405     int kill_session = 0;
1406     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1407
1408     if (new_id)
1409         *new_id = m_referenceId;
1410
1411     if (apdu->which == Z_APDU_searchResponse)
1412     {
1413         Z_SearchResponse *sr = apdu->u.searchResponse;
1414         Z_Records *p = sr->records;
1415         if (p && p->which == Z_Records_NSD)
1416         {
1417             Z_DiagRec dr, *dr_p = &dr;
1418             dr.which = Z_DiagRec_defaultFormat;
1419             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1420
1421             *sr->searchStatus = 0;
1422             display_diagrecs(&dr_p, 1);
1423         }
1424         else
1425         {
1426             if (p && p->which == Z_Records_DBOSD)
1427             {
1428                 if (m_backend_type
1429 #if HAVE_USEMARCON
1430                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1431 #endif
1432                     )
1433                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1434                 if (m_marcxml_mode == marcxml)
1435                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1436                                        m_backend_charset);
1437                 else
1438                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1439                                             m_backend_charset);
1440                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1441                     return 0;
1442
1443             }
1444             if (sr->resultCount)
1445             {
1446                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1447                         *sr->resultCount);
1448                 if (*sr->resultCount < 0)
1449                 {
1450                     m_flag_invalid_session = 1;
1451                     kill_session = 1;
1452
1453                     *sr->searchStatus = 0;
1454                     sr->records =
1455                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1456                     *sr->resultCount = 0;
1457                 }
1458             }
1459         }
1460     }
1461     else if (apdu->which == Z_APDU_presentResponse)
1462     {
1463         Z_PresentResponse *sr = apdu->u.presentResponse;
1464         Z_Records *p = sr->records;
1465         if (p && p->which == Z_Records_NSD)
1466         {
1467             Z_DiagRec dr, *dr_p = &dr;
1468             dr.which = Z_DiagRec_defaultFormat;
1469             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1470             if (*sr->presentStatus == Z_PresentStatus_success)
1471                 *sr->presentStatus = Z_PresentStatus_failure;
1472             display_diagrecs(&dr_p, 1);
1473         }
1474         if (p && p->which == Z_Records_DBOSD)
1475         {
1476             if (m_backend_type
1477 #if HAVE_USEMARCON
1478                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1479 #endif
1480                 )
1481                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1482             if (m_marcxml_mode == marcxml)
1483                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1484                                    m_backend_charset);
1485             else
1486                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1487                                         m_backend_charset);
1488             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1489                 return 0;
1490         }
1491     }
1492     else if (apdu->which == Z_APDU_initResponse)
1493     {
1494         //Get and check negotiation record
1495         //from init response.
1496         handle_charset_lang_negotiation(apdu);
1497
1498         if (m_initRequest_options)
1499         {
1500             Z_Options *nopt =
1501                 (Odr_bitmask *)odr_malloc(odr_encode(),
1502                                           sizeof(Odr_bitmask));
1503             ODR_MASK_ZERO(nopt);
1504
1505             int i;
1506             for (i = 0; i<24; i++)
1507                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1508                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1509                     ODR_MASK_SET(nopt, i);
1510             apdu->u.initResponse->options = nopt;
1511         }
1512         if (m_initRequest_version)
1513         {
1514             Z_ProtocolVersion *nopt =
1515                 (Odr_bitmask *)odr_malloc(odr_encode(),
1516                                           sizeof(Odr_bitmask));
1517             ODR_MASK_ZERO(nopt);
1518
1519             int i;
1520             for (i = 0; i<8; i++)
1521                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1522                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1523                     ODR_MASK_SET(nopt, i);
1524             apdu->u.initResponse->protocolVersion = nopt;
1525         }
1526         apdu->u.initResponse->preferredMessageSize =
1527             odr_intdup(odr_encode(),
1528                        m_client->m_initResponse_preferredMessageSize >
1529                        m_initRequest_preferredMessageSize ?
1530                        m_initRequest_preferredMessageSize :
1531                        m_client->m_initResponse_preferredMessageSize);
1532         apdu->u.initResponse->maximumRecordSize =
1533             odr_intdup(odr_encode(),
1534                        m_client->m_initResponse_maximumRecordSize >
1535                        m_initRequest_maximumRecordSize ?
1536                        m_initRequest_maximumRecordSize :
1537                        m_client->m_initResponse_maximumRecordSize);
1538     }
1539
1540     int r = send_PDU_convert(apdu);
1541     if (r)
1542         return r;
1543     if (kill_session)
1544     {
1545         delete m_client;
1546         m_client = 0;
1547         m_parent->pre_init();
1548     }
1549     return r;
1550 }
1551
1552 void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu)
1553 {
1554     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1555
1556     odr_reset(m_idAuthentication_odr);
1557     z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0);
1558     m_idAuthentication_ber_buf =
1559         odr_getbuf(m_idAuthentication_odr,
1560                    &m_idAuthentication_ber_size, 0);
1561 }
1562
1563 bool Yaz_ProxyClient::compare_charset(Z_APDU *apdu)
1564 {
1565     return true;
1566 }
1567
1568 bool Yaz_ProxyClient::compare_idAuthentication(Z_APDU *apdu)
1569 {
1570     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1571     ODR odr = odr_createmem(ODR_ENCODE);
1572
1573     z_IdAuthentication(odr, &t, 1, 0);
1574     int sz;
1575     char *buf = odr_getbuf(odr, &sz, 0);
1576     if (buf && m_idAuthentication_ber_buf
1577         && sz == m_idAuthentication_ber_size
1578         && !memcmp(m_idAuthentication_ber_buf, buf, sz))
1579     {
1580         odr_destroy(odr);
1581         return true;
1582     }
1583     odr_destroy(odr);
1584     if (!buf && !m_idAuthentication_ber_buf)
1585         return true;
1586     return false;
1587 }
1588
1589 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1590 {
1591     int len = 0;
1592     const char *apdu_name_tmp = apdu_name(apdu);
1593     int r = send_Z_PDU(apdu, &len);
1594     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1595         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1596                  get_session_str(),
1597                  apdu_name_tmp, get_hostname(), len);
1598     m_bytes_sent += len;
1599     return r;
1600 }
1601
1602 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1603 {
1604     if (apdu->which == Z_APDU_presentRequest)
1605     {
1606         Z_PresentRequest *pr = apdu->u.presentRequest;
1607         int toget = *pr->numberOfRecordsRequested;
1608         int start = *pr->resultSetStartPoint;
1609
1610         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1611                 pr->resultSetId, start, toget);
1612
1613         if (*m_parent->m_optimize == '0')
1614             return apdu;
1615
1616         if (!m_client->m_last_resultSetId)
1617         {
1618             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1619             new_apdu->u.presentResponse->records =
1620                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1621                                                pr->resultSetId);
1622             send_to_client(new_apdu);
1623             return 0;
1624         }
1625         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1626         {
1627             if (start+toget-1 > m_client->m_last_resultCount)
1628             {
1629                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1630                 new_apdu->u.presentResponse->records =
1631                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1632                 send_to_client(new_apdu);
1633                 return 0;
1634             }
1635             Z_NamePlusRecordList *npr;
1636 #if 0
1637             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1638                     m_session_str, start, toget, yaz_z3950oid_to_str(
1639                         pr->preferredRecordSyntax, &oclass));
1640 #endif
1641             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1642                                           pr->preferredRecordSyntax,
1643                                           pr->recordComposition))
1644             {
1645                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1646                          m_session_str);
1647                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1648                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1649
1650                 new_apdu->u.presentResponse->numberOfRecordsReturned
1651                     = odr_intdup(odr_encode(), toget);
1652
1653                 new_apdu->u.presentResponse->records = (Z_Records*)
1654                     odr_malloc(odr_encode(), sizeof(Z_Records));
1655                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1656                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1657                 new_apdu->u.presentResponse->nextResultSetPosition =
1658                     odr_intdup(odr_encode(), start+toget);
1659
1660                 send_to_client(new_apdu);
1661                 return 0;
1662             }
1663         }
1664     }
1665
1666     if (apdu->which != Z_APDU_searchRequest)
1667         return apdu;
1668     Z_SearchRequest *sr = apdu->u.searchRequest;
1669     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1670     Yaz_Z_Databases this_databases;
1671
1672     this_databases.set(sr->num_databaseNames, (const char **)
1673                        sr->databaseNames);
1674
1675     this_query->set_Z_Query(sr->query);
1676
1677     char query_str[120];
1678     this_query->print(query_str, sizeof(query_str)-1);
1679     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1680
1681     if (*m_parent->m_optimize != '0' &&
1682         m_client->m_last_ok && m_client->m_last_query &&
1683         m_client->m_last_query->match(this_query) &&
1684         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1685         m_client->m_last_databases.match(this_databases))
1686     {
1687         delete this_query;
1688         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1689             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1690         {
1691             Z_NamePlusRecordList *npr;
1692             int toget = *sr->mediumSetPresentNumber;
1693             Z_RecordComposition *comp = 0;
1694
1695             if (toget > m_client->m_last_resultCount)
1696                 toget = m_client->m_last_resultCount;
1697
1698             if (sr->mediumSetElementSetNames)
1699             {
1700                 comp = (Z_RecordComposition *)
1701                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1702                 comp->which = Z_RecordComp_simple;
1703                 comp->u.simple = sr->mediumSetElementSetNames;
1704             }
1705
1706             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1707                                           sr->preferredRecordSyntax, comp))
1708             {
1709                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1710                          m_session_str);
1711                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1712                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1713                 new_apdu->u.searchResponse->resultCount =
1714                     &m_client->m_last_resultCount;
1715
1716                 new_apdu->u.searchResponse->numberOfRecordsReturned
1717                     = odr_intdup(odr_encode(), toget);
1718
1719                 new_apdu->u.searchResponse->presentStatus =
1720                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1721                 new_apdu->u.searchResponse->records = (Z_Records*)
1722                     odr_malloc(odr_encode(), sizeof(Z_Records));
1723                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1724                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1725                 new_apdu->u.searchResponse->nextResultSetPosition =
1726                     odr_intdup(odr_encode(), toget+1);
1727                 send_to_client(new_apdu);
1728                 return 0;
1729             }
1730             else
1731             {
1732                 // medium Set
1733                 // send present request (medium size)
1734                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1735                          m_session_str);
1736
1737                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1738                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1739                 pr->referenceId = sr->referenceId;
1740                 pr->resultSetId = sr->resultSetName;
1741                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1742                 *pr->numberOfRecordsRequested = toget;
1743                 pr->recordComposition = comp;
1744                 m_client->m_sr_transform = 1;
1745                 return new_apdu;
1746             }
1747         }
1748         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1749             m_client->m_last_resultCount <= 0)
1750         {
1751             // large set. Return pseudo-search response immediately
1752             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1753                      m_session_str);
1754             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1755             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1756             new_apdu->u.searchResponse->resultCount =
1757                 &m_client->m_last_resultCount;
1758             send_to_client(new_apdu);
1759             return 0;
1760         }
1761         else
1762         {
1763             Z_NamePlusRecordList *npr;
1764             int toget = m_client->m_last_resultCount;
1765             Z_RecordComposition *comp = 0;
1766             // small set
1767             // send a present request (small set)
1768
1769             if (sr->smallSetElementSetNames)
1770             {
1771                 comp = (Z_RecordComposition *)
1772                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1773                 comp->which = Z_RecordComp_simple;
1774                 comp->u.simple = sr->smallSetElementSetNames;
1775             }
1776
1777             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1778                                           sr->preferredRecordSyntax, comp))
1779             {
1780                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1781                          m_session_str);
1782                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1783                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1784                 new_apdu->u.searchResponse->resultCount =
1785                     &m_client->m_last_resultCount;
1786
1787                 new_apdu->u.searchResponse->numberOfRecordsReturned
1788                     = odr_intdup(odr_encode(), toget);
1789
1790                 new_apdu->u.searchResponse->presentStatus =
1791                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1792                 new_apdu->u.searchResponse->records = (Z_Records*)
1793                     odr_malloc(odr_encode(), sizeof(Z_Records));
1794                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1795                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1796                 new_apdu->u.searchResponse->nextResultSetPosition =
1797                     odr_intdup(odr_encode(), toget+1);
1798                 send_to_client(new_apdu);
1799                 return 0;
1800             }
1801             else
1802             {
1803                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1804                          m_session_str);
1805                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1806                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1807                 pr->referenceId = sr->referenceId;
1808                 pr->resultSetId = sr->resultSetName;
1809                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1810                 *pr->numberOfRecordsRequested = toget;
1811                 pr->recordComposition = comp;
1812                 m_client->m_sr_transform = 1;
1813                 return new_apdu;
1814             }
1815         }
1816     }
1817     else  // query doesn't match
1818     {
1819         delete m_client->m_last_query;
1820         m_client->m_last_query = this_query;
1821         m_client->m_last_ok = 0;
1822         m_client->m_cache.clear();
1823         m_client->m_resultSetStartPoint = 0;
1824
1825         xfree (m_client->m_last_resultSetId);
1826         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1827
1828         m_client->m_last_databases.set(sr->num_databaseNames,
1829                                        (const char **) sr->databaseNames);
1830     }
1831     return apdu;
1832 }
1833
1834
1835 void Yaz_Proxy::inc_request_no()
1836 {
1837     char *cp = m_session_str + strlen(m_session_str)-1;
1838     if (*cp == ' ')
1839         cp--;
1840     while (*cp && *cp != ' ')
1841         cp--;
1842     if (*cp)
1843         sprintf(cp+1, "%d ", m_request_no);
1844 }
1845
1846 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1847 {
1848     inc_request_no();
1849
1850     m_bytes_recv += len;
1851
1852     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1853         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1854                  m_session_str, gdu_name(apdu), len);
1855
1856 #if HAVE_GETTIMEOFDAY
1857     gettimeofday((struct timeval *) m_time_tv, 0);
1858 #endif
1859     m_bw_stat.add_bytes(len);
1860     m_pdu_stat.add_bytes(1);
1861
1862     GDU *gdu = new GDU(apdu);
1863     m_in_queue.enqueue(gdu);
1864
1865     recv_GDU_more(false);
1866 }
1867
1868 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
1869 {
1870     int reduce = m_initial_reduce; // initial reduce from connect phase..
1871     m_initial_reduce = 0;  // reset it..
1872
1873     int bw_total = m_bw_stat.get_total();
1874     int pdu_total = m_pdu_stat.get_total();
1875     int search_total = m_search_stat.get_total();
1876
1877     assert(m_timeout_mode == timeout_busy);
1878     assert(m_timeout_gdu == 0);
1879
1880     if (m_search_max)
1881         reduce += search_total / m_search_max;
1882     if (m_bw_max)
1883         reduce += (bw_total/m_bw_max);
1884     if (m_pdu_max)
1885     {
1886         if (pdu_total > m_pdu_max)
1887         {
1888             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1889             reduce = (reduce > nreduce) ? reduce : nreduce;
1890         }
1891     }
1892     m_http_version = 0;
1893
1894 #if 0
1895     /* uncomment to force a big reduce */
1896     m_timeout_mode = timeout_reduce;
1897     m_timeout_gdu = gdu;
1898     timeout(3);       // call us reduce seconds later
1899     return;
1900 #endif
1901     if (reduce)
1902     {
1903         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d search=%d limit-bw=%d limit-pdu=%d limit-search=%d",
1904                 m_session_str, reduce, bw_total, pdu_total, search_total,
1905                 m_bw_max, m_pdu_max, m_search_max);
1906
1907         m_timeout_mode = timeout_reduce;
1908         m_timeout_gdu = gdu;
1909         timeout(reduce);       // call us reduce seconds later
1910     }
1911     else
1912         recv_GDU_normal(gdu);
1913 }
1914
1915 void Yaz_Proxy::recv_GDU_more(bool normal)
1916 {
1917     GDU *g;
1918     if (normal && m_timeout_mode == timeout_busy)
1919         m_timeout_mode = timeout_normal;
1920     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
1921     {
1922         m_timeout_mode = timeout_busy;
1923         recv_GDU_reduce(g);
1924     }
1925 }
1926
1927 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
1928 {
1929     Z_GDU *apdu = 0;
1930     gdu->move_away_gdu(odr_decode(), &apdu);
1931     delete gdu;
1932
1933     if (apdu->which == Z_GDU_Z3950)
1934         handle_incoming_Z_PDU(apdu->u.z3950);
1935     else if (apdu->which == Z_GDU_HTTP_Request)
1936         handle_incoming_HTTP(apdu->u.HTTP_Request);
1937 }
1938
1939 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1940 {
1941     if (m_max_record_retrieve)
1942     {
1943         if (apdu->which == Z_APDU_presentRequest)
1944         {
1945             Z_PresentRequest *pr = apdu->u.presentRequest;
1946             if (pr->numberOfRecordsRequested &&
1947                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1948                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1949         }
1950     }
1951 }
1952
1953 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1954 {
1955     if (apdu->which == Z_APDU_initRequest)
1956     {
1957         if (m_initRequest_options &&
1958             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1959             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1960         {
1961             // There is no negotiation proposal from
1962             // client's side. OK. The proxy negotiation
1963             // in use, only.
1964             Z_InitRequest *initRequest = apdu->u.initRequest;
1965             Z_OtherInformation **otherInfo;
1966             Z_OtherInformationUnit *oi;
1967             get_otherInfoAPDU(apdu, &otherInfo);
1968             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1969             if (oi)
1970             {
1971                 ODR_MASK_SET(initRequest->options,
1972                     Z_Options_negotiationModel);
1973                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1974                 oi->information.externallyDefinedInfo =
1975                 yaz_set_proposal_charneg(odr_encode(),
1976                     (const char**)&m_proxy_negotiation_charset,
1977                     m_proxy_negotiation_charset ? 1:0,
1978                     (const char**)&m_proxy_negotiation_lang,
1979                     m_proxy_negotiation_lang ? 1:0,
1980                     1);
1981             }
1982         }
1983         else if (m_initRequest_options &&
1984                  ODR_MASK_GET(m_initRequest_options,
1985                               Z_Options_negotiationModel) &&
1986                  m_charset_converter->get_target_query_charset())
1987         {
1988             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1989                     m_session_str,
1990                     m_charset_converter->get_target_query_charset());
1991             Z_InitRequest *initRequest = apdu->u.initRequest;
1992             Z_CharSetandLanguageNegotiation *negotiation =
1993                 yaz_get_charneg_record (initRequest->otherInfo);
1994             if (negotiation &&
1995                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1996             {
1997                 NMEM nmem = nmem_create();
1998                 char **charsets = 0;
1999                 int num_charsets = 0;
2000                 char **langs = 0;
2001                 int num_langs = 0;
2002                 int selected = 0;
2003                 yaz_get_proposal_charneg (nmem, negotiation,
2004                                           &charsets, &num_charsets,
2005                                           &langs, &num_langs, &selected);
2006                 int i;
2007                 for (i = 0; i<num_charsets; i++)
2008                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
2009                             charsets[i]);
2010                 for (i = 0; i<num_langs; i++)
2011                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
2012                             langs[i]);
2013
2014                 const char *t_charset =
2015                     m_charset_converter->get_target_query_charset();
2016                 // sweep through charsets and pick the first supported
2017                 // conversion
2018                 for (i = 0; i<num_charsets; i++)
2019                 {
2020                     const char *c_charset = charsets[i];
2021                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
2022                         break;
2023                 }
2024                 if (i != num_charsets)
2025                 {
2026                     // got one .. set up ODR for reverse direction
2027                     const char *c_charset = charsets[i];
2028                     odr_set_charset(odr_encode(), c_charset, t_charset);
2029                     m_charset_converter->set_client_query_charset(c_charset);
2030                     m_charset_converter->set_client_charset_selected(selected);
2031                 }
2032                 nmem_destroy(nmem);
2033                 ODR_MASK_CLEAR(m_initRequest_options,
2034                                Z_Options_negotiationModel);
2035                 yaz_del_charneg_record(&initRequest->otherInfo);
2036             }
2037             else
2038             {
2039                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
2040                         m_session_str);
2041             }
2042         }
2043         else if (m_charset_converter->get_target_query_charset() &&
2044             m_proxy_negotiation_default_charset)
2045         {
2046             m_charset_converter->
2047                 set_client_query_charset(m_proxy_negotiation_default_charset);
2048         }
2049     }
2050     else if (apdu->which == Z_APDU_initResponse)
2051     {
2052         Z_InitResponse *initResponse = apdu->u.initResponse;
2053         Z_OtherInformation **otherInfo;
2054         get_otherInfoAPDU(apdu, &otherInfo);
2055
2056         Z_CharSetandLanguageNegotiation *charneg = 0;
2057
2058         if (otherInfo && *otherInfo &&
2059             ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel)
2060             && (charneg = yaz_get_charneg_record(*otherInfo)))
2061         {
2062             char *charset = 0;
2063             char *lang = 0;
2064             int selected = 0;
2065
2066             yaz_get_response_charneg(m_referenceId_mem, charneg,
2067                 &charset, &lang, &selected);
2068
2069             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
2070                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
2071
2072             if (m_initRequest_options &&
2073                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
2074             {
2075                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
2076                     m_session_str);
2077             }
2078             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2079             {
2080                 // negotiation-charset, negotiation-lang
2081                 // elements of config file in use.
2082
2083                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2084                     m_session_str);
2085
2086                 // clear negotiation option.
2087                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2088
2089                 // Delete negotiation (charneg-3) entry.
2090                 yaz_del_charneg_record(otherInfo);
2091             }
2092         }
2093         else
2094         {
2095             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2096             {
2097                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2098                         m_session_str);
2099             }
2100             else if (m_charset_converter->get_client_query_charset())
2101             {
2102                 Z_OtherInformation **otherInfo;
2103                 Z_OtherInformationUnit *oi;
2104                 get_otherInfoAPDU(apdu, &otherInfo);
2105                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2106                 if (oi)
2107                 {
2108                     ODR_MASK_SET(initResponse->options,
2109                                  Z_Options_negotiationModel);
2110                     if (m_initRequest_options)
2111                         ODR_MASK_SET(m_initRequest_options,
2112                                      Z_Options_negotiationModel);
2113
2114                     oi->which = Z_OtherInfo_externallyDefinedInfo;
2115                     oi->information.externallyDefinedInfo =
2116                         yaz_set_response_charneg(
2117                             odr_encode(),
2118                             m_charset_converter->get_client_query_charset(),
2119                             0 /* no lang */,
2120                             m_charset_converter->get_client_charset_selected());
2121                 }
2122             }
2123         }
2124     }
2125 }
2126
2127 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2128                                                      int error,
2129                                                      const char *addinfo)
2130 {
2131     Z_Records *rec = (Z_Records *)
2132         odr_malloc (odr, sizeof(*rec));
2133     int *err = (int *)
2134         odr_malloc (odr, sizeof(*err));
2135     Z_DiagRec *drec = (Z_DiagRec *)
2136         odr_malloc (odr, sizeof(*drec));
2137     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2138         odr_malloc (odr, sizeof(*dr));
2139     *err = error;
2140     rec->which = Z_Records_NSD;
2141     rec->u.nonSurrogateDiagnostic = dr;
2142     dr->diagnosticSetId =
2143         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
2144     dr->condition = err;
2145     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2146     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2147     return rec;
2148 }
2149
2150 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2151 {
2152     if (apdu->which == Z_APDU_searchRequest &&
2153         apdu->u.searchRequest->query &&
2154         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2155         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2156     {
2157         Z_RPNQuery *rpnquery = 0;
2158         Z_SearchRequest *sr = apdu->u.searchRequest;
2159         char *addinfo = 0;
2160
2161         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2162                 sr->query->u.type_104->u.cql);
2163
2164         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2165                                           &rpnquery, odr_encode(),
2166                                           &addinfo);
2167         if (r == -3)
2168             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2169         else if (r)
2170         {
2171             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2172             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2173
2174             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2175             new_apdu->u.searchResponse->records =
2176                 create_nonSurrogateDiagnostics(odr_encode(),
2177                                                yaz_diag_srw_to_bib1(r),
2178                                                addinfo);
2179             *new_apdu->u.searchResponse->searchStatus = 0;
2180
2181             send_to_client(new_apdu);
2182
2183             return 0;
2184         }
2185         else
2186         {
2187             sr->query->which = Z_Query_type_1;
2188             sr->query->u.type_1 = rpnquery;
2189         }
2190         return apdu;
2191     }
2192     return apdu;
2193 }
2194
2195 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2196 {
2197     if (apdu->which == Z_APDU_searchRequest &&
2198         apdu->u.searchRequest->query)
2199     {
2200         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2201             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2202         {
2203             if (m_http_version)
2204                 m_charset_converter->set_client_query_charset("UTF-8");
2205             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2206             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2207         }
2208     }
2209     return apdu;
2210 }
2211
2212
2213 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2214 {
2215     if (apdu->which == Z_APDU_searchRequest)
2216     {
2217         Z_SearchRequest *sr = apdu->u.searchRequest;
2218         int err = 0;
2219         char *addinfo = 0;
2220
2221         Yaz_ProxyConfig *cfg = check_reconfigure();
2222         if (cfg)
2223             err = cfg->check_query(odr_encode(), m_default_target,
2224                                    sr->query, &addinfo);
2225         if (err)
2226         {
2227             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2228
2229             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2230             new_apdu->u.searchResponse->records =
2231                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2232             *new_apdu->u.searchResponse->searchStatus = 0;
2233
2234             send_to_client(new_apdu);
2235
2236             return 0;
2237         }
2238     }
2239     return apdu;
2240 }
2241
2242 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2243 {
2244     if (apdu->which != Z_APDU_initRequest)
2245         return 1;  // pass if no init request
2246     Z_InitRequest *req = apdu->u.initRequest;
2247
2248     Yaz_ProxyConfig *cfg = check_reconfigure();
2249     if (!cfg)
2250         return 1;  // pass if no config
2251
2252     int ret;
2253     if (req->idAuthentication == 0)
2254     {
2255         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2256                                          m_peername);
2257     }
2258     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2259     {
2260         ret = cfg->client_authentication(
2261             m_default_target,
2262             req->idAuthentication->u.idPass->userId,
2263             req->idAuthentication->u.idPass->groupId,
2264             req->idAuthentication->u.idPass->password,
2265             m_peername);
2266     }
2267     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2268     {
2269         char user[64], pass[64];
2270         *user = '\0';
2271         *pass = '\0';
2272         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2273         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2274                                          m_peername);
2275     }
2276     else
2277         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2278                                          m_peername);
2279     return ret;
2280 }
2281
2282 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2283 {
2284     m_marcxml_mode = none;
2285     if (apdu->which == Z_APDU_searchRequest)
2286     {
2287         Z_SearchRequest *sr = apdu->u.searchRequest;
2288         int err = 0;
2289         char *addinfo = 0;
2290         Yaz_ProxyConfig *cfg = check_reconfigure();
2291
2292         Z_RecordComposition rc_temp, *rc = 0;
2293         if (sr->smallSetElementSetNames)
2294         {
2295             rc_temp.which = Z_RecordComp_simple;
2296             rc_temp.u.simple = sr->smallSetElementSetNames;
2297             rc = &rc_temp;
2298         }
2299
2300         if (sr->preferredRecordSyntax)
2301         {
2302             struct oident *ent;
2303             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2304             m_frontend_type = ent->value;
2305         }
2306         else
2307             m_frontend_type = VAL_NONE;
2308
2309         char *stylesheet_name = 0;
2310         if (cfg)
2311             err = cfg->check_syntax(odr_encode(),
2312                                     m_default_target,
2313                                     sr->preferredRecordSyntax, rc,
2314                                     &addinfo, &stylesheet_name, &m_schema,
2315                                     &m_backend_type, &m_backend_charset,
2316                                     &m_usemarcon_ini_stage1,
2317                                     &m_usemarcon_ini_stage2);
2318         if (stylesheet_name)
2319         {
2320             m_parent->low_socket_close();
2321
2322 #if HAVE_XSLT
2323             if (m_stylesheet_xsp)
2324                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2325             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2326                                                        stylesheet_name);
2327 #endif
2328             m_stylesheet_offset = 0;
2329             xfree(stylesheet_name);
2330
2331             m_parent->low_socket_open();
2332         }
2333         if (err == -1)
2334         {
2335             sr->smallSetElementSetNames = 0;
2336             sr->mediumSetElementSetNames = 0;
2337             m_marcxml_mode = marcxml;
2338             if (m_backend_type)
2339             {
2340
2341                 sr->preferredRecordSyntax =
2342                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2343                                         m_backend_type);
2344             }
2345             else
2346                 sr->preferredRecordSyntax =
2347                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2348                                            VAL_USMARC);
2349         }
2350         else if (err)
2351         {
2352             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2353
2354             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2355             new_apdu->u.searchResponse->records =
2356                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2357             *new_apdu->u.searchResponse->searchStatus = 0;
2358
2359             send_to_client(new_apdu);
2360
2361             return 0;
2362         }
2363         else if (m_backend_type)
2364         {
2365             sr->preferredRecordSyntax =
2366                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2367         }
2368     }
2369     else if (apdu->which == Z_APDU_presentRequest)
2370     {
2371         Z_PresentRequest *pr = apdu->u.presentRequest;
2372         int err = 0;
2373         char *addinfo = 0;
2374         Yaz_ProxyConfig *cfg = check_reconfigure();
2375
2376         if (pr->preferredRecordSyntax)
2377         {
2378             struct oident *ent;
2379             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2380             m_frontend_type = ent->value;
2381         }
2382         else
2383             m_frontend_type = VAL_NONE;
2384
2385         char *stylesheet_name = 0;
2386         if (cfg)
2387             err = cfg->check_syntax(odr_encode(), m_default_target,
2388                                     pr->preferredRecordSyntax,
2389                                     pr->recordComposition,
2390                                     &addinfo, &stylesheet_name, &m_schema,
2391                                     &m_backend_type, &m_backend_charset,
2392                                     &m_usemarcon_ini_stage1,
2393                                     &m_usemarcon_ini_stage2
2394                                     );
2395         if (stylesheet_name)
2396         {
2397             m_parent->low_socket_close();
2398
2399 #if HAVE_XSLT
2400             if (m_stylesheet_xsp)
2401                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2402             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2403                                                        stylesheet_name);
2404 #endif
2405             m_stylesheet_offset = 0;
2406             xfree(stylesheet_name);
2407
2408             m_parent->low_socket_open();
2409         }
2410         if (err == -1)
2411         {
2412             pr->recordComposition = 0;
2413             m_marcxml_mode = marcxml;
2414             if (m_backend_type)
2415             {
2416
2417                 pr->preferredRecordSyntax =
2418                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2419                                         m_backend_type);
2420             }
2421             else
2422                 pr->preferredRecordSyntax =
2423                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2424                                            VAL_USMARC);
2425         }
2426         else if (err)
2427         {
2428             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2429
2430             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2431             new_apdu->u.presentResponse->records =
2432                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2433             *new_apdu->u.presentResponse->presentStatus =
2434                 Z_PresentStatus_failure;
2435
2436             send_to_client(new_apdu);
2437
2438             return 0;
2439         }
2440         else if (m_backend_type)
2441         {
2442             pr->preferredRecordSyntax =
2443                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2444         }
2445     }
2446     return apdu;
2447 }
2448
2449 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2450 {
2451     if (!schema)
2452         return 0;
2453     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2454         odr_malloc(o, sizeof(Z_ElementSetNames));
2455     esn->which = Z_ElementSetNames_generic;
2456     esn->u.generic = odr_strdup(o, schema);
2457     return esn;
2458 }
2459
2460 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2461 {
2462     const char *t = 0;
2463     Yaz_ProxyConfig *cfg = check_reconfigure();
2464     if (cfg)
2465         t = cfg->get_explain_name(db, backend_db);
2466
2467     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2468     {
2469         releaseClient();
2470     }
2471
2472     if (t)
2473     {
2474         xfree(m_default_target);
2475         m_default_target = xstrdup(t);
2476     }
2477 }
2478
2479 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2480 {
2481     struct stat sbuf;
2482     if (strcmp(hreq->method, "GET"))
2483         return 0;
2484     if (hreq->path[0] != '/')
2485         return 0;
2486     const char *cp = hreq->path;
2487     while (*cp)
2488     {
2489         if (*cp == '/' && strchr("/.", cp[1]))
2490         {
2491             yaz_log(YLOG_LOG, "%sRejecting path %s", m_session_str,
2492                     hreq->path);
2493             return 0;
2494         }
2495         cp++;
2496     }
2497
2498     Yaz_ProxyConfig *cfg = check_reconfigure();
2499
2500     if (!cfg->get_file_access_info(hreq->path+1))
2501         return 0;
2502
2503     const char *fname = hreq->path+1;
2504     if (stat(fname, &sbuf))
2505     {
2506         yaz_log(YLOG_LOG|YLOG_ERRNO, "%sstat failed for %s", m_session_str,
2507                 fname);
2508         return 0;
2509     }
2510     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2511     {
2512         yaz_log(YLOG_LOG, "%sNot a regular file %s", m_session_str, fname);
2513         return 0;
2514     }
2515     if (sbuf.st_size > (off_t) 1000000)
2516     {
2517         yaz_log(YLOG_WARN, "%sFile %s too large for transfer", m_session_str,
2518                 fname);
2519         return 0;
2520     }
2521
2522     ODR o = odr_encode();
2523
2524     const char *ctype = cfg->check_mime_type(fname);
2525     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2526     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2527     if (m_http_version)
2528         hres->version = odr_strdup(o, m_http_version);
2529     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2530     if (m_http_keepalive)
2531         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2532     else
2533         timeout(0);
2534
2535     hres->content_len = sbuf.st_size;
2536     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2537     FILE *f = fopen(fname, "rb");
2538     if (f)
2539     {
2540         fread(hres->content_buf, 1, hres->content_len, f);
2541         fclose(f);
2542     }
2543     else
2544     {
2545         return 0;
2546     }
2547     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2548     {
2549         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2550                  fname);
2551     }
2552     int len;
2553     send_GDU(gdu, &len);
2554     recv_GDU_more(true);
2555     return 1;
2556 }
2557
2558 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2559 {
2560     if (m_s2z_odr_init)
2561     {
2562         odr_destroy(m_s2z_odr_init);
2563         m_s2z_odr_init = 0;
2564     }
2565     if (m_s2z_odr_search)
2566     {
2567         odr_destroy(m_s2z_odr_search);
2568         m_s2z_odr_search = 0;
2569     }
2570
2571     m_http_keepalive = 0;
2572     m_http_version = 0;
2573     if (!strcmp(hreq->version, "1.0"))
2574     {
2575         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2576         if (v && !strcmp(v, "Keep-Alive"))
2577             m_http_keepalive = 1;
2578         else
2579             m_http_keepalive = 0;
2580         m_http_version = "1.0";
2581     }
2582     else
2583     {
2584         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2585         if (v && !strcmp(v, "close"))
2586             m_http_keepalive = 0;
2587         else
2588             m_http_keepalive = 1;
2589         m_http_version = "1.1";
2590     }
2591
2592     const char *a = z_HTTP_header_lookup(hreq->headers, "Authorization");
2593     char authorization_str[255];
2594     *authorization_str = '\0';
2595     if (a && strncasecmp(a, "Basic ", 6) == 0)
2596         base64_decode(a + 6, authorization_str, 254);
2597
2598     Z_SRW_PDU *srw_pdu = 0;
2599     Z_SOAP *soap_package = 0;
2600     char *charset = 0;
2601     Z_SRW_diagnostic *diagnostic = 0;
2602     int num_diagnostic = 0;
2603
2604     if (file_access(hreq))
2605     {
2606         return;
2607     }
2608     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2609                             &charset) == 0
2610              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2611                                &charset, &diagnostic, &num_diagnostic) == 0)
2612     {
2613         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2614         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2615         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2616         m_s2z_init_apdu = 0;
2617         m_s2z_search_apdu = 0;
2618         m_s2z_present_apdu = 0;
2619
2620         m_s2z_stylesheet = 0;
2621
2622         Z_IdAuthentication *auth = NULL;
2623         if (*authorization_str)
2624         {
2625             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
2626             auth->which = Z_IdAuthentication_idPass;
2627             auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
2628             auth->u.idPass->groupId = NULL;
2629             char *p = strchr(authorization_str, ':');
2630             if (p)
2631             {
2632                 *p = '\0';
2633                 p++;
2634                 auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
2635             }
2636             auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
2637         }
2638
2639         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2640         {
2641
2642             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2643
2644             const char *backend_db = srw_req->database;
2645             srw_get_client(srw_req->database, &backend_db);
2646
2647             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2648             // recordXPath unsupported.
2649             if (srw_req->recordXPath)
2650             {
2651                 yaz_add_srw_diagnostic(odr_decode(),
2652                                        &diagnostic, &num_diagnostic,
2653                                        72, 0);
2654             }
2655             // sort unsupported
2656             if (srw_req->sort_type != Z_SRW_sort_type_none)
2657             {
2658                 yaz_add_srw_diagnostic(odr_decode(),
2659                                        &diagnostic, &num_diagnostic,
2660                                        80, 0);
2661             }
2662             // save stylesheet
2663             if (srw_req->stylesheet)
2664                 m_s2z_stylesheet =
2665                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2666
2667             // set packing for response records ..
2668             if (srw_req->recordPacking &&
2669                 !strcmp(srw_req->recordPacking, "xml"))
2670                 m_s2z_packing = Z_SRW_recordPacking_XML;
2671             else
2672                 m_s2z_packing = Z_SRW_recordPacking_string;
2673
2674             if (num_diagnostic)
2675             {
2676                 Z_SRW_PDU *srw_pdu =
2677                     yaz_srw_get(odr_encode(),
2678                                 Z_SRW_searchRetrieve_response);
2679                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2680
2681                 srw_res->diagnostics = diagnostic;
2682                 srw_res->num_diagnostics = num_diagnostic;
2683                 send_srw_response(srw_pdu);
2684                 return;
2685             }
2686
2687             // prepare search PDU
2688             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2689                                           Z_APDU_searchRequest);
2690             Z_SearchRequest *z_searchRequest =
2691                 m_s2z_search_apdu->u.searchRequest;
2692
2693             z_searchRequest->num_databaseNames = 1;
2694             z_searchRequest->databaseNames = (char**)
2695                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2696             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2697                                                            backend_db);
2698
2699             // query transformation
2700             Z_Query *query = (Z_Query *)
2701                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2702             z_searchRequest->query = query;
2703
2704             if (srw_req->query_type == Z_SRW_query_type_cql)
2705             {
2706                 Z_External *ext = (Z_External *)
2707                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2708                 ext->direct_reference =
2709                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2710                 ext->indirect_reference = 0;
2711                 ext->descriptor = 0;
2712                 ext->which = Z_External_CQL;
2713                 ext->u.cql = srw_req->query.cql;
2714
2715                 query->which = Z_Query_type_104;
2716                 query->u.type_104 =  ext;
2717             }
2718             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2719             {
2720                 Z_RPNQuery *RPNquery;
2721                 YAZ_PQF_Parser pqf_parser;
2722
2723                 pqf_parser = yaz_pqf_create ();
2724
2725                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2726                                           srw_req->query.pqf);
2727                 if (!RPNquery)
2728                 {
2729                     const char *pqf_msg;
2730                     size_t off;
2731                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2732                     int ioff = off;
2733                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
2734                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2735
2736                     send_to_srw_client_error(10, 0);
2737                     return;
2738                 }
2739                 query->which = Z_Query_type_1;
2740                 query->u.type_1 =  RPNquery;
2741
2742                 yaz_pqf_destroy (pqf_parser);
2743             }
2744             else
2745             {
2746                 send_to_srw_client_error(7, "query");
2747                 return;
2748             }
2749
2750             // present
2751             m_s2z_present_apdu = 0;
2752             int max = 0;
2753             if (srw_req->maximumRecords)
2754                 max = *srw_req->maximumRecords;
2755             int start = 1;
2756             if (srw_req->startRecord)
2757                 start = *srw_req->startRecord;
2758             if (max > 0)
2759             {
2760                 // Some backend, such as Voyager doesn't honor piggyback
2761                 // So we use present always (0 &&).
2762                 if (0 && start <= 1)  // Z39.50 piggyback
2763                 {
2764                     *z_searchRequest->smallSetUpperBound = max;
2765                     *z_searchRequest->mediumSetPresentNumber = max;
2766                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2767
2768                     z_searchRequest->preferredRecordSyntax =
2769                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2770                                                VAL_TEXT_XML);
2771                     if (srw_req->recordSchema)
2772                     {
2773                         z_searchRequest->smallSetElementSetNames =
2774                             z_searchRequest->mediumSetElementSetNames =
2775                             mk_esn_from_schema(m_s2z_odr_search,
2776                                                srw_req->recordSchema);
2777                     }
2778                 }
2779                 else   // Z39.50 present
2780                 {
2781                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2782                                                    Z_APDU_presentRequest);
2783                     Z_PresentRequest *z_presentRequest =
2784                         m_s2z_present_apdu->u.presentRequest;
2785                     *z_presentRequest->resultSetStartPoint = start;
2786                     *z_presentRequest->numberOfRecordsRequested = max;
2787                     z_presentRequest->preferredRecordSyntax =
2788                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2789                                                VAL_TEXT_XML);
2790                     if (srw_req->recordSchema)
2791                     {
2792                         z_presentRequest->recordComposition =
2793                             (Z_RecordComposition *)
2794                             odr_malloc(m_s2z_odr_search,
2795                                        sizeof(Z_RecordComposition));
2796                         z_presentRequest->recordComposition->which =
2797                             Z_RecordComp_simple;
2798                         z_presentRequest->recordComposition->u.simple =
2799                             mk_esn_from_schema(m_s2z_odr_search,
2800                                                srw_req->recordSchema);
2801                     }
2802                 }
2803             }
2804             if (!m_client)
2805             {
2806                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2807                                             Z_APDU_initRequest);
2808
2809                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2810
2811                 // prevent m_initRequest_apdu memory from being grabbed
2812                 // in Yaz_Proxy::handle_incoming_Z_PDU
2813                 m_initRequest_apdu = m_s2z_init_apdu;
2814                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2815                 return;
2816             }
2817             else
2818             {
2819                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2820                 return;
2821             }
2822         }
2823         else if (srw_pdu->which == Z_SRW_explain_request)
2824         {
2825             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2826
2827             const char *backend_db = srw_req->database;
2828             srw_get_client(srw_req->database, &backend_db);
2829
2830             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2831
2832             // save stylesheet
2833             if (srw_req->stylesheet)
2834                 m_s2z_stylesheet =
2835                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2836
2837             if (srw_req->recordPacking &&
2838                 !strcmp(srw_req->recordPacking, "xml"))
2839                 m_s2z_packing = Z_SRW_recordPacking_XML;
2840             else
2841                 m_s2z_packing = Z_SRW_recordPacking_string;
2842
2843             if (num_diagnostic)
2844             {
2845                 send_srw_explain_response(diagnostic, num_diagnostic);
2846                 return;
2847             }
2848
2849             if (!m_client)
2850             {
2851                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2852                                             Z_APDU_initRequest);
2853
2854                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2855
2856                 // prevent m_initRequest_apdu memory from being grabbed
2857                 // in Yaz_Proxy::handle_incoming_Z_PDU
2858                 m_initRequest_apdu = m_s2z_init_apdu;
2859                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2860             }
2861             else
2862                 send_srw_explain_response(0, 0);
2863             return;
2864         }
2865         else if (srw_pdu->which == Z_SRW_scan_request)
2866         {
2867             m_s2z_database = odr_strdup(m_s2z_odr_init,
2868                                         srw_pdu->u.scan_request->database);
2869
2870             yaz_add_srw_diagnostic(odr_decode(),
2871                                    &diagnostic, &num_diagnostic,
2872                                    4, "scan");
2873             Z_SRW_PDU *srw_pdu =
2874                 yaz_srw_get(odr_encode(),
2875                             Z_SRW_scan_response);
2876             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2877
2878             srw_res->diagnostics = diagnostic;
2879             srw_res->num_diagnostics = num_diagnostic;
2880             send_srw_response(srw_pdu);
2881             return;
2882         }
2883         else
2884         {
2885             m_s2z_database = 0;
2886
2887             send_to_srw_client_error(4, 0);
2888         }
2889     }
2890     send_http_response(400);
2891 }
2892
2893 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2894 {
2895
2896     Z_OtherInformation **oi;
2897     get_otherInfoAPDU(apdu, &oi);
2898
2899     if (apdu->u.initRequest->implementationId)
2900         yaz_log(YLOG_LOG, "%simplementationId: %s",
2901                 m_session_str, apdu->u.initRequest->implementationId);
2902     if (apdu->u.initRequest->implementationName)
2903         yaz_log(YLOG_LOG, "%simplementationName: %s",
2904                 m_session_str, apdu->u.initRequest->implementationName);
2905     if (apdu->u.initRequest->implementationVersion)
2906         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2907                 m_session_str, apdu->u.initRequest->implementationVersion);
2908     if (m_initRequest_apdu == 0)
2909     {
2910         if (m_initRequest_mem)
2911             nmem_destroy(m_initRequest_mem);
2912
2913         m_initRequest_apdu = apdu;
2914         m_initRequest_mem = odr_extract_mem(odr_decode());
2915
2916         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2917             preferredMessageSize;
2918         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2919         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2920             maximumRecordSize;
2921         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2922
2923         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2924             yaz_get_charneg_record(*oi);
2925
2926         // Save proposal charsets and langs.
2927         if (ODR_MASK_GET(apdu->u.initRequest->options,
2928                          Z_Options_negotiationModel)
2929             && charSetandLangRecord)
2930         {
2931
2932             yaz_get_proposal_charneg(m_referenceId_mem,
2933                                      charSetandLangRecord,
2934                                      &m_initRequest_oi_negotiation_charsets,
2935                                      &m_initRequest_oi_negotiation_num_charsets,
2936                                      &m_initRequest_oi_negotiation_langs,
2937                                      &m_initRequest_oi_negotiation_num_langs,
2938                                      &m_initRequest_oi_negotiation_selected);
2939
2940             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2941             {
2942                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2943                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2944                         m_initRequest_oi_negotiation_charsets[i]:"none");
2945             }
2946             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2947             {
2948                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2949                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2950                         m_initRequest_oi_negotiation_langs[i]:"none");
2951             }
2952             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2953                     m_session_str, m_initRequest_oi_negotiation_selected);
2954         }
2955         // save init options for the response..
2956         m_initRequest_options = apdu->u.initRequest->options;
2957
2958         apdu->u.initRequest->options =
2959             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2960                                        sizeof(Odr_bitmask));
2961         ODR_MASK_ZERO(apdu->u.initRequest->options);
2962         int i;
2963         for (i = 0; i<= 24; i++)
2964             ODR_MASK_SET(apdu->u.initRequest->options, i);
2965         // check negotiation option
2966         if (!ODR_MASK_GET(m_initRequest_options,
2967                           Z_Options_negotiationModel))
2968         {
2969             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2970                            Z_Options_negotiationModel);
2971         }
2972         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2973                        Z_Options_concurrentOperations);
2974         // make new version
2975         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2976         apdu->u.initRequest->protocolVersion =
2977             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2978                                        sizeof(Odr_bitmask));
2979         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2980
2981         for (i = 0; i<= 8; i++)
2982             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2983     }
2984     handle_charset_lang_negotiation(apdu);
2985     if (m_client->m_init_flag)
2986     {
2987         if (handle_init_response_for_invalid_session(apdu))
2988             return;
2989         if (m_client->m_initResponse)
2990         {
2991             Z_APDU *apdu2 = m_client->m_initResponse;
2992             apdu2->u.initResponse->otherInfo = 0;
2993             if (m_client->m_cookie && *m_client->m_cookie)
2994                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2995                                            m_client->m_cookie);
2996             apdu2->u.initResponse->referenceId =
2997                 apdu->u.initRequest->referenceId;
2998             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2999             apdu2->u.initResponse->protocolVersion =
3000                 m_client->m_initResponse_version;
3001
3002             handle_charset_lang_negotiation(apdu2);
3003
3004             send_to_client(apdu2);
3005             m_timeout_mode = timeout_normal;
3006             return;
3007         }
3008     }
3009     m_client->m_init_flag = 1;
3010
3011 #if USE_AUTH_MSG
3012     Auth_Msg *m = new Auth_Msg;
3013     m->m_proxy = this;
3014     z_APDU(odr_encode(), &apdu, 0, "encode");
3015     char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
3016     m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
3017     memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
3018     odr_reset(odr_encode());
3019
3020     inc_ref();
3021     m_my_thread->put(m);
3022 #else
3023     int ret = handle_authentication(apdu);
3024     result_authentication(apdu, ret);
3025 #endif
3026 }
3027
3028 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
3029 {
3030     Z_ReferenceId **refid = get_referenceIdP(apdu);
3031     nmem_reset(m_referenceId_mem);
3032     if (refid && *refid)
3033     {
3034         m_referenceId = (Z_ReferenceId *)
3035             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
3036         m_referenceId->len = m_referenceId->size = (*refid)->len;
3037         m_referenceId->buf = (unsigned char *)
3038             nmem_malloc(m_referenceId_mem, (*refid)->len);
3039         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
3040     }
3041     else
3042         m_referenceId = 0;
3043
3044     if (!m_client && m_flag_invalid_session)
3045     {
3046         // Got request for a session that is invalid..
3047         m_apdu_invalid_session = apdu; // save package
3048         m_mem_invalid_session = odr_extract_mem(odr_decode());
3049         apdu = m_initRequest_apdu;     // but throw an init to the target
3050     }
3051
3052     if (apdu->which == Z_APDU_searchRequest)
3053         m_search_stat.add_bytes(1);
3054
3055     // Determine our client.
3056     Z_OtherInformation **oi;
3057     get_otherInfoAPDU(apdu, &oi);
3058     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
3059     if (!m_client)
3060     {
3061         if (m_http_version)
3062         {   // HTTP. Send not found
3063             send_http_response(404);
3064             return;
3065         }
3066         else
3067         {
3068             // Z39.50 just shutdown
3069             delete this;
3070             return;
3071         }
3072     }
3073
3074     m_client->m_server = this;
3075
3076     if (apdu->which == Z_APDU_initRequest)
3077         handle_init(apdu);
3078     else
3079         handle_incoming_Z_PDU_2(apdu);
3080 }
3081
3082 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
3083 {
3084     handle_max_record_retrieve(apdu);
3085
3086     if (apdu)
3087         apdu = handle_syntax_validation(apdu);
3088
3089     if (apdu)
3090         apdu = handle_query_transformation(apdu);
3091
3092     if (apdu)
3093         apdu = handle_target_charset_conversion(apdu);
3094
3095     if (apdu)
3096         apdu = handle_query_validation(apdu);
3097
3098     if (apdu)
3099         apdu = result_set_optimize(apdu);
3100
3101     if (!apdu)
3102     {
3103         m_client->timeout(m_target_idletime);  // mark it active even
3104         recv_GDU_more(true);
3105         // though we didn't use it
3106         return;
3107     }
3108
3109     // delete other info construct completely if 0 elements
3110     Z_OtherInformation **oi;
3111     get_otherInfoAPDU(apdu, &oi);
3112     if (oi && *oi && (*oi)->num_elements == 0)
3113         *oi = 0;
3114
3115     if (apdu->which == Z_APDU_presentRequest &&
3116         m_client->m_resultSetStartPoint == 0)
3117     {
3118         Z_PresentRequest *pr = apdu->u.presentRequest;
3119         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3120         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3121     } else {
3122         m_client->m_resultSetStartPoint = 0;
3123     }
3124     if (m_client->send_to_target(apdu) < 0)
3125     {
3126         m_client->shutdown();
3127     }
3128     else
3129         m_client->m_waiting = 1;
3130 }
3131
3132 void Yaz_Proxy::connectNotify()
3133 {
3134 }
3135
3136 void Yaz_Proxy::releaseClient()
3137 {
3138     xfree(m_proxyTarget);
3139     m_proxyTarget = 0;
3140     m_flag_invalid_session = 0;
3141     // only keep if keep_alive flag is set...
3142     if (m_client &&
3143         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3144         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3145         m_client->m_waiting == 0)
3146     {
3147         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3148                  m_session_str,
3149                  m_client->get_hostname());
3150         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3151                 m_session_str, m_client->m_pdu_recv,
3152                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3153                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3154         assert (m_client->m_waiting != 2);
3155         // Tell client (if any) that no server connection is there..
3156         m_client->m_server = 0;
3157         m_client = 0;
3158     }
3159     else if (m_client)
3160     {
3161         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
3162                  m_session_str,
3163                  m_client->get_hostname());
3164         assert (m_client->m_waiting != 2);
3165         delete m_client;
3166         m_client = 0;
3167     }
3168     else if (!m_parent)
3169     {
3170         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
3171                  m_session_str);
3172         assert (m_parent);
3173     }
3174     else
3175     {
3176         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
3177                  m_session_str);
3178     }
3179     if (m_parent)
3180         m_parent->pre_init();
3181 }
3182
3183 bool Yaz_Proxy::dec_ref(bool main_ptr)
3184 {
3185     assert(m_ref_count > 0);
3186     if (main_ptr)
3187     {
3188         if (m_main_ptr_dec)
3189             return false;
3190         m_main_ptr_dec = true;
3191     }
3192
3193     m_http_keepalive = 0;
3194
3195     --m_ref_count;
3196     if (m_ref_count > 0)
3197         return false;
3198
3199     releaseClient();
3200
3201     delete this;
3202     return true;
3203 }
3204
3205 const char *Yaz_ProxyClient::get_session_str()
3206 {
3207     if (!m_server)
3208         return "0 ";
3209     return m_server->get_session_str();
3210 }
3211
3212 void Yaz_ProxyClient::shutdown()
3213 {
3214     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3215              get_hostname());
3216
3217     if (m_server)
3218     {
3219         m_waiting = 1;      // ensure it's released from Proxy in releaseClient
3220         m_server->dec_ref(true);
3221     }
3222     else
3223         delete this;
3224 }
3225
3226 void Yaz_Proxy::failNotify()
3227 {
3228     inc_request_no();
3229     yaz_log (YLOG_LOG, "%sConnection closed by client",
3230              get_session_str());
3231     dec_ref(true);
3232 }
3233
3234 void Yaz_Proxy::send_response_fail_client(const char *addr)
3235 {
3236     if (m_http_version)
3237     {
3238         Z_SRW_diagnostic *diagnostic = 0;
3239         int num_diagnostic = 0;
3240
3241         yaz_add_srw_diagnostic(odr_encode(),
3242                                &diagnostic, &num_diagnostic,
3243                                YAZ_SRW_SYSTEM_TEMPORARILY_UNAVAILABLE, addr);
3244         if (m_s2z_search_apdu)
3245             send_srw_search_response(diagnostic, num_diagnostic);
3246         else
3247             send_srw_explain_response(diagnostic, num_diagnostic);
3248     }
3249 }
3250 void Yaz_ProxyClient::failNotify()
3251 {
3252     if (m_server)
3253         m_server->inc_request_no();
3254     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
3255              get_session_str(), get_hostname());
3256
3257     if (m_server)
3258         m_server->send_response_fail_client(get_hostname());
3259     shutdown();
3260 }
3261
3262 void Yaz_ProxyClient::connectNotify()
3263 {
3264     const char *s = get_session_str();
3265     const char *h = get_hostname();
3266     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3267              m_target_idletime);
3268     timeout(m_target_idletime);
3269     if (!m_server)
3270         pre_init_client();
3271 }
3272
3273 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3274                                               *the_PDU_Observable, int fd)
3275 {
3276     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3277 }
3278
3279 Yaz_ProxyClient::~Yaz_ProxyClient()
3280 {
3281     if (m_prev)
3282         *m_prev = m_next;
3283     if (m_next)
3284         m_next->m_prev = m_prev;
3285     m_waiting = 2;     // for debugging purposes only.
3286     odr_destroy(m_init_odr);
3287     odr_destroy(m_idAuthentication_odr);
3288     delete m_last_query;
3289     xfree (m_last_resultSetId);
3290     xfree (m_cookie);
3291 }
3292
3293 void Yaz_ProxyClient::pre_init_client()
3294 {
3295     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3296     Z_InitRequest *req = apdu->u.initRequest;
3297
3298     int i;
3299     for (i = 0; i<= 24; i++)
3300         ODR_MASK_SET(req->options, i);
3301     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3302                    Z_Options_negotiationModel);
3303     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3304                    Z_Options_concurrentOperations);
3305     for (i = 0; i<= 10; i++)
3306         ODR_MASK_SET(req->protocolVersion, i);
3307
3308     if (send_to_target(apdu) < 0)
3309     {
3310         delete this;
3311     }
3312     else
3313     {
3314         m_waiting = 1;
3315         m_init_flag = 1;
3316     }
3317 }
3318
3319 void Yaz_Proxy::pre_init()
3320 {
3321     int i;
3322     const char *name = 0;
3323     const char *zurl_in_use[MAX_ZURL_PLEX];
3324     int limit_bw, limit_pdu, limit_req, limit_search;
3325     int target_idletime, client_idletime;
3326     int max_clients;
3327     int keepalive_limit_bw, keepalive_limit_pdu;
3328     int pre_init;
3329     const char *cql2rpn = 0;
3330     const char *authentication = 0;
3331     const char *negotiation_charset = 0;
3332     const char *negotiation_lang = 0;
3333
3334     Yaz_ProxyConfig *cfg = check_reconfigure();
3335
3336     zurl_in_use[0] = 0;
3337
3338     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3339         set_APDU_yazlog(1);
3340     else
3341         set_APDU_yazlog(0);
3342
3343     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3344                                           &limit_bw, &limit_pdu, &limit_req,
3345                                           &limit_search,
3346                                           &target_idletime, &client_idletime,
3347                                           &max_clients,
3348                                           &keepalive_limit_bw,
3349                                           &keepalive_limit_pdu,
3350                                           &pre_init,
3351                                           &cql2rpn,
3352                                           &authentication,
3353                                           &negotiation_charset,
3354                                           &negotiation_lang,
3355                                           0,
3356                                           0) ; i++)
3357     {
3358         if (pre_init)
3359         {
3360             int j;
3361             for (j = 0; zurl_in_use[j]; j++)
3362             {
3363                 Yaz_ProxyClient *c;
3364                 int spare = 0;
3365                 int spare_waiting = 0;
3366                 int in_use = 0;
3367                 int other = 0;
3368                 for (c = m_clientPool; c; c = c->m_next)
3369                 {
3370                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3371                     {
3372                         if (c->m_cookie == 0)
3373                         {
3374                             if (c->m_server == 0)
3375                                 if (c->m_waiting)
3376                                     spare_waiting++;
3377                                 else
3378                                     spare++;
3379                             else
3380                                 in_use++;
3381                         }
3382                         else
3383                             other++;
3384                     }
3385                 }
3386                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3387                         "sparew=%d preinit=%d",m_session_str,
3388                         name, zurl_in_use[j], in_use, other,
3389                         spare, spare_waiting, pre_init);
3390                 if (spare + spare_waiting < pre_init)
3391                 {
3392                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3393                     c->m_next = m_clientPool;
3394                     if (c->m_next)
3395                         c->m_next->m_prev = &c->m_next;
3396                     m_clientPool = c;
3397                     c->m_prev = &m_clientPool;
3398
3399                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3400                         c->set_APDU_yazlog(1);
3401                     else
3402                         c->set_APDU_yazlog(0);
3403
3404                     if (c->client(zurl_in_use[j]))
3405                     {
3406                         timeout(60);
3407                         delete c;
3408                         return;
3409                     }
3410                     c->timeout(30);
3411                     c->m_waiting = 1;
3412                     c->m_target_idletime = target_idletime;
3413                     c->m_seqno = m_seqno++;
3414                 }
3415             }
3416         }
3417     }
3418 }
3419
3420 void Yaz_Proxy::timeoutNotify()
3421 {
3422     if (m_parent)
3423     {
3424         GDU *gdu;
3425         switch(m_timeout_mode)
3426         {
3427         case timeout_normal:
3428         case timeout_busy:
3429             inc_request_no();
3430             m_in_queue.clear();
3431             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3432             dec_ref(true);
3433             break;
3434         case timeout_reduce:
3435             timeout(m_client_idletime);
3436             m_timeout_mode = timeout_busy;
3437             gdu = m_timeout_gdu;
3438             m_timeout_gdu = 0;
3439             recv_GDU_normal(gdu);
3440             break;
3441         case timeout_xsl:
3442             assert(m_stylesheet_nprl);
3443             convert_xsl_delay();
3444             recv_GDU_more(true);
3445         }
3446     }
3447     else
3448     {
3449         timeout(600);
3450         pre_init();
3451     }
3452 }
3453
3454 void Yaz_Proxy::markInvalid()
3455 {
3456     m_client = 0;
3457     m_flag_invalid_session = 1;
3458 }
3459
3460 void Yaz_ProxyClient::timeoutNotify()
3461 {
3462     if (m_server)
3463         m_server->inc_request_no();
3464
3465     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3466              get_hostname());
3467
3468     if (m_server)
3469         m_server->send_response_fail_client(get_hostname());
3470     shutdown();
3471
3472     m_root->pre_init();
3473 }
3474
3475 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
3476                                  Yaz_Proxy *parent) :
3477     Z_Assoc (the_PDU_Observable)
3478 {
3479     m_cookie = 0;
3480     m_next = 0;
3481     m_prev = 0;
3482     m_init_flag = 0;
3483     m_last_query = 0;
3484     m_last_resultSetId = 0;
3485     m_last_resultCount = 0;
3486     m_last_ok = 0;
3487     m_sr_transform = 0;
3488     m_waiting = 0;
3489     m_init_odr = odr_createmem (ODR_DECODE);
3490     m_initResponse = 0;
3491     m_initResponse_options = 0;
3492     m_initResponse_version = 0;
3493     m_initResponse_preferredMessageSize = 0;
3494     m_initResponse_maximumRecordSize = 0;
3495     m_resultSetStartPoint = 0;
3496     m_bytes_sent = m_bytes_recv = 0;
3497     m_pdu_recv = 0;
3498     m_server = 0;
3499     m_seqno = 0;
3500     m_target_idletime = 600;
3501     m_root = parent;
3502     m_idAuthentication_odr = odr_createmem(ODR_ENCODE);
3503     m_idAuthentication_ber_buf = 0;
3504     m_idAuthentication_ber_size = 0;
3505 }
3506
3507 const char *Yaz_Proxy::option(const char *name, const char *value)
3508 {
3509     if (!strcmp (name, "optimize")) {
3510         if (value) {
3511             xfree (m_optimize);
3512             m_optimize = xstrdup (value);
3513         }
3514         return m_optimize;
3515     }
3516     return 0;
3517 }
3518
3519 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
3520 {
3521
3522 }
3523
3524 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
3525 {
3526     if (apdu->which == Z_GDU_Z3950)
3527         recv_Z_PDU(apdu->u.z3950, len);
3528     else if (apdu->which == Z_GDU_HTTP_Response)
3529         recv_HTTP_response(apdu->u.HTTP_Response, len);
3530     else
3531         shutdown();
3532 }
3533
3534 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
3535 {
3536     if (!m_flag_invalid_session)
3537         return 0;
3538     m_flag_invalid_session = 0;
3539     handle_incoming_Z_PDU(m_apdu_invalid_session);
3540     assert (m_mem_invalid_session);
3541     nmem_destroy(m_mem_invalid_session);
3542     m_mem_invalid_session = 0;
3543     return 1;
3544 }
3545
3546 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
3547 {
3548     m_bytes_recv += len;
3549
3550     m_pdu_recv++;
3551     m_waiting = 0;
3552     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
3553         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
3554                  apdu_name(apdu), get_hostname(), len);
3555     if (apdu->which == Z_APDU_initResponse)
3556     {
3557         if (!m_server)  // if this is a pre init session , check for more
3558             m_root->pre_init();
3559         NMEM nmem = odr_extract_mem (odr_decode());
3560         odr_reset (m_init_odr);
3561         nmem_transfer (m_init_odr->mem, nmem);
3562         m_initResponse = apdu;
3563         m_initResponse_options = apdu->u.initResponse->options;
3564         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3565         m_initResponse_preferredMessageSize =
3566             *apdu->u.initResponse->preferredMessageSize;
3567         m_initResponse_maximumRecordSize =
3568             *apdu->u.initResponse->maximumRecordSize;
3569
3570         Z_InitResponse *ir = apdu->u.initResponse;
3571         char *im0 = ir->implementationName;
3572
3573         char *im1 = (char*)
3574             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3575         *im1 = '\0';
3576         if (im0)
3577         {
3578             strcat(im1, im0);
3579             strcat(im1, " ");
3580         }
3581         strcat(im1, "(YAZ Proxy)");
3582         ir->implementationName = im1;
3583
3584         nmem_destroy (nmem);
3585
3586         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3587             return;
3588     }
3589     if (apdu->which == Z_APDU_searchResponse)
3590     {
3591         Z_SearchResponse *sr = apdu->u.searchResponse;
3592         m_last_resultCount = *sr->resultCount;
3593         int status = *sr->searchStatus;
3594         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3595         {
3596             m_last_ok = 1;
3597
3598             if (sr->records && sr->records->which == Z_Records_DBOSD)
3599             {
3600                 m_cache.add(odr_decode(),
3601                             sr->records->u.databaseOrSurDiagnostics, 1,
3602                             *sr->resultCount);
3603             }
3604         }
3605     }
3606     if (apdu->which == Z_APDU_presentResponse)
3607     {
3608         Z_PresentResponse *pr = apdu->u.presentResponse;
3609         if (m_sr_transform)
3610         {
3611             m_sr_transform = 0;
3612             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3613             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3614             sr->referenceId = pr->referenceId;
3615             *sr->resultCount = m_last_resultCount;
3616             sr->records = pr->records;
3617             sr->nextResultSetPosition = pr->nextResultSetPosition;
3618             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3619             apdu = new_apdu;
3620         }
3621         if (pr->records &&
3622             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3623         {
3624             m_cache.add(odr_decode(),
3625                         pr->records->u.databaseOrSurDiagnostics,
3626                         m_resultSetStartPoint, -1);
3627             m_resultSetStartPoint = 0;
3628         }
3629     }
3630     if (m_cookie)
3631         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3632
3633     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
3634
3635     if (server)
3636         server->send_to_client(apdu);
3637     if (apdu->which == Z_APDU_close)
3638         shutdown();
3639     else if (server)
3640         server->recv_GDU_more(true);
3641 }
3642
3643 void Yaz_Proxy::low_socket_close()
3644 {
3645 #if WIN32
3646 #else
3647     int i;
3648     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3649         if  (m_lo_fd[i] >= 0)
3650             ::close(m_lo_fd[i]);
3651 #endif
3652 }
3653
3654 void Yaz_Proxy::low_socket_open()
3655 {
3656 #if WIN32
3657 #else
3658     int i;
3659     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3660         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3661 #endif
3662 }
3663
3664 int Yaz_Proxy::server(const char *addr)
3665 {
3666     int r = Z_Assoc::server(addr);
3667     if (!r)
3668     {
3669         yaz_log(YLOG_LOG, "%sStarted proxy "
3670 #ifdef VERSION
3671             VERSION
3672 #endif
3673             " on %s", m_session_str, addr);
3674         timeout(1);
3675     }
3676     return r;
3677 }
3678
3679 void Yaz_Proxy::base64_decode(const char *base64, char *buf, int buf_len)
3680 {
3681     const char *base64_chars =
3682         "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
3683     int len = strlen(base64);
3684     int buf_pos = 0;
3685     int index = 1;
3686
3687     for (int pos = 0; pos <= len; pos++)
3688     {
3689         if (base64[pos] == '=' || buf_pos + 1 >= buf_len)
3690             break;
3691
3692         const char *ch_ptr = strchr(base64_chars, base64[pos]);
3693         if (!ch_ptr)
3694             break;
3695         char ch = (char) (ch_ptr - base64_chars);
3696         switch (index)
3697         {
3698             case 1:
3699                 buf[buf_pos] = ch << 2;
3700                 break;
3701             case 2:
3702                 buf[buf_pos++] += (ch & 0x30) >> 4;
3703                 buf[buf_pos] = (ch & 0x0f) << 4;
3704                 break;
3705             case 3:
3706                 buf[buf_pos++] += (ch & 0x3c) >> 2;
3707                 buf[buf_pos] = (ch & 0x03) << 6;
3708                 break;
3709             case 4:
3710                 buf[buf_pos++] += ch;
3711         }
3712         if (index < 4)
3713             index++;
3714         else
3715             index = 1;
3716     }
3717     buf[buf_pos] = '\0';
3718 }
3719
3720 /*
3721  * Local variables:
3722  * c-basic-offset: 4
3723  * indent-tabs-mode: nil
3724  * End:
3725  * vim: shiftwidth=4 tabstop=8 expandtab
3726  */
3727