src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.46 2006-03-30 11:59:34 adam Exp $
   2    Copyright (c) 1998-2006, Index Data.
   3
   4 This file is part of the yazproxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <stdlib.h>
  42 #include <time.h>
  43 #include <fcntl.h>
  44
  45 #include <yaz/srw.h>
  46 #include <yaz/marcdisp.h>
  47 #include <yaz/yaz-iconv.h>
  48 #include <yaz/log.h>
  49 #include <yaz/diagbib1.h>
  50 #include "proxyp.h"
  51 #include <yaz/pquery.h>
  52 #include <yaz/otherinfo.h>
  53 #include <yaz/charneg.h>
  54 #include "msg-thread.h"
  55
  56 using namespace yazpp_1;
  57
  58 #ifdef WIN32
  59 #define strncasecmp _strnicmp
  60 #endif
  61
  62 #define USE_AUTH_MSG 1
  63
  64 #if USE_AUTH_MSG
  65 class Auth_Msg : public IMsg_Thread {
  66 public:
  67     int m_ret;
  68     IMsg_Thread *handle();
  69     void result();
  70     Yaz_Proxy *m_proxy;
  71     NMEM m_nmem;
  72     char *m_apdu_buf;
  73     int m_apdu_len;
  74     Auth_Msg();
  75     virtual ~Auth_Msg();
  76 };
  77
  78 Auth_Msg::Auth_Msg()
  79 {
  80     m_nmem = nmem_create();
  81 }
  82
  83 Auth_Msg::~Auth_Msg()
  84 {
  85     nmem_destroy(m_nmem);
  86 }
  87
  88 IMsg_Thread *Auth_Msg::handle()
  89 {
  90     ODR decode = odr_createmem(ODR_DECODE);
  91     Z_APDU *apdu;
  92
  93     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
  94     int r = z_APDU(decode, &apdu, 0, 0);
  95     if (!r)
  96     {
  97         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
  98     }
  99     else
 100     {
 101         m_ret = m_proxy->handle_authentication(apdu);
 102     }
 103     odr_destroy(decode);
 104     return this;
 105 }
 106
 107 void Auth_Msg::result()
 108 {
 109     if (m_proxy->dec_ref(false))
 110     {
 111         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
 112     }
 113     else
 114     {
 115         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
 116         Z_APDU *apdu = 0;
 117         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
 118         if (!r)
 119             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
 120         m_proxy->result_authentication(apdu, m_ret);
 121     }
 122     delete this;
 123 }
 124
 125 #endif
 126
 127 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
 128 {
 129     if (apdu == 0 || ret == 0)
 130     {
 131         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
 132         *apdu_reject->u.initResponse->result = 0;
 133         send_to_client(apdu_reject);
 134         dec_ref(false);
 135     }
 136     else
 137     {
 138         if (apdu->which == Z_APDU_initRequest)
 139         {
 140             Yaz_ProxyConfig *cfg = check_reconfigure();
 141             if (cfg)
 142                 cfg->target_authentication(m_default_target, odr_encode(),
 143                                            apdu->u.initRequest);
 144         }
 145         handle_incoming_Z_PDU_2(apdu);
 146     }
 147 }
 148
 149 static const char *apdu_name(Z_APDU *apdu)
 150 {
 151     switch (apdu->which)
 152     {
 153     case Z_APDU_initRequest:
 154         return "initRequest";
 155     case Z_APDU_initResponse:
 156         return "initResponse";
 157     case Z_APDU_searchRequest:
 158         return "searchRequest";
 159     case Z_APDU_searchResponse:
 160         return "searchResponse";
 161     case Z_APDU_presentRequest:
 162         return "presentRequest";
 163     case Z_APDU_presentResponse:
 164         return "presentResponse";
 165     case Z_APDU_deleteResultSetRequest:
 166         return "deleteResultSetRequest";
 167     case Z_APDU_deleteResultSetResponse:
 168         return "deleteResultSetResponse";
 169     case Z_APDU_scanRequest:
 170         return "scanRequest";
 171     case Z_APDU_scanResponse:
 172         return "scanResponse";
 173     case Z_APDU_sortRequest:
 174         return "sortRequest";
 175     case Z_APDU_sortResponse:
 176         return "sortResponse";
 177     case Z_APDU_extendedServicesRequest:
 178         return "extendedServicesRequest";
 179     case Z_APDU_extendedServicesResponse:
 180         return "extendedServicesResponse";
 181     case Z_APDU_close:
 182         return "close";
 183     }
 184     return "other";
 185 }
 186
 187 static const char *gdu_name(Z_GDU *gdu)
 188 {
 189     switch(gdu->which)
 190     {
 191     case Z_GDU_Z3950:
 192         return apdu_name(gdu->u.z3950);
 193     case Z_GDU_HTTP_Request:
 194         return "HTTP Request";
 195     case Z_GDU_HTTP_Response:
 196         return "HTTP Response";
 197     }
 198     return "Unknown request/response";
 199 }
 200
 201 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
 202                      ISocketObservable *the_socket_observable,
 203                      Yaz_Proxy *parent)
 204     :
 205     Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 206 {
 207     m_PDU_Observable = the_PDU_Observable;
 208     m_socket_observable = the_socket_observable;
 209     m_client = 0;
 210     m_parent = parent;
 211     m_clientPool = 0;
 212     m_seqno = 1;
 213     m_keepalive_limit_bw = 500000;
 214     m_keepalive_limit_pdu = 1000;
 215     m_proxyTarget = 0;
 216     m_default_target = 0;
 217     m_proxy_negotiation_charset = 0;
 218     m_proxy_negotiation_lang = 0;
 219     m_proxy_negotiation_default_charset = 0;
 220     m_charset_converter = new Yaz_CharsetConverter;
 221     m_max_clients = 150;
 222     m_log_mask = 0;
 223     m_seed = time(0);
 224     m_client_idletime = 600;
 225     m_target_idletime = 600;
 226     m_optimize = xstrdup ("1");
 227     strcpy(m_session_str, "0 ");
 228     m_session_no = 0;
 229     m_bytes_sent = 0;
 230     m_bytes_recv = 0;
 231     m_bw_max = 0;
 232     m_pdu_max = 0;
 233     m_search_max = 0;
 234     m_connect_max = 0;
 235     m_timeout_mode = timeout_normal;
 236     m_timeout_gdu = 0;
 237     m_max_record_retrieve = 0;
 238     m_reconfig_flag = 0;
 239     m_config_fname = 0;
 240     m_request_no = 0;
 241     m_flag_invalid_session = 0;
 242     m_referenceId = 0;
 243     m_referenceId_mem = nmem_create();
 244     m_config = 0;
 245     m_marcxml_mode = none;
 246     m_stylesheet_xsp = 0;
 247     m_stylesheet_nprl = 0;
 248     m_stylesheet_apdu = 0;
 249     m_s2z_stylesheet = 0;
 250     m_s2z_database = 0;
 251     m_schema = 0;
 252     m_backend_type = 0;
 253     m_backend_charset = 0;
 254     m_frontend_type = 0;
 255     m_initRequest_apdu = 0;
 256     m_initRequest_mem = 0;
 257     m_initRequest_preferredMessageSize = 0;
 258     m_initRequest_maximumRecordSize = 0;
 259     m_initRequest_options = 0;
 260     m_initRequest_version = 0;
 261     m_initRequest_oi_negotiation_charsets = 0;
 262     m_initRequest_oi_negotiation_num_charsets = 0;
 263     m_initRequest_oi_negotiation_langs = 0;
 264     m_initRequest_oi_negotiation_num_langs = 0;
 265     m_initRequest_oi_negotiation_selected = 0;
 266     m_apdu_invalid_session = 0;
 267     m_mem_invalid_session = 0;
 268     m_s2z_odr_init = 0;
 269     m_s2z_odr_search = 0;
 270     m_s2z_init_apdu = 0;
 271     m_s2z_search_apdu = 0;
 272     m_s2z_present_apdu = 0;
 273     m_http_keepalive = 0;
 274     m_http_version = 0;
 275     m_soap_ns = 0;
 276     m_s2z_packing = Z_SRW_recordPacking_string;
 277 #if HAVE_GETTIMEOFDAY
 278     m_time_tv = xmalloc(sizeof(struct timeval));
 279     struct timeval *tv = (struct timeval *) m_time_tv;
 280     tv->tv_sec = 0;
 281     tv->tv_usec = 0;
 282 #else
 283     m_time_tv = 0;
 284 #endif
 285     m_usemarcon_ini_stage1 = 0;
 286     m_usemarcon_ini_stage2 = 0;
 287     m_usemarcon = new Yaz_usemarcon();
 288     if (!m_parent)
 289         low_socket_open();
 290     m_my_thread = 0;
 291     m_ref_count = 1;
 292     m_main_ptr_dec = false;
 293     m_peername = 0;
 294 }
 295
 296 void Yaz_Proxy::inc_ref()
 297 {
 298     m_ref_count++;
 299 }
 300
 301 Yaz_Proxy::~Yaz_Proxy()
 302 {
 303     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 304             m_bytes_sent, m_bytes_recv);
 305     nmem_destroy(m_initRequest_mem);
 306     nmem_destroy(m_mem_invalid_session);
 307     nmem_destroy(m_referenceId_mem);
 308
 309     xfree(m_proxyTarget);
 310     xfree(m_default_target);
 311     xfree(m_proxy_negotiation_charset);
 312     xfree(m_proxy_negotiation_lang);
 313     xfree(m_proxy_negotiation_default_charset);
 314     delete m_charset_converter;
 315     xfree(m_optimize);
 316
 317 #if HAVE_XSLT
 318     if (m_stylesheet_xsp)
 319         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 320 #endif
 321     xfree (m_time_tv);
 322
 323     xfree (m_peername);
 324     xfree (m_schema);
 325     xfree (m_backend_type);
 326     xfree (m_backend_charset);
 327     xfree (m_usemarcon_ini_stage1);
 328     xfree (m_usemarcon_ini_stage2);
 329     delete m_usemarcon;
 330     if (m_s2z_odr_init)
 331         odr_destroy(m_s2z_odr_init);
 332     if (m_s2z_odr_search)
 333         odr_destroy(m_s2z_odr_search);
 334     if (!m_parent)
 335         low_socket_close();
 336     if (!m_parent)
 337         delete m_my_thread;
 338     delete m_config;
 339 }
 340
 341 void Yaz_Proxy::set_debug_mode(int mode)
 342 {
 343     m_debug_mode = mode;
 344 }
 345
 346 int Yaz_Proxy::set_config(const char *config)
 347 {
 348     delete m_config;
 349     m_config = new Yaz_ProxyConfig();
 350     xfree(m_config_fname);
 351     m_config_fname = xstrdup(config);
 352     int r = m_config->read_xml(config);
 353     if (!r)
 354         m_config->get_generic_info(&m_log_mask, &m_max_clients,
 355                                    &m_connect_max);
 356     return r;
 357 }
 358
 359 void Yaz_Proxy::set_default_target(const char *target)
 360 {
 361     xfree (m_default_target);
 362     m_default_target = 0;
 363     if (target)
 364         m_default_target = (char *) xstrdup (target);
 365 }
 366
 367 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang,
 368                                        const char *default_charset)
 369 {
 370     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
 371         "default charset to '%s', language to '%s'", m_session_str,
 372         charset?charset:"none",
 373         default_charset?default_charset:"none",
 374         lang?lang:"none");
 375     xfree (m_proxy_negotiation_charset);
 376     xfree (m_proxy_negotiation_lang);
 377     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 378     if (charset)
 379         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 380     if (lang)
 381         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 382     if (default_charset)
 383         m_proxy_negotiation_default_charset =
 384             (char *) xstrdup (default_charset);
 385 }
 386
 387 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 388 {
 389     if (m_parent)
 390         return m_parent->check_reconfigure();
 391
 392     Yaz_ProxyConfig *cfg = m_config;
 393     if (m_reconfig_flag)
 394     {
 395         yaz_log(YLOG_LOG, "reconfigure");
 396         yaz_log_reopen();
 397         if (m_config_fname && cfg)
 398         {
 399             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 400             int r = cfg->read_xml(m_config_fname);
 401             if (r)
 402                 yaz_log(YLOG_WARN, "reconfigure failed");
 403             else
 404             {
 405                 m_log_mask = 0;
 406                 cfg->get_generic_info(&m_log_mask, &m_max_clients,
 407                                       &m_connect_max);
 408             }
 409         }
 410         else
 411             yaz_log(YLOG_LOG, "reconfigure");
 412         m_reconfig_flag = 0;
 413     }
 414     return cfg;
 415 }
 416
 417 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 418                                         *the_PDU_Observable, int fd)
 419 {
 420     check_reconfigure();
 421
 422     char session_str[200];
 423     sprintf(session_str, "%ld:%d ", (long) time(0), m_session_no);
 424     m_session_no++;
 425
 426     const char *peername = the_PDU_Observable->getpeername();
 427     yaz_log (YLOG_LOG, "%sNew session %s", session_str, peername);
 428
 429     m_connect.cleanup(false);
 430     m_connect.add_connect(peername);
 431
 432     int connect_total = m_connect.get_total(peername);
 433     int connect_max = m_connect_max;
 434     if (connect_max && connect_total > connect_max)
 435     {
 436         yaz_log(YLOG_LOG, "%sconnect not accepted total=%d max=%d",
 437                 session_str, connect_total, connect_max);
 438         return 0;
 439     }
 440     yaz_log(YLOG_LOG, "%sconnect accepted total=%d", session_str,
 441             connect_total);
 442
 443     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 444                                          m_socket_observable, this);
 445     new_proxy->m_config = 0;
 446     new_proxy->m_config_fname = 0;
 447     new_proxy->timeout(m_client_idletime);
 448     new_proxy->m_target_idletime = m_target_idletime;
 449     new_proxy->set_default_target(m_default_target);
 450     new_proxy->m_max_clients = m_max_clients;
 451     new_proxy->m_log_mask = m_log_mask;
 452     new_proxy->set_APDU_log(get_APDU_log());
 453     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 454         new_proxy->set_APDU_yazlog(1);
 455     else
 456         new_proxy->set_APDU_yazlog(0);
 457     strcpy(new_proxy->m_session_str, session_str);
 458     new_proxy->m_peername = xstrdup(peername);
 459     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 460         m_proxy_negotiation_lang, m_proxy_negotiation_default_charset);
 461     // create thread object the first time we get an incoming connection
 462     if (!m_my_thread)
 463         m_my_thread = new Msg_Thread(m_socket_observable, 1);
 464     new_proxy->m_my_thread = m_my_thread;
 465     return new_proxy;
 466 }
 467
 468 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 469 {
 470     int oid[OID_SIZE];
 471     Z_OtherInformationUnit *oi;
 472     struct oident ent;
 473     ent.proto = PROTO_Z3950;
 474     ent.oclass = CLASS_USERINFO;
 475     ent.value = (oid_value) VAL_COOKIE;
 476     assert (oid_ent_to_oid (&ent, oid));
 477
 478     if (oid_ent_to_oid (&ent, oid) &&
 479         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 480         oi->which == Z_OtherInfo_characterInfo)
 481         return oi->information.characterInfo;
 482     return 0;
 483 }
 484 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 485 {
 486     int oid[OID_SIZE];
 487     Z_OtherInformationUnit *oi;
 488     struct oident ent;
 489     ent.proto = PROTO_Z3950;
 490     ent.oclass = CLASS_USERINFO;
 491     ent.value = (oid_value) VAL_PROXY;
 492     if (oid_ent_to_oid (&ent, oid) &&
 493         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 494         oi->which == Z_OtherInfo_characterInfo)
 495         return oi->information.characterInfo;
 496     return 0;
 497 }
 498 const char *Yaz_Proxy::load_balance(const char **url)
 499 {
 500     int zurl_in_use[MAX_ZURL_PLEX];
 501     int zurl_in_spare[MAX_ZURL_PLEX];
 502     Yaz_ProxyClient *c;
 503     int i;
 504
 505     for (i = 0; i<MAX_ZURL_PLEX; i++)
 506     {
 507         zurl_in_use[i] = 0;
 508         zurl_in_spare[i] = 0;
 509     }
 510     for (c = m_parent->m_clientPool; c; c = c->m_next)
 511     {
 512         for (i = 0; url[i]; i++)
 513             if (!strcmp(url[i], c->get_hostname()))
 514             {
 515                 zurl_in_use[i]++;
 516                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 517                     zurl_in_spare[i]++;
 518             }
 519     }
 520     int min_use = 100000;
 521     int spare_for_min = 0;
 522     int max_spare = 0;
 523     const char *ret_min = 0;
 524     const char *ret_spare = 0;
 525     for (i = 0; url[i]; i++)
 526     {
 527         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 528                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 529         if (min_use > zurl_in_use[i])
 530         {
 531             ret_min = url[i];
 532             min_use = zurl_in_use[i];
 533             spare_for_min = zurl_in_spare[i];
 534         }
 535         if (max_spare < zurl_in_spare[i])
 536         {
 537             ret_spare = url[i];
 538             max_spare = zurl_in_spare[i];
 539         }
 540     }
 541     return ret_min;
 542 }
 543
 544 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 545                                        const char *proxy_host)
 546 {
 547     assert (m_parent);
 548     Yaz_Proxy *parent = m_parent;
 549     Yaz_ProxyClient *c = m_client;
 550
 551     if (!m_proxyTarget)
 552     {
 553         const char *url[MAX_ZURL_PLEX];
 554         Yaz_ProxyConfig *cfg = check_reconfigure();
 555         if (proxy_host)
 556         {
 557             if (parent && parent->m_debug_mode)
 558             {
 559                 // only to be enabled for debugging...
 560                 if (!strcmp(proxy_host, "stop"))
 561                     exit(0);
 562             }
 563             xfree(m_default_target);
 564             m_default_target = xstrdup(proxy_host);
 565         }
 566         proxy_host = m_default_target;
 567         int client_idletime = -1;
 568         const char *cql2rpn_fname = 0;
 569         const char *negotiation_charset = 0;
 570         const char *negotiation_lang = 0;
 571         const char *query_charset = 0;
 572         const char *default_client_query_charset = 0;
 573         url[0] = m_default_target;
 574         url[1] = 0;
 575         if (cfg)
 576         {
 577             int pre_init = 0;
 578             cfg->get_target_info(proxy_host, url, &m_bw_max,
 579                                  &m_pdu_max, &m_max_record_retrieve,
 580                                  &m_search_max,
 581                                  &m_target_idletime, &client_idletime,
 582                                  &parent->m_max_clients,
 583                                  &m_keepalive_limit_bw,
 584                                  &m_keepalive_limit_pdu,
 585                                  &pre_init,
 586                                  &cql2rpn_fname,
 587                                  &negotiation_charset,
 588                                  &negotiation_lang,
 589                                  &query_charset,
 590                                  &default_client_query_charset);
 591         }
 592         if (client_idletime != -1)
 593         {
 594             m_client_idletime = client_idletime;
 595             timeout(m_client_idletime);
 596         }
 597         if (cql2rpn_fname)
 598             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 599         if (negotiation_charset || negotiation_lang || default_client_query_charset)
 600         {
 601             set_proxy_negotiation(negotiation_charset,
 602                 negotiation_lang, default_client_query_charset);
 603         }
 604         m_charset_converter->set_target_query_charset(query_charset);
 605         if (!url[0])
 606         {
 607             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 608             return 0;
 609         }
 610         // we don't handle multiplexing for cookie session, so we just
 611         // pick the first one in this case (anonymous users will be able
 612         // to use any backend)
 613         if (cookie && *cookie)
 614             m_proxyTarget = (char*) xstrdup(url[0]);
 615         else
 616             m_proxyTarget = (char*) xstrdup(load_balance(url));
 617     }
 618     if (cookie && *cookie)
 619     {   // search in sessions with a cookie
 620         for (c = parent->m_clientPool; c; c = c->m_next)
 621         {
 622             assert (c->m_prev);
 623             assert (*c->m_prev == c);
 624             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 625                 !strcmp(m_proxyTarget, c->get_hostname()))
 626             {
 627                 // Found it in cache
 628                 // The following handles "cancel"
 629                 // If connection is busy (waiting for PDU) and
 630                 // we have an initRequest we can safely do re-open
 631                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 632                 {
 633                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 634                              c->get_hostname());
 635                     c->close();
 636                     c->m_init_flag = 0;
 637
 638                     c->m_last_ok = 0;
 639                     c->m_cache.clear();
 640                     c->m_last_resultCount = 0;
 641                     c->m_sr_transform = 0;
 642                     c->m_waiting = 0;
 643                     c->m_resultSetStartPoint = 0;
 644                     c->m_target_idletime = m_target_idletime;
 645                     if (c->client(m_proxyTarget))
 646                     {
 647                         delete c;
 648                         return 0;
 649                     }
 650                     c->timeout(30);
 651                 }
 652                 c->m_seqno = parent->m_seqno;
 653                 if (c->m_server && c->m_server != this)
 654                     c->m_server->m_client = 0;
 655                 c->m_server = this;
 656                 (parent->m_seqno)++;
 657                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 658                 return c;
 659             }
 660         }
 661     }
 662     else if (!c && apdu->which == Z_APDU_initRequest )
 663     {
 664         // anonymous sessions without cookie.
 665         // if authentication is set it is NOT anonymous se we can't share them.
 666         // If charset and lang negotiation is use it is NOT anonymous session too.
 667         for (c = parent->m_clientPool; c; c = c->m_next)
 668         {
 669             assert(c->m_prev);
 670             assert(*c->m_prev == c);
 671             if (c->m_server == 0 && c->m_cookie == 0 &&  c->m_waiting == 0
 672                 && c->compare_idAuthentication(apdu)
 673                 && c->compare_charset(apdu)
 674                 && !strcmp(m_proxyTarget, c->get_hostname()))
 675             {
 676                 // found it in cache
 677                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 678                          m_session_str, parent->m_seqno, c->get_hostname());
 679
 680                 c->m_seqno = parent->m_seqno;
 681                 assert(c->m_server == 0);
 682                 c->m_server = this;
 683
 684                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 685                     c->set_APDU_yazlog(1);
 686                 else
 687                     c->set_APDU_yazlog(0);
 688
 689                 (parent->m_seqno)++;
 690
 691                 parent->pre_init();
 692
 693                 return c;
 694             }
 695         }
 696     }
 697     if (!m_client)
 698     {
 699         if (apdu->which != Z_APDU_initRequest)
 700         {
 701             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 702             return 0;
 703         }
 704         // go through list of clients - and find the lowest/oldest one.
 705         Yaz_ProxyClient *c_min = 0;
 706         int min_seq = -1;
 707         int no_of_clients = 0;
 708         if (parent->m_clientPool)
 709             yaz_log (YLOG_DEBUG, "Existing sessions");
 710         for (c = parent->m_clientPool; c; c = c->m_next)
 711         {
 712             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 713                                c->m_waiting, c->get_hostname(),
 714                                c->m_cookie ? c->m_cookie : "");
 715             no_of_clients++;
 716             if (min_seq < 0 || c->m_seqno < min_seq)
 717             {
 718                 min_seq = c->m_seqno;
 719                 c_min = c;
 720             }
 721         }
 722         if (no_of_clients >= parent->m_max_clients)
 723         {
 724             c = c_min;
 725             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 726             {
 727                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 728                          m_session_str, parent->m_max_clients, c->m_seqno);
 729                 if (c->m_server && c->m_server != this)
 730                     delete c->m_server;   // PROBLEM: m_ref_count!
 731                 c->m_server = 0;
 732             }
 733             else
 734             {
 735                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 736                          m_session_str, parent->m_max_clients,
 737                          c->m_seqno, parent->m_seqno, c->get_hostname());
 738                 xfree (c->m_cookie);
 739                 c->m_cookie = 0;
 740                 if (cookie)
 741                     c->m_cookie = xstrdup(cookie);
 742                 c->m_seqno = parent->m_seqno;
 743                 if (c->m_server && c->m_server != this)
 744                 {
 745                     c->m_server->m_client = 0;
 746                     delete c->m_server;   // PROBLEM: m_ref_count!
 747                 }
 748                 (parent->m_seqno)++;
 749                 c->m_target_idletime = m_target_idletime;
 750                 c->timeout(m_target_idletime);
 751
 752                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 753                     c->set_APDU_yazlog(1);
 754                 else
 755                     c->set_APDU_yazlog(0);
 756
 757                 return c;
 758             }
 759         }
 760         else
 761         {
 762             yaz_log (YLOG_LOG, "%sNEW %d %s",
 763                      m_session_str, parent->m_seqno, m_proxyTarget);
 764             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 765             c->m_next = parent->m_clientPool;
 766             if (c->m_next)
 767                 c->m_next->m_prev = &c->m_next;
 768             parent->m_clientPool = c;
 769             c->m_prev = &parent->m_clientPool;
 770         }
 771
 772         xfree (c->m_cookie);
 773         c->m_cookie = 0;
 774         if (cookie)
 775             c->m_cookie = xstrdup(cookie);
 776
 777         c->m_seqno = parent->m_seqno;
 778         c->m_init_flag = 0;
 779         c->m_last_resultCount = 0;
 780         c->m_last_ok = 0;
 781         c->m_cache.clear();
 782         c->m_sr_transform = 0;
 783         c->m_waiting = 0;
 784         c->m_resultSetStartPoint = 0;
 785         (parent->m_seqno)++;
 786         if (c->client(m_proxyTarget))
 787         {
 788             delete c;
 789             return 0;
 790         }
 791         c->m_target_idletime = m_target_idletime;
 792         c->timeout(30);
 793
 794         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 795             c->set_APDU_yazlog(1);
 796         else
 797             c->set_APDU_yazlog(0);
 798
 799         c->set_idAuthentication(apdu);
 800     }
 801     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 802     return c;
 803 }
 804
 805 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 806 {
 807     int i;
 808     for (i = 0; i<num; i++)
 809     {
 810         oident *ent;
 811         Z_DefaultDiagFormat *r;
 812         Z_DiagRec *p = pp[i];
 813         if (p->which != Z_DiagRec_defaultFormat)
 814         {
 815             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 816             return;
 817         }
 818         else
 819             r = p->u.defaultFormat;
 820         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 821             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 822             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 823         switch (r->which)
 824         {
 825         case Z_DefaultDiagFormat_v2Addinfo:
 826             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 827                     m_session_str,
 828                     *r->condition, diagbib1_str(*r->condition),
 829                     r->u.v2Addinfo);
 830             break;
 831         case Z_DefaultDiagFormat_v3Addinfo:
 832             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 833                     m_session_str,
 834                     *r->condition, diagbib1_str(*r->condition),
 835                     r->u.v3Addinfo);
 836             break;
 837         }
 838     }
 839 }
 840
 841 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 842 {
 843     if (!m_stylesheet_xsp || p->num_records <= 0)
 844     {
 845         return 0;  /* no XSLT to be done ... */
 846     }
 847
 848     m_stylesheet_offset = 0;
 849     m_stylesheet_nprl = p;
 850     m_stylesheet_apdu = apdu;
 851     m_timeout_mode = timeout_xsl;
 852
 853     timeout(0);
 854     return 1;
 855 }
 856
 857 void Yaz_Proxy::convert_xsl_delay()
 858 {
 859 #if HAVE_XSLT
 860     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 861     if (npr->which == Z_NamePlusRecord_databaseRecord)
 862     {
 863         Z_External *r = npr->u.databaseRecord;
 864         if (r->which == Z_External_octet)
 865         {
 866 #if 0
 867             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 868 #endif
 869             xmlDocPtr res, doc = xmlParseMemory(
 870                 (char*) r->u.octet_aligned->buf,
 871                 r->u.octet_aligned->len);
 872
 873
 874             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 875                     m_session_str, m_stylesheet_offset);
 876             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 877                                       doc, 0);
 878
 879             if (res)
 880             {
 881                 xmlChar *out_buf;
 882                 int out_len;
 883                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 884
 885                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 886                     u.databaseRecord =
 887                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 888                                  (char*) out_buf, out_len);
 889                 xmlFree(out_buf);
 890                 xmlFreeDoc(res);
 891             }
 892
 893             xmlFreeDoc(doc);
 894         }
 895     }
 896 #endif
 897     m_stylesheet_offset++;
 898     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 899     {
 900         m_timeout_mode = timeout_normal;
 901         m_stylesheet_nprl = 0;
 902 #if HAVE_XSLT
 903         if (m_stylesheet_xsp)
 904             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 905 #endif
 906         m_stylesheet_xsp = 0;
 907         timeout(m_client_idletime);
 908         send_PDU_convert(m_stylesheet_apdu);
 909     }
 910     else
 911         timeout(0);
 912 }
 913
 914 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 915 {
 916     if (m_frontend_type != VAL_NONE)
 917     {
 918         int i;
 919         for (i = 0; i < p->num_records; i++)
 920         {
 921             Z_NamePlusRecord *npr = p->records[i];
 922             if (npr->which == Z_NamePlusRecord_databaseRecord)
 923             {
 924                 Z_External *r = npr->u.databaseRecord;
 925                 if (r->which == Z_External_octet)
 926                 {
 927 #if HAVE_USEMARCON
 928                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 929                     {
 930                         if (!m_usemarcon->m_stage1)
 931                         {
 932                             m_usemarcon->m_stage1 = new CDetails();
 933                         }
 934                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 935                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 936                         int res = m_usemarcon->m_stage1->Start();
 937                         if (res == 0)
 938                         {
 939                             char *converted;
 940                             int convlen;
 941                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 942                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 943                             {
 944                                 if (!m_usemarcon->m_stage2)
 945                                 {
 946                                     m_usemarcon->m_stage2 = new CDetails();
 947                                 }
 948                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 949                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 950                                 res = m_usemarcon->m_stage2->Start();
 951                                 if (res == 0)
 952                                 {
 953                                     free(converted);
 954                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 955                                 }
 956                                 else
 957                                 {
 958                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 959                                 }
 960                             }
 961                             npr->u.databaseRecord =
 962                                 z_ext_record(odr_encode(),
 963                                              m_frontend_type,
 964                                              converted,
 965                                              strlen(converted));
 966                             free(converted);
 967                         }
 968                         else
 969                         {
 970                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 971                         }
 972                         continue;
 973                     }
 974 #endif
 975 /* HAVE_USEMARCON */
 976                     npr->u.databaseRecord =
 977                         z_ext_record(odr_encode(),
 978                                      m_frontend_type,
 979                                      (char*) r->u.octet_aligned->buf,
 980                                      r->u.octet_aligned->len);
 981                 }
 982             }
 983         }
 984     }
 985 }
 986
 987 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
 988                                         const char *backend_charset)
 989 {
 990     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
 991     int sel =   m_charset_converter->get_client_charset_selected();
 992     const char *client_record_charset =
 993         m_charset_converter->get_client_query_charset();
 994     if (sel && backend_charset && client_record_charset &&
 995         strcmp(backend_charset, client_record_charset))
 996     {
 997         int i;
 998         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
 999                                         backend_charset);
1000         yaz_marc_t mt = yaz_marc_create();
1001         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
1002         yaz_marc_iconv(mt, cd);
1003         for (i = 0; i < p->num_records; i++)
1004         {
1005             Z_NamePlusRecord *npr = p->records[i];
1006             if (npr->which == Z_NamePlusRecord_databaseRecord)
1007             {
1008                 Z_External *r = npr->u.databaseRecord;
1009                 oident *ent = oid_getentbyoid(r->direct_reference);
1010                 if (!ent || ent->value == VAL_NONE)
1011                     continue;
1012
1013                 if (ent->value == VAL_SUTRS)
1014                 {
1015                     WRBUF w = wrbuf_alloc();
1016
1017                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
1018                                       r->u.octet_aligned->len);
1019                     npr->u.databaseRecord =
1020                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
1021                                      wrbuf_len(w));
1022                     wrbuf_free(w, 1);
1023                 }
1024                 else if (ent->value == VAL_TEXT_XML)
1025                 {
1026                     ;
1027                 }
1028                 else if (r->which == Z_External_octet)
1029                 {
1030                     int rlen;
1031                     char *result;
1032                     if (yaz_marc_decode_buf(mt,
1033                                             (char*) r->u.octet_aligned->buf,
1034                                             r->u.octet_aligned->len,
1035                                             &result, &rlen))
1036                     {
1037                         npr->u.databaseRecord =
1038                             z_ext_record(odr_encode(), ent->value, result, rlen);
1039                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1040                                 m_session_str);
1041                     }
1042                 }
1043             }
1044         }
1045         if (cd)
1046             yaz_iconv_close(cd);
1047         yaz_marc_destroy(mt);
1048     }
1049     else
1050     {
1051         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
1052     }
1053 }
1054
1055 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1056                                    const char *backend_charset)
1057 {
1058     int i;
1059     if (!backend_charset)
1060         backend_charset = "MARC-8";
1061     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1062     yaz_marc_t mt = yaz_marc_create();
1063     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1064     yaz_marc_iconv(mt, cd);
1065     for (i = 0; i < p->num_records; i++)
1066     {
1067         Z_NamePlusRecord *npr = p->records[i];
1068         if (npr->which == Z_NamePlusRecord_databaseRecord)
1069         {
1070             Z_External *r = npr->u.databaseRecord;
1071             if (r->which == Z_External_OPAC)
1072             {
1073                 WRBUF w = wrbuf_alloc();
1074
1075                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
1076                 npr->u.databaseRecord = z_ext_record(
1077                     odr_encode(), VAL_TEXT_XML,
1078                     wrbuf_buf(w), wrbuf_len(w)
1079                     );
1080                 wrbuf_free(w, 1);
1081             }
1082             else if (r->which == Z_External_octet)
1083             {
1084                 int rlen;
1085                 char *result;
1086                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1087                                         r->u.octet_aligned->len,
1088                                         &result, &rlen))
1089                 {
1090                     npr->u.databaseRecord =
1091                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
1092                 }
1093             }
1094         }
1095     }
1096     if (cd)
1097         yaz_iconv_close(cd);
1098     yaz_marc_destroy(mt);
1099 }
1100
1101 void Yaz_Proxy::logtime()
1102 {
1103 #if HAVE_GETTIMEOFDAY
1104     struct timeval *tv = (struct timeval*) m_time_tv;
1105     if (tv->tv_sec)
1106     {
1107         struct timeval tv1;
1108         gettimeofday(&tv1, 0);
1109         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1110             (tv1.tv_usec - tv->tv_usec);
1111         if (diff >= 0)
1112             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1113                     diff/1000000, (diff/1000)%1000);
1114     }
1115     tv->tv_sec = 0;
1116     tv->tv_usec = 0;
1117 #endif
1118 }
1119
1120 int Yaz_Proxy::send_http_response(int code)
1121 {
1122     ODR o = odr_encode();
1123     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1124     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1125     if (m_http_version)
1126         hres->version = odr_strdup(o, m_http_version);
1127     if (m_http_keepalive)
1128         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1129     else
1130         timeout(0);
1131
1132     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1133     {
1134         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1135                  gdu_name(gdu));
1136     }
1137     int len;
1138     int r = send_GDU(gdu, &len);
1139     m_bytes_sent += len;
1140     m_bw_stat.add_bytes(len);
1141     logtime();
1142
1143     recv_GDU_more(true);
1144
1145     return r;
1146 }
1147
1148 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */)
1149 {
1150     ODR o = odr_encode();
1151     const char *ctype = "text/xml";
1152     Z_GDU *gdu = z_get_HTTP_Response(o, http_code);
1153     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1154     if (m_http_version)
1155         hres->version = odr_strdup(o, m_http_version);
1156     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1157     if (m_http_keepalive)
1158         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1159     else
1160         timeout(0);
1161     if (http_code == 401)
1162         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\"");
1163
1164     static Z_SOAP_Handler soap_handlers[2] = {
1165 #if HAVE_XSLT
1166         {"http://www.loc.gov/zing/srw/", 0,
1167          (Z_SOAP_fun) yaz_srw_codec},
1168 #endif
1169         {0, 0, 0}
1170     };
1171
1172     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1173     soap_package->which = Z_SOAP_generic;
1174     soap_package->u.generic =
1175         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1176     soap_package->u.generic->no = 0;
1177     soap_package->u.generic->ns = soap_handlers[0].ns;
1178     soap_package->u.generic->p = (void *) srw_pdu;
1179     soap_package->ns = m_soap_ns;
1180     z_soap_codec_enc_xsl(o, &soap_package,
1181                          &hres->content_buf, &hres->content_len,
1182                          soap_handlers, 0, m_s2z_stylesheet);
1183     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1184     {
1185         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1186                  gdu_name(gdu));
1187     }
1188     int len;
1189     int r = send_GDU(gdu, &len);
1190     m_bytes_sent += len;
1191     m_bw_stat.add_bytes(len);
1192     logtime();
1193
1194     recv_GDU_more(true);
1195
1196     return r;
1197 }
1198
1199 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1200 {
1201     ODR o = odr_encode();
1202     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1203         odr_malloc(o, sizeof(*diagnostic));
1204     int num_diagnostic = 1;
1205     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1206     return send_srw_search_response(diagnostic, num_diagnostic, srw_error == 3 ? 401 : 200);
1207 }
1208
1209 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1210                              Z_DefaultDiagFormat *ddf)
1211 {
1212     int bib1_code = *ddf->condition;
1213     if (bib1_code == 109)
1214         return 404;
1215     srw_res->num_diagnostics = 1;
1216     srw_res->diagnostics = (Z_SRW_diagnostic *)
1217         odr_malloc(o, sizeof(*srw_res->diagnostics));
1218     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1219                           yaz_diag_bib1_to_srw(*ddf->condition),
1220                           ddf->u.v2Addinfo);
1221     return 0;
1222 }
1223
1224 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1225 {
1226     ODR o = odr_encode();
1227     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1228     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1229
1230     srw_res->numberOfRecords = odr_intdup (o, hits);
1231     if (records && records->which == Z_Records_DBOSD)
1232     {
1233         srw_res->num_records =
1234             records->u.databaseOrSurDiagnostics->num_records;
1235         int i;
1236         srw_res->records = (Z_SRW_record *)
1237             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1238         for (i = 0; i < srw_res->num_records; i++)
1239         {
1240             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1241             if (npr->which != Z_NamePlusRecord_databaseRecord)
1242             {
1243                 srw_res->records[i].recordSchema = "diagnostic";
1244                 srw_res->records[i].recordPacking = m_s2z_packing;
1245                 srw_res->records[i].recordData_buf = "67";
1246                 srw_res->records[i].recordData_len = 2;
1247                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1248                 continue;
1249             }
1250             Z_External *r = npr->u.databaseRecord;
1251             oident *ent = oid_getentbyoid(r->direct_reference);
1252             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1253             {
1254                 srw_res->records[i].recordSchema = m_schema;
1255                 srw_res->records[i].recordPacking = m_s2z_packing;
1256                 srw_res->records[i].recordData_buf = (char*)
1257                     r->u.octet_aligned->buf;
1258                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1259                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1260             }
1261             else
1262             {
1263                 srw_res->records[i].recordSchema = "diagnostic";
1264                 srw_res->records[i].recordPacking = m_s2z_packing;
1265                 srw_res->records[i].recordData_buf = "67";
1266                 srw_res->records[i].recordData_len = 2;
1267                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1268             }
1269         }
1270     }
1271     if (records && records->which == Z_Records_NSD)
1272     {
1273         int http_code;
1274         http_code = z_to_srw_diag(odr_encode(), srw_res,
1275                                    records->u.nonSurrogateDiagnostic);
1276         if (http_code)
1277             return send_http_response(http_code);
1278     }
1279     return send_srw_response(srw_pdu);
1280
1281 }
1282
1283 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1284                                         int num_diagnostics, int http_code /* = 200 */)
1285 {
1286     ODR o = odr_encode();
1287     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1288     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1289
1290     srw_res->num_diagnostics = num_diagnostics;
1291     srw_res->diagnostics = diagnostics;
1292     return send_srw_response(srw_pdu, http_code);
1293 }
1294
1295 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1296                                         int num_diagnostics)
1297 {
1298     Yaz_ProxyConfig *cfg = check_reconfigure();
1299     if (cfg)
1300     {
1301         int len;
1302         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1303                                        m_s2z_database, &len);
1304         if (b)
1305         {
1306             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1307             Z_SRW_explainResponse *er = res->u.explain_response;
1308
1309             er->record.recordData_buf = b;
1310             er->record.recordData_len = len;
1311             er->record.recordPacking = m_s2z_packing;
1312             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1313
1314             er->diagnostics = diagnostics;
1315             er->num_diagnostics = num_diagnostics;
1316             return send_srw_response(res);
1317         }
1318     }
1319     return send_http_response(404);
1320 }
1321
1322 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1323 {
1324     if (m_http_version)
1325     {
1326         if (apdu->which == Z_APDU_initResponse)
1327         {
1328             Z_InitResponse *res = apdu->u.initResponse;
1329             if (*res->result == 0)
1330             {
1331                 send_to_srw_client_error(3, 0);
1332             }
1333             else if (!m_s2z_search_apdu)
1334             {
1335                 send_srw_explain_response(0, 0);
1336             }
1337             else
1338             {
1339                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1340             }
1341         }
1342         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1343         {
1344             m_s2z_search_apdu = 0;
1345             Z_SearchResponse *res = apdu->u.searchResponse;
1346             m_s2z_hit_count = *res->resultCount;
1347             if (res->records && res->records->which == Z_Records_NSD)
1348             {
1349                 send_to_srw_client_ok(0, res->records, 1);
1350             }
1351             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1352             {
1353                 // adjust
1354                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1355
1356                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1357                 {
1358                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1359                         > m_s2z_hit_count)
1360                         *pr->numberOfRecordsRequested =
1361                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1362                 }
1363                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1364             }
1365             else
1366             {
1367                 m_s2z_present_apdu = 0;
1368                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1369             }
1370         }
1371         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1372         {
1373             int start =
1374                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1375
1376             m_s2z_present_apdu = 0;
1377             Z_PresentResponse *res = apdu->u.presentResponse;
1378             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1379         }
1380     }
1381     else
1382     {
1383         int len = 0;
1384         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1385             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1386                      apdu_name(apdu));
1387         int r = send_Z_PDU(apdu, &len);
1388         m_bytes_sent += len;
1389         m_bw_stat.add_bytes(len);
1390         logtime();
1391         return r;
1392     }
1393     return 0;
1394 }
1395
1396 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1397 {
1398     int kill_session = 0;
1399     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1400
1401     if (new_id)
1402         *new_id = m_referenceId;
1403
1404     if (apdu->which == Z_APDU_searchResponse)
1405     {
1406         Z_SearchResponse *sr = apdu->u.searchResponse;
1407         Z_Records *p = sr->records;
1408         if (p && p->which == Z_Records_NSD)
1409         {
1410             Z_DiagRec dr, *dr_p = &dr;
1411             dr.which = Z_DiagRec_defaultFormat;
1412             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1413
1414             *sr->searchStatus = 0;
1415             display_diagrecs(&dr_p, 1);
1416         }
1417         else
1418         {
1419             if (p && p->which == Z_Records_DBOSD)
1420             {
1421                 if (m_backend_type
1422 #if HAVE_USEMARCON
1423                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1424 #endif
1425                     )
1426                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1427                 if (m_marcxml_mode == marcxml)
1428                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1429                                        m_backend_charset);
1430                 else
1431                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1432                                             m_backend_charset);
1433                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1434                     return 0;
1435
1436             }
1437             if (sr->resultCount)
1438             {
1439                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1440                         *sr->resultCount);
1441                 if (*sr->resultCount < 0)
1442                 {
1443                     m_flag_invalid_session = 1;
1444                     kill_session = 1;
1445
1446                     *sr->searchStatus = 0;
1447                     sr->records =
1448                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1449                     *sr->resultCount = 0;
1450                 }
1451             }
1452         }
1453     }
1454     else if (apdu->which == Z_APDU_presentResponse)
1455     {
1456         Z_PresentResponse *sr = apdu->u.presentResponse;
1457         Z_Records *p = sr->records;
1458         if (p && p->which == Z_Records_NSD)
1459         {
1460             Z_DiagRec dr, *dr_p = &dr;
1461             dr.which = Z_DiagRec_defaultFormat;
1462             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1463             if (*sr->presentStatus == Z_PresentStatus_success)
1464                 *sr->presentStatus = Z_PresentStatus_failure;
1465             display_diagrecs(&dr_p, 1);
1466         }
1467         if (p && p->which == Z_Records_DBOSD)
1468         {
1469             if (m_backend_type
1470 #if HAVE_USEMARCON
1471                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1472 #endif
1473                 )
1474                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1475             if (m_marcxml_mode == marcxml)
1476                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1477                                    m_backend_charset);
1478             else
1479                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1480                                         m_backend_charset);
1481             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1482                 return 0;
1483         }
1484     }
1485     else if (apdu->which == Z_APDU_initResponse)
1486     {
1487         //Get and check negotiation record
1488         //from init response.
1489         handle_charset_lang_negotiation(apdu);
1490
1491         if (m_initRequest_options)
1492         {
1493             Z_Options *nopt =
1494                 (Odr_bitmask *)odr_malloc(odr_encode(),
1495                                           sizeof(Odr_bitmask));
1496             ODR_MASK_ZERO(nopt);
1497
1498             int i;
1499             for (i = 0; i<24; i++)
1500                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1501                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1502                     ODR_MASK_SET(nopt, i);
1503             apdu->u.initResponse->options = nopt;
1504         }
1505         if (m_initRequest_version)
1506         {
1507             Z_ProtocolVersion *nopt =
1508                 (Odr_bitmask *)odr_malloc(odr_encode(),
1509                                           sizeof(Odr_bitmask));
1510             ODR_MASK_ZERO(nopt);
1511
1512             int i;
1513             for (i = 0; i<8; i++)
1514                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1515                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1516                     ODR_MASK_SET(nopt, i);
1517             apdu->u.initResponse->protocolVersion = nopt;
1518         }
1519         apdu->u.initResponse->preferredMessageSize =
1520             odr_intdup(odr_encode(),
1521                        m_client->m_initResponse_preferredMessageSize >
1522                        m_initRequest_preferredMessageSize ?
1523                        m_initRequest_preferredMessageSize :
1524                        m_client->m_initResponse_preferredMessageSize);
1525         apdu->u.initResponse->maximumRecordSize =
1526             odr_intdup(odr_encode(),
1527                        m_client->m_initResponse_maximumRecordSize >
1528                        m_initRequest_maximumRecordSize ?
1529                        m_initRequest_maximumRecordSize :
1530                        m_client->m_initResponse_maximumRecordSize);
1531     }
1532
1533     int r = send_PDU_convert(apdu);
1534     if (r)
1535         return r;
1536     if (kill_session)
1537     {
1538         delete m_client;
1539         m_client = 0;
1540         m_parent->pre_init();
1541     }
1542     return r;
1543 }
1544
1545 void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu)
1546 {
1547     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1548
1549     odr_reset(m_idAuthentication_odr);
1550     z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0);
1551     m_idAuthentication_ber_buf =
1552         odr_getbuf(m_idAuthentication_odr,
1553                    &m_idAuthentication_ber_size, 0);
1554 }
1555
1556 bool Yaz_ProxyClient::compare_charset(Z_APDU *apdu)
1557 {
1558     return true;
1559 }
1560
1561 bool Yaz_ProxyClient::compare_idAuthentication(Z_APDU *apdu)
1562 {
1563     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1564     ODR odr = odr_createmem(ODR_ENCODE);
1565
1566     z_IdAuthentication(odr, &t, 1, 0);
1567     int sz;
1568     char *buf = odr_getbuf(odr, &sz, 0);
1569     if (buf && m_idAuthentication_ber_buf
1570         && sz == m_idAuthentication_ber_size
1571         && !memcmp(m_idAuthentication_ber_buf, buf, sz))
1572     {
1573         odr_destroy(odr);
1574         return true;
1575     }
1576     odr_destroy(odr);
1577     if (!buf && !m_idAuthentication_ber_buf)
1578         return true;
1579     return false;
1580 }
1581
1582 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1583 {
1584     int len = 0;
1585     const char *apdu_name_tmp = apdu_name(apdu);
1586     int r = send_Z_PDU(apdu, &len);
1587     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1588         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1589                  get_session_str(),
1590                  apdu_name_tmp, get_hostname(), len);
1591     m_bytes_sent += len;
1592     return r;
1593 }
1594
1595 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1596 {
1597     if (apdu->which == Z_APDU_presentRequest)
1598     {
1599         Z_PresentRequest *pr = apdu->u.presentRequest;
1600         int toget = *pr->numberOfRecordsRequested;
1601         int start = *pr->resultSetStartPoint;
1602
1603         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1604                 pr->resultSetId, start, toget);
1605
1606         if (*m_parent->m_optimize == '0')
1607             return apdu;
1608
1609         if (!m_client->m_last_resultSetId)
1610         {
1611             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1612             new_apdu->u.presentResponse->records =
1613                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1614                                                pr->resultSetId);
1615             send_to_client(new_apdu);
1616             return 0;
1617         }
1618         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1619         {
1620             if (start+toget-1 > m_client->m_last_resultCount)
1621             {
1622                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1623                 new_apdu->u.presentResponse->records =
1624                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1625                 send_to_client(new_apdu);
1626                 return 0;
1627             }
1628             Z_NamePlusRecordList *npr;
1629 #if 0
1630             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1631                     m_session_str, start, toget, yaz_z3950oid_to_str(
1632                         pr->preferredRecordSyntax, &oclass));
1633 #endif
1634             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1635                                           pr->preferredRecordSyntax,
1636                                           pr->recordComposition))
1637             {
1638                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1639                          m_session_str);
1640                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1641                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1642
1643                 new_apdu->u.presentResponse->numberOfRecordsReturned
1644                     = odr_intdup(odr_encode(), toget);
1645
1646                 new_apdu->u.presentResponse->records = (Z_Records*)
1647                     odr_malloc(odr_encode(), sizeof(Z_Records));
1648                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1649                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1650                 new_apdu->u.presentResponse->nextResultSetPosition =
1651                     odr_intdup(odr_encode(), start+toget);
1652
1653                 send_to_client(new_apdu);
1654                 return 0;
1655             }
1656         }
1657     }
1658
1659     if (apdu->which != Z_APDU_searchRequest)
1660         return apdu;
1661     Z_SearchRequest *sr = apdu->u.searchRequest;
1662     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1663     Yaz_Z_Databases this_databases;
1664
1665     this_databases.set(sr->num_databaseNames, (const char **)
1666                        sr->databaseNames);
1667
1668     this_query->set_Z_Query(sr->query);
1669
1670     char query_str[120];
1671     this_query->print(query_str, sizeof(query_str)-1);
1672     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1673
1674     if (*m_parent->m_optimize != '0' &&
1675         m_client->m_last_ok && m_client->m_last_query &&
1676         m_client->m_last_query->match(this_query) &&
1677         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1678         m_client->m_last_databases.match(this_databases))
1679     {
1680         delete this_query;
1681         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1682             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1683         {
1684             Z_NamePlusRecordList *npr;
1685             int toget = *sr->mediumSetPresentNumber;
1686             Z_RecordComposition *comp = 0;
1687
1688             if (toget > m_client->m_last_resultCount)
1689                 toget = m_client->m_last_resultCount;
1690
1691             if (sr->mediumSetElementSetNames)
1692             {
1693                 comp = (Z_RecordComposition *)
1694                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1695                 comp->which = Z_RecordComp_simple;
1696                 comp->u.simple = sr->mediumSetElementSetNames;
1697             }
1698
1699             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1700                                           sr->preferredRecordSyntax, comp))
1701             {
1702                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1703                          m_session_str);
1704                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1705                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1706                 new_apdu->u.searchResponse->resultCount =
1707                     &m_client->m_last_resultCount;
1708
1709                 new_apdu->u.searchResponse->numberOfRecordsReturned
1710                     = odr_intdup(odr_encode(), toget);
1711
1712                 new_apdu->u.searchResponse->presentStatus =
1713                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1714                 new_apdu->u.searchResponse->records = (Z_Records*)
1715                     odr_malloc(odr_encode(), sizeof(Z_Records));
1716                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1717                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1718                 new_apdu->u.searchResponse->nextResultSetPosition =
1719                     odr_intdup(odr_encode(), toget+1);
1720                 send_to_client(new_apdu);
1721                 return 0;
1722             }
1723             else
1724             {
1725                 // medium Set
1726                 // send present request (medium size)
1727                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1728                          m_session_str);
1729
1730                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1731                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1732                 pr->referenceId = sr->referenceId;
1733                 pr->resultSetId = sr->resultSetName;
1734                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1735                 *pr->numberOfRecordsRequested = toget;
1736                 pr->recordComposition = comp;
1737                 m_client->m_sr_transform = 1;
1738                 return new_apdu;
1739             }
1740         }
1741         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1742             m_client->m_last_resultCount <= 0)
1743         {
1744             // large set. Return pseudo-search response immediately
1745             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1746                      m_session_str);
1747             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1748             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1749             new_apdu->u.searchResponse->resultCount =
1750                 &m_client->m_last_resultCount;
1751             send_to_client(new_apdu);
1752             return 0;
1753         }
1754         else
1755         {
1756             Z_NamePlusRecordList *npr;
1757             int toget = m_client->m_last_resultCount;
1758             Z_RecordComposition *comp = 0;
1759             // small set
1760             // send a present request (small set)
1761
1762             if (sr->smallSetElementSetNames)
1763             {
1764                 comp = (Z_RecordComposition *)
1765                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1766                 comp->which = Z_RecordComp_simple;
1767                 comp->u.simple = sr->smallSetElementSetNames;
1768             }
1769
1770             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1771                                           sr->preferredRecordSyntax, comp))
1772             {
1773                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1774                          m_session_str);
1775                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1776                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1777                 new_apdu->u.searchResponse->resultCount =
1778                     &m_client->m_last_resultCount;
1779
1780                 new_apdu->u.searchResponse->numberOfRecordsReturned
1781                     = odr_intdup(odr_encode(), toget);
1782
1783                 new_apdu->u.searchResponse->presentStatus =
1784                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1785                 new_apdu->u.searchResponse->records = (Z_Records*)
1786                     odr_malloc(odr_encode(), sizeof(Z_Records));
1787                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1788                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1789                 new_apdu->u.searchResponse->nextResultSetPosition =
1790                     odr_intdup(odr_encode(), toget+1);
1791                 send_to_client(new_apdu);
1792                 return 0;
1793             }
1794             else
1795             {
1796                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1797                          m_session_str);
1798                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1799                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1800                 pr->referenceId = sr->referenceId;
1801                 pr->resultSetId = sr->resultSetName;
1802                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1803                 *pr->numberOfRecordsRequested = toget;
1804                 pr->recordComposition = comp;
1805                 m_client->m_sr_transform = 1;
1806                 return new_apdu;
1807             }
1808         }
1809     }
1810     else  // query doesn't match
1811     {
1812         delete m_client->m_last_query;
1813         m_client->m_last_query = this_query;
1814         m_client->m_last_ok = 0;
1815         m_client->m_cache.clear();
1816         m_client->m_resultSetStartPoint = 0;
1817
1818         xfree (m_client->m_last_resultSetId);
1819         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1820
1821         m_client->m_last_databases.set(sr->num_databaseNames,
1822                                        (const char **) sr->databaseNames);
1823     }
1824     return apdu;
1825 }
1826
1827
1828 void Yaz_Proxy::inc_request_no()
1829 {
1830     char *cp = strchr(m_session_str, ' ');
1831     m_request_no++;
1832     if (cp)
1833         sprintf(cp+1, "%d ", m_request_no);
1834 }
1835
1836 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1837 {
1838     inc_request_no();
1839
1840     m_bytes_recv += len;
1841
1842     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1843         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1844                  m_session_str, gdu_name(apdu), len);
1845
1846 #if HAVE_GETTIMEOFDAY
1847     gettimeofday((struct timeval *) m_time_tv, 0);
1848 #endif
1849     m_bw_stat.add_bytes(len);
1850     m_pdu_stat.add_bytes(1);
1851
1852     GDU *gdu = new GDU(apdu);
1853     m_in_queue.enqueue(gdu);
1854
1855     recv_GDU_more(false);
1856 }
1857
1858 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
1859 {
1860     int reduce = 0;
1861
1862     int bw_total = m_bw_stat.get_total();
1863     int pdu_total = m_pdu_stat.get_total();
1864
1865     assert(m_timeout_mode == timeout_busy);
1866     assert(m_timeout_gdu == 0);
1867
1868     if (m_bw_max)
1869     {
1870         if (bw_total > m_bw_max)
1871         {
1872             reduce = (bw_total/m_bw_max);
1873         }
1874     }
1875     if (m_pdu_max)
1876     {
1877         if (pdu_total > m_pdu_max)
1878         {
1879             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1880             reduce = (reduce > nreduce) ? reduce : nreduce;
1881         }
1882     }
1883     m_http_version = 0;
1884
1885 #if 0
1886     /* uncomment to force a big reduce */
1887     m_timeout_mode = timeout_reduce;
1888     m_timeout_gdu = gdu;
1889     timeout(3);       // call us reduce seconds later
1890     return;
1891 #endif
1892     if (reduce)
1893     {
1894         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1895                 m_session_str, reduce, bw_total, pdu_total,
1896                 m_bw_max, m_pdu_max);
1897
1898         m_timeout_mode = timeout_reduce;
1899         m_timeout_gdu = gdu;
1900         timeout(reduce);       // call us reduce seconds later
1901     }
1902     else
1903         recv_GDU_normal(gdu);
1904 }
1905
1906 void Yaz_Proxy::recv_GDU_more(bool normal)
1907 {
1908     GDU *g;
1909     if (normal && m_timeout_mode == timeout_busy)
1910         m_timeout_mode = timeout_normal;
1911     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
1912     {
1913         m_timeout_mode = timeout_busy;
1914         recv_GDU_reduce(g);
1915     }
1916 }
1917
1918 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
1919 {
1920     Z_GDU *apdu = 0;
1921     gdu->move_away_gdu(odr_decode(), &apdu);
1922     delete gdu;
1923
1924     if (apdu->which == Z_GDU_Z3950)
1925         handle_incoming_Z_PDU(apdu->u.z3950);
1926     else if (apdu->which == Z_GDU_HTTP_Request)
1927         handle_incoming_HTTP(apdu->u.HTTP_Request);
1928 }
1929
1930 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1931 {
1932     if (m_max_record_retrieve)
1933     {
1934         if (apdu->which == Z_APDU_presentRequest)
1935         {
1936             Z_PresentRequest *pr = apdu->u.presentRequest;
1937             if (pr->numberOfRecordsRequested &&
1938                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1939                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1940         }
1941     }
1942 }
1943
1944 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1945 {
1946     if (apdu->which == Z_APDU_initRequest)
1947     {
1948         if (m_initRequest_options &&
1949             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1950             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1951         {
1952             // There is no negotiation proposal from
1953             // client's side. OK. The proxy negotiation
1954             // in use, only.
1955             Z_InitRequest *initRequest = apdu->u.initRequest;
1956             Z_OtherInformation **otherInfo;
1957             Z_OtherInformationUnit *oi;
1958             get_otherInfoAPDU(apdu, &otherInfo);
1959             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1960             if (oi)
1961             {
1962                 ODR_MASK_SET(initRequest->options,
1963                     Z_Options_negotiationModel);
1964                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1965                 oi->information.externallyDefinedInfo =
1966                 yaz_set_proposal_charneg(odr_encode(),
1967                     (const char**)&m_proxy_negotiation_charset,
1968                     m_proxy_negotiation_charset ? 1:0,
1969                     (const char**)&m_proxy_negotiation_lang,
1970                     m_proxy_negotiation_lang ? 1:0,
1971                     1);
1972             }
1973         }
1974         else if (m_initRequest_options &&
1975                  ODR_MASK_GET(m_initRequest_options,
1976                               Z_Options_negotiationModel) &&
1977                  m_charset_converter->get_target_query_charset())
1978         {
1979             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1980                     m_session_str,
1981                     m_charset_converter->get_target_query_charset());
1982             Z_InitRequest *initRequest = apdu->u.initRequest;
1983             Z_CharSetandLanguageNegotiation *negotiation =
1984                 yaz_get_charneg_record (initRequest->otherInfo);
1985             if (negotiation &&
1986                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1987             {
1988                 NMEM nmem = nmem_create();
1989                 char **charsets = 0;
1990                 int num_charsets = 0;
1991                 char **langs = 0;
1992                 int num_langs = 0;
1993                 int selected = 0;
1994                 yaz_get_proposal_charneg (nmem, negotiation,
1995                                           &charsets, &num_charsets,
1996                                           &langs, &num_langs, &selected);
1997                 int i;
1998                 for (i = 0; i<num_charsets; i++)
1999                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
2000                             charsets[i]);
2001                 for (i = 0; i<num_langs; i++)
2002                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
2003                             langs[i]);
2004
2005                 const char *t_charset =
2006                     m_charset_converter->get_target_query_charset();
2007                 // sweep through charsets and pick the first supported
2008                 // conversion
2009                 for (i = 0; i<num_charsets; i++)
2010                 {
2011                     const char *c_charset = charsets[i];
2012                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
2013                         break;
2014                 }
2015                 if (i != num_charsets)
2016                 {
2017                     // got one .. set up ODR for reverse direction
2018                     const char *c_charset = charsets[i];
2019                     odr_set_charset(odr_encode(), c_charset, t_charset);
2020                     m_charset_converter->set_client_query_charset(c_charset);
2021                     m_charset_converter->set_client_charset_selected(selected);
2022                 }
2023                 nmem_destroy(nmem);
2024                 ODR_MASK_CLEAR(m_initRequest_options,
2025                                Z_Options_negotiationModel);
2026                 yaz_del_charneg_record(&initRequest->otherInfo);
2027             }
2028             else
2029             {
2030                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
2031                         m_session_str);
2032             }
2033         }
2034         else if (m_charset_converter->get_target_query_charset() &&
2035             m_proxy_negotiation_default_charset)
2036         {
2037             m_charset_converter->
2038                 set_client_query_charset(m_proxy_negotiation_default_charset);
2039         }
2040     }
2041     else if (apdu->which == Z_APDU_initResponse)
2042     {
2043         Z_InitResponse *initResponse = apdu->u.initResponse;
2044         Z_OtherInformation **otherInfo;
2045         get_otherInfoAPDU(apdu, &otherInfo);
2046
2047         Z_CharSetandLanguageNegotiation *charneg = 0;
2048
2049         if (otherInfo && *otherInfo &&
2050             ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel)
2051             && (charneg = yaz_get_charneg_record(*otherInfo)))
2052         {
2053             char *charset = 0;
2054             char *lang = 0;
2055             int selected = 0;
2056
2057             yaz_get_response_charneg(m_referenceId_mem, charneg,
2058                 &charset, &lang, &selected);
2059
2060             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
2061                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
2062
2063             if (m_initRequest_options &&
2064                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
2065             {
2066                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
2067                     m_session_str);
2068             }
2069             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2070             {
2071                 // negotiation-charset, negotiation-lang
2072                 // elements of config file in use.
2073
2074                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2075                     m_session_str);
2076
2077                 // clear negotiation option.
2078                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2079
2080                 // Delete negotiation (charneg-3) entry.
2081                 yaz_del_charneg_record(otherInfo);
2082             }
2083         }
2084         else
2085         {
2086             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2087             {
2088                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2089                         m_session_str);
2090             }
2091             else if (m_charset_converter->get_client_query_charset())
2092             {
2093                 Z_OtherInformation **otherInfo;
2094                 Z_OtherInformationUnit *oi;
2095                 get_otherInfoAPDU(apdu, &otherInfo);
2096                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2097                 if (oi)
2098                 {
2099                     ODR_MASK_SET(initResponse->options,
2100                                  Z_Options_negotiationModel);
2101                     if (m_initRequest_options)
2102                         ODR_MASK_SET(m_initRequest_options,
2103                                      Z_Options_negotiationModel);
2104
2105                     oi->which = Z_OtherInfo_externallyDefinedInfo;
2106                     oi->information.externallyDefinedInfo =
2107                         yaz_set_response_charneg(
2108                             odr_encode(),
2109                             m_charset_converter->get_client_query_charset(),
2110                             0 /* no lang */,
2111                             m_charset_converter->get_client_charset_selected());
2112                 }
2113             }
2114         }
2115     }
2116 }
2117
2118 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2119                                                      int error,
2120                                                      const char *addinfo)
2121 {
2122     Z_Records *rec = (Z_Records *)
2123         odr_malloc (odr, sizeof(*rec));
2124     int *err = (int *)
2125         odr_malloc (odr, sizeof(*err));
2126     Z_DiagRec *drec = (Z_DiagRec *)
2127         odr_malloc (odr, sizeof(*drec));
2128     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2129         odr_malloc (odr, sizeof(*dr));
2130     *err = error;
2131     rec->which = Z_Records_NSD;
2132     rec->u.nonSurrogateDiagnostic = dr;
2133     dr->diagnosticSetId =
2134         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
2135     dr->condition = err;
2136     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2137     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2138     return rec;
2139 }
2140
2141 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2142 {
2143     if (apdu->which == Z_APDU_searchRequest &&
2144         apdu->u.searchRequest->query &&
2145         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2146         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2147     {
2148         Z_RPNQuery *rpnquery = 0;
2149         Z_SearchRequest *sr = apdu->u.searchRequest;
2150         char *addinfo = 0;
2151
2152         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2153                 sr->query->u.type_104->u.cql);
2154
2155         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2156                                           &rpnquery, odr_encode(),
2157                                           &addinfo);
2158         if (r == -3)
2159             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2160         else if (r)
2161         {
2162             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2163             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2164
2165             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2166             new_apdu->u.searchResponse->records =
2167                 create_nonSurrogateDiagnostics(odr_encode(),
2168                                                yaz_diag_srw_to_bib1(r),
2169                                                addinfo);
2170             *new_apdu->u.searchResponse->searchStatus = 0;
2171
2172             send_to_client(new_apdu);
2173
2174             return 0;
2175         }
2176         else
2177         {
2178             sr->query->which = Z_Query_type_1;
2179             sr->query->u.type_1 = rpnquery;
2180         }
2181         return apdu;
2182     }
2183     return apdu;
2184 }
2185
2186 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2187 {
2188     if (apdu->which == Z_APDU_searchRequest &&
2189         apdu->u.searchRequest->query)
2190     {
2191         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2192             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2193         {
2194             if (m_http_version)
2195                 m_charset_converter->set_client_query_charset("UTF-8");
2196             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2197             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2198         }
2199     }
2200     return apdu;
2201 }
2202
2203
2204 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2205 {
2206     if (apdu->which == Z_APDU_searchRequest)
2207     {
2208         Z_SearchRequest *sr = apdu->u.searchRequest;
2209         int err = 0;
2210         char *addinfo = 0;
2211
2212         Yaz_ProxyConfig *cfg = check_reconfigure();
2213         if (cfg)
2214             err = cfg->check_query(odr_encode(), m_default_target,
2215                                    sr->query, &addinfo);
2216         if (err)
2217         {
2218             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2219
2220             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2221             new_apdu->u.searchResponse->records =
2222                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2223             *new_apdu->u.searchResponse->searchStatus = 0;
2224
2225             send_to_client(new_apdu);
2226
2227             return 0;
2228         }
2229     }
2230     return apdu;
2231 }
2232
2233 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2234 {
2235     if (apdu->which != Z_APDU_initRequest)
2236         return 1;  // pass if no init request
2237     Z_InitRequest *req = apdu->u.initRequest;
2238
2239     Yaz_ProxyConfig *cfg = check_reconfigure();
2240     if (!cfg)
2241         return 1;  // pass if no config
2242
2243     int ret;
2244     if (req->idAuthentication == 0)
2245     {
2246         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2247                                          m_peername);
2248     }
2249     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2250     {
2251         ret = cfg->client_authentication(
2252             m_default_target,
2253             req->idAuthentication->u.idPass->userId,
2254             req->idAuthentication->u.idPass->groupId,
2255             req->idAuthentication->u.idPass->password,
2256             m_peername);
2257     }
2258     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2259     {
2260         char user[64], pass[64];
2261         *user = '\0';
2262         *pass = '\0';
2263         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2264         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2265                                          m_peername);
2266     }
2267     else
2268         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2269                                          m_peername);
2270     return ret;
2271 }
2272
2273 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2274 {
2275     m_marcxml_mode = none;
2276     if (apdu->which == Z_APDU_searchRequest)
2277     {
2278         Z_SearchRequest *sr = apdu->u.searchRequest;
2279         int err = 0;
2280         char *addinfo = 0;
2281         Yaz_ProxyConfig *cfg = check_reconfigure();
2282
2283         Z_RecordComposition rc_temp, *rc = 0;
2284         if (sr->smallSetElementSetNames)
2285         {
2286             rc_temp.which = Z_RecordComp_simple;
2287             rc_temp.u.simple = sr->smallSetElementSetNames;
2288             rc = &rc_temp;
2289         }
2290
2291         if (sr->preferredRecordSyntax)
2292         {
2293             struct oident *ent;
2294             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2295             m_frontend_type = ent->value;
2296         }
2297         else
2298             m_frontend_type = VAL_NONE;
2299
2300         char *stylesheet_name = 0;
2301         if (cfg)
2302             err = cfg->check_syntax(odr_encode(),
2303                                     m_default_target,
2304                                     sr->preferredRecordSyntax, rc,
2305                                     &addinfo, &stylesheet_name, &m_schema,
2306                                     &m_backend_type, &m_backend_charset,
2307                                     &m_usemarcon_ini_stage1,
2308                                     &m_usemarcon_ini_stage2);
2309         if (stylesheet_name)
2310         {
2311             m_parent->low_socket_close();
2312
2313 #if HAVE_XSLT
2314             if (m_stylesheet_xsp)
2315                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2316             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2317                                                        stylesheet_name);
2318 #endif
2319             m_stylesheet_offset = 0;
2320             xfree(stylesheet_name);
2321
2322             m_parent->low_socket_open();
2323         }
2324         if (err == -1)
2325         {
2326             sr->smallSetElementSetNames = 0;
2327             sr->mediumSetElementSetNames = 0;
2328             m_marcxml_mode = marcxml;
2329             if (m_backend_type)
2330             {
2331
2332                 sr->preferredRecordSyntax =
2333                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2334                                         m_backend_type);
2335             }
2336             else
2337                 sr->preferredRecordSyntax =
2338                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2339                                            VAL_USMARC);
2340         }
2341         else if (err)
2342         {
2343             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2344
2345             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2346             new_apdu->u.searchResponse->records =
2347                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2348             *new_apdu->u.searchResponse->searchStatus = 0;
2349
2350             send_to_client(new_apdu);
2351
2352             return 0;
2353         }
2354         else if (m_backend_type)
2355         {
2356             sr->preferredRecordSyntax =
2357                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2358         }
2359     }
2360     else if (apdu->which == Z_APDU_presentRequest)
2361     {
2362         Z_PresentRequest *pr = apdu->u.presentRequest;
2363         int err = 0;
2364         char *addinfo = 0;
2365         Yaz_ProxyConfig *cfg = check_reconfigure();
2366
2367         if (pr->preferredRecordSyntax)
2368         {
2369             struct oident *ent;
2370             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2371             m_frontend_type = ent->value;
2372         }
2373         else
2374             m_frontend_type = VAL_NONE;
2375
2376         char *stylesheet_name = 0;
2377         if (cfg)
2378             err = cfg->check_syntax(odr_encode(), m_default_target,
2379                                     pr->preferredRecordSyntax,
2380                                     pr->recordComposition,
2381                                     &addinfo, &stylesheet_name, &m_schema,
2382                                     &m_backend_type, &m_backend_charset,
2383                                     &m_usemarcon_ini_stage1,
2384                                     &m_usemarcon_ini_stage2
2385                                     );
2386         if (stylesheet_name)
2387         {
2388             m_parent->low_socket_close();
2389
2390 #if HAVE_XSLT
2391             if (m_stylesheet_xsp)
2392                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2393             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2394                                                        stylesheet_name);
2395 #endif
2396             m_stylesheet_offset = 0;
2397             xfree(stylesheet_name);
2398
2399             m_parent->low_socket_open();
2400         }
2401         if (err == -1)
2402         {
2403             pr->recordComposition = 0;
2404             m_marcxml_mode = marcxml;
2405             if (m_backend_type)
2406             {
2407
2408                 pr->preferredRecordSyntax =
2409                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2410                                         m_backend_type);
2411             }
2412             else
2413                 pr->preferredRecordSyntax =
2414                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2415                                            VAL_USMARC);
2416         }
2417         else if (err)
2418         {
2419             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2420
2421             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2422             new_apdu->u.presentResponse->records =
2423                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2424             *new_apdu->u.presentResponse->presentStatus =
2425                 Z_PresentStatus_failure;
2426
2427             send_to_client(new_apdu);
2428
2429             return 0;
2430         }
2431         else if (m_backend_type)
2432         {
2433             pr->preferredRecordSyntax =
2434                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2435         }
2436     }
2437     return apdu;
2438 }
2439
2440 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2441 {
2442     if (!schema)
2443         return 0;
2444     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2445         odr_malloc(o, sizeof(Z_ElementSetNames));
2446     esn->which = Z_ElementSetNames_generic;
2447     esn->u.generic = odr_strdup(o, schema);
2448     return esn;
2449 }
2450
2451 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2452 {
2453     const char *t = 0;
2454     Yaz_ProxyConfig *cfg = check_reconfigure();
2455     if (cfg)
2456         t = cfg->get_explain_name(db, backend_db);
2457
2458     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2459     {
2460         releaseClient();
2461     }
2462
2463     if (t)
2464     {
2465         xfree(m_default_target);
2466         m_default_target = xstrdup(t);
2467     }
2468 }
2469
2470 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2471 {
2472     struct stat sbuf;
2473     if (strcmp(hreq->method, "GET"))
2474         return 0;
2475     if (hreq->path[0] != '/')
2476     {
2477         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2478         return 0;
2479     }
2480     const char *cp = hreq->path;
2481     while (*cp)
2482     {
2483         if (*cp == '/' && strchr("/.", cp[1]))
2484         {
2485             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2486             return 0;
2487         }
2488         cp++;
2489     }
2490     const char *fname = hreq->path+1;
2491     if (stat(fname, &sbuf))
2492     {
2493         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2494         return 0;
2495     }
2496     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2497     {
2498         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2499         return 0;
2500     }
2501     if (sbuf.st_size > (off_t) 1000000)
2502     {
2503         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2504         return 0;
2505     }
2506
2507     ODR o = odr_encode();
2508     Yaz_ProxyConfig *cfg = check_reconfigure();
2509     const char *ctype = cfg->check_mime_type(fname);
2510     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2511     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2512     if (m_http_version)
2513         hres->version = odr_strdup(o, m_http_version);
2514     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2515     if (m_http_keepalive)
2516         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2517     else
2518         timeout(0);
2519
2520     hres->content_len = sbuf.st_size;
2521     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2522     FILE *f = fopen(fname, "rb");
2523     if (f)
2524     {
2525         fread(hres->content_buf, 1, hres->content_len, f);
2526         fclose(f);
2527     }
2528     else
2529     {
2530         return 0;
2531     }
2532     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2533     {
2534         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2535                  fname);
2536     }
2537     int len;
2538     send_GDU(gdu, &len);
2539     recv_GDU_more(true);
2540     return 1;
2541 }
2542
2543 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2544 {
2545     if (m_s2z_odr_init)
2546     {
2547         odr_destroy(m_s2z_odr_init);
2548         m_s2z_odr_init = 0;
2549     }
2550     if (m_s2z_odr_search)
2551     {
2552         odr_destroy(m_s2z_odr_search);
2553         m_s2z_odr_search = 0;
2554     }
2555
2556     m_http_keepalive = 0;
2557     m_http_version = 0;
2558     if (!strcmp(hreq->version, "1.0"))
2559     {
2560         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2561         if (v && !strcmp(v, "Keep-Alive"))
2562             m_http_keepalive = 1;
2563         else
2564             m_http_keepalive = 0;
2565         m_http_version = "1.0";
2566     }
2567     else
2568     {
2569         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2570         if (v && !strcmp(v, "close"))
2571             m_http_keepalive = 0;
2572         else
2573             m_http_keepalive = 1;
2574         m_http_version = "1.1";
2575     }
2576
2577     const char *a = z_HTTP_header_lookup(hreq->headers, "Authorization");
2578     char authorization_str[255];
2579     *authorization_str = '\0';
2580     if (a && strncasecmp(a, "Basic ", 6) == 0)
2581         base64_decode(a + 6, authorization_str, 254);
2582
2583     Z_SRW_PDU *srw_pdu = 0;
2584     Z_SOAP *soap_package = 0;
2585     char *charset = 0;
2586     Z_SRW_diagnostic *diagnostic = 0;
2587     int num_diagnostic = 0;
2588
2589     if (file_access(hreq))
2590     {
2591         return;
2592     }
2593     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2594                             &charset) == 0
2595              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2596                                &charset, &diagnostic, &num_diagnostic) == 0)
2597     {
2598         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2599         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2600         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2601         m_s2z_init_apdu = 0;
2602         m_s2z_search_apdu = 0;
2603         m_s2z_present_apdu = 0;
2604
2605         m_s2z_stylesheet = 0;
2606
2607         Z_IdAuthentication *auth = NULL;
2608         if (*authorization_str)
2609         {
2610             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
2611             auth->which = Z_IdAuthentication_idPass;
2612             auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
2613             auth->u.idPass->groupId = NULL;
2614             char *p = strchr(authorization_str, ':');
2615             if (p)
2616             {
2617                 *p = '\0';
2618                 p++;
2619                 auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
2620             }
2621             auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
2622         }
2623
2624         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2625         {
2626             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2627
2628             const char *backend_db = srw_req->database;
2629             srw_get_client(srw_req->database, &backend_db);
2630
2631             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2632             // recordXPath unsupported.
2633             if (srw_req->recordXPath)
2634             {
2635                 yaz_add_srw_diagnostic(odr_decode(),
2636                                        &diagnostic, &num_diagnostic,
2637                                        72, 0);
2638             }
2639             // sort unsupported
2640             if (srw_req->sort_type != Z_SRW_sort_type_none)
2641             {
2642                 yaz_add_srw_diagnostic(odr_decode(),
2643                                        &diagnostic, &num_diagnostic,
2644                                        80, 0);
2645             }
2646             // save stylesheet
2647             if (srw_req->stylesheet)
2648                 m_s2z_stylesheet =
2649                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2650
2651             // set packing for response records ..
2652             if (srw_req->recordPacking &&
2653                 !strcmp(srw_req->recordPacking, "xml"))
2654                 m_s2z_packing = Z_SRW_recordPacking_XML;
2655             else
2656                 m_s2z_packing = Z_SRW_recordPacking_string;
2657
2658             if (num_diagnostic)
2659             {
2660                 Z_SRW_PDU *srw_pdu =
2661                     yaz_srw_get(odr_encode(),
2662                                 Z_SRW_searchRetrieve_response);
2663                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2664
2665                 srw_res->diagnostics = diagnostic;
2666                 srw_res->num_diagnostics = num_diagnostic;
2667                 send_srw_response(srw_pdu);
2668                 return;
2669             }
2670
2671             // prepare search PDU
2672             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2673                                           Z_APDU_searchRequest);
2674             Z_SearchRequest *z_searchRequest =
2675                 m_s2z_search_apdu->u.searchRequest;
2676
2677             z_searchRequest->num_databaseNames = 1;
2678             z_searchRequest->databaseNames = (char**)
2679                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2680             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2681                                                            backend_db);
2682
2683             // query transformation
2684             Z_Query *query = (Z_Query *)
2685                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2686             z_searchRequest->query = query;
2687
2688             if (srw_req->query_type == Z_SRW_query_type_cql)
2689             {
2690                 Z_External *ext = (Z_External *)
2691                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2692                 ext->direct_reference =
2693                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2694                 ext->indirect_reference = 0;
2695                 ext->descriptor = 0;
2696                 ext->which = Z_External_CQL;
2697                 ext->u.cql = srw_req->query.cql;
2698
2699                 query->which = Z_Query_type_104;
2700                 query->u.type_104 =  ext;
2701             }
2702             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2703             {
2704                 Z_RPNQuery *RPNquery;
2705                 YAZ_PQF_Parser pqf_parser;
2706
2707                 pqf_parser = yaz_pqf_create ();
2708
2709                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2710                                           srw_req->query.pqf);
2711                 if (!RPNquery)
2712                 {
2713                     const char *pqf_msg;
2714                     size_t off;
2715                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2716                     int ioff = off;
2717                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
2718                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2719
2720                     send_to_srw_client_error(10, 0);
2721                     return;
2722                 }
2723                 query->which = Z_Query_type_1;
2724                 query->u.type_1 =  RPNquery;
2725
2726                 yaz_pqf_destroy (pqf_parser);
2727             }
2728             else
2729             {
2730                 send_to_srw_client_error(7, "query");
2731                 return;
2732             }
2733
2734             // present
2735             m_s2z_present_apdu = 0;
2736             int max = 0;
2737             if (srw_req->maximumRecords)
2738                 max = *srw_req->maximumRecords;
2739             int start = 1;
2740             if (srw_req->startRecord)
2741                 start = *srw_req->startRecord;
2742             if (max > 0)
2743             {
2744                 // Some backend, such as Voyager doesn't honor piggyback
2745                 // So we use present always (0 &&).
2746                 if (0 && start <= 1)  // Z39.50 piggyback
2747                 {
2748                     *z_searchRequest->smallSetUpperBound = max;
2749                     *z_searchRequest->mediumSetPresentNumber = max;
2750                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2751
2752                     z_searchRequest->preferredRecordSyntax =
2753                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2754                                                VAL_TEXT_XML);
2755                     if (srw_req->recordSchema)
2756                     {
2757                         z_searchRequest->smallSetElementSetNames =
2758                             z_searchRequest->mediumSetElementSetNames =
2759                             mk_esn_from_schema(m_s2z_odr_search,
2760                                                srw_req->recordSchema);
2761                     }
2762                 }
2763                 else   // Z39.50 present
2764                 {
2765                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2766                                                    Z_APDU_presentRequest);
2767                     Z_PresentRequest *z_presentRequest =
2768                         m_s2z_present_apdu->u.presentRequest;
2769                     *z_presentRequest->resultSetStartPoint = start;
2770                     *z_presentRequest->numberOfRecordsRequested = max;
2771                     z_presentRequest->preferredRecordSyntax =
2772                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2773                                                VAL_TEXT_XML);
2774                     if (srw_req->recordSchema)
2775                     {
2776                         z_presentRequest->recordComposition =
2777                             (Z_RecordComposition *)
2778                             odr_malloc(m_s2z_odr_search,
2779                                        sizeof(Z_RecordComposition));
2780                         z_presentRequest->recordComposition->which =
2781                             Z_RecordComp_simple;
2782                         z_presentRequest->recordComposition->u.simple =
2783                             mk_esn_from_schema(m_s2z_odr_search,
2784                                                srw_req->recordSchema);
2785                     }
2786                 }
2787             }
2788             if (!m_client)
2789             {
2790                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2791                                             Z_APDU_initRequest);
2792
2793                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2794
2795                 // prevent m_initRequest_apdu memory from being grabbed
2796                 // in Yaz_Proxy::handle_incoming_Z_PDU
2797                 m_initRequest_apdu = m_s2z_init_apdu;
2798                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2799                 return;
2800             }
2801             else
2802             {
2803                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2804                 return;
2805             }
2806         }
2807         else if (srw_pdu->which == Z_SRW_explain_request)
2808         {
2809             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2810
2811             const char *backend_db = srw_req->database;
2812             srw_get_client(srw_req->database, &backend_db);
2813
2814             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2815
2816             // save stylesheet
2817             if (srw_req->stylesheet)
2818                 m_s2z_stylesheet =
2819                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2820
2821             if (srw_req->recordPacking &&
2822                 !strcmp(srw_req->recordPacking, "xml"))
2823                 m_s2z_packing = Z_SRW_recordPacking_XML;
2824             else
2825                 m_s2z_packing = Z_SRW_recordPacking_string;
2826
2827             if (num_diagnostic)
2828             {
2829                 send_srw_explain_response(diagnostic, num_diagnostic);
2830                 return;
2831             }
2832
2833             if (!m_client)
2834             {
2835                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2836                                             Z_APDU_initRequest);
2837
2838                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2839
2840                 // prevent m_initRequest_apdu memory from being grabbed
2841                 // in Yaz_Proxy::handle_incoming_Z_PDU
2842                 m_initRequest_apdu = m_s2z_init_apdu;
2843                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2844             }
2845             else
2846                 send_srw_explain_response(0, 0);
2847             return;
2848         }
2849         else if (srw_pdu->which == Z_SRW_scan_request)
2850         {
2851             m_s2z_database = odr_strdup(m_s2z_odr_init,
2852                                         srw_pdu->u.scan_request->database);
2853
2854             yaz_add_srw_diagnostic(odr_decode(),
2855                                    &diagnostic, &num_diagnostic,
2856                                    4, "scan");
2857             Z_SRW_PDU *srw_pdu =
2858                 yaz_srw_get(odr_encode(),
2859                             Z_SRW_scan_response);
2860             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2861
2862             srw_res->diagnostics = diagnostic;
2863             srw_res->num_diagnostics = num_diagnostic;
2864             send_srw_response(srw_pdu);
2865             return;
2866         }
2867         else
2868         {
2869             m_s2z_database = 0;
2870
2871             send_to_srw_client_error(4, 0);
2872         }
2873     }
2874     send_http_response(400);
2875 }
2876
2877 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2878 {
2879
2880     Z_OtherInformation **oi;
2881     get_otherInfoAPDU(apdu, &oi);
2882
2883     if (apdu->u.initRequest->implementationId)
2884         yaz_log(YLOG_LOG, "%simplementationId: %s",
2885                 m_session_str, apdu->u.initRequest->implementationId);
2886     if (apdu->u.initRequest->implementationName)
2887         yaz_log(YLOG_LOG, "%simplementationName: %s",
2888                 m_session_str, apdu->u.initRequest->implementationName);
2889     if (apdu->u.initRequest->implementationVersion)
2890         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2891                 m_session_str, apdu->u.initRequest->implementationVersion);
2892     if (m_initRequest_apdu == 0)
2893     {
2894         if (m_initRequest_mem)
2895             nmem_destroy(m_initRequest_mem);
2896
2897         m_initRequest_apdu = apdu;
2898         m_initRequest_mem = odr_extract_mem(odr_decode());
2899
2900         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2901             preferredMessageSize;
2902         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2903         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2904             maximumRecordSize;
2905         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2906
2907         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2908             yaz_get_charneg_record(*oi);
2909
2910         // Save proposal charsets and langs.
2911         if (ODR_MASK_GET(apdu->u.initRequest->options,
2912                          Z_Options_negotiationModel)
2913             && charSetandLangRecord)
2914         {
2915
2916             yaz_get_proposal_charneg(m_referenceId_mem,
2917                                      charSetandLangRecord,
2918                                      &m_initRequest_oi_negotiation_charsets,
2919                                      &m_initRequest_oi_negotiation_num_charsets,
2920                                      &m_initRequest_oi_negotiation_langs,
2921                                      &m_initRequest_oi_negotiation_num_langs,
2922                                      &m_initRequest_oi_negotiation_selected);
2923
2924             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2925             {
2926                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2927                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2928                         m_initRequest_oi_negotiation_charsets[i]:"none");
2929             }
2930             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2931             {
2932                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2933                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2934                         m_initRequest_oi_negotiation_langs[i]:"none");
2935             }
2936             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2937                     m_session_str, m_initRequest_oi_negotiation_selected);
2938         }
2939         // save init options for the response..
2940         m_initRequest_options = apdu->u.initRequest->options;
2941
2942         apdu->u.initRequest->options =
2943             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2944                                        sizeof(Odr_bitmask));
2945         ODR_MASK_ZERO(apdu->u.initRequest->options);
2946         int i;
2947         for (i = 0; i<= 24; i++)
2948             ODR_MASK_SET(apdu->u.initRequest->options, i);
2949         // check negotiation option
2950         if (!ODR_MASK_GET(m_initRequest_options,
2951                           Z_Options_negotiationModel))
2952         {
2953             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2954                            Z_Options_negotiationModel);
2955         }
2956         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2957                        Z_Options_concurrentOperations);
2958         // make new version
2959         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2960         apdu->u.initRequest->protocolVersion =
2961             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2962                                        sizeof(Odr_bitmask));
2963         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2964
2965         for (i = 0; i<= 8; i++)
2966             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2967     }
2968     handle_charset_lang_negotiation(apdu);
2969     if (m_client->m_init_flag)
2970     {
2971         if (handle_init_response_for_invalid_session(apdu))
2972             return;
2973         if (m_client->m_initResponse)
2974         {
2975             Z_APDU *apdu2 = m_client->m_initResponse;
2976             apdu2->u.initResponse->otherInfo = 0;
2977             if (m_client->m_cookie && *m_client->m_cookie)
2978                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2979                                            m_client->m_cookie);
2980             apdu2->u.initResponse->referenceId =
2981                 apdu->u.initRequest->referenceId;
2982             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2983             apdu2->u.initResponse->protocolVersion =
2984                 m_client->m_initResponse_version;
2985
2986             handle_charset_lang_negotiation(apdu2);
2987
2988             send_to_client(apdu2);
2989             m_timeout_mode = timeout_normal;
2990             return;
2991         }
2992     }
2993     m_client->m_init_flag = 1;
2994
2995 #if USE_AUTH_MSG
2996     Auth_Msg *m = new Auth_Msg;
2997     m->m_proxy = this;
2998     z_APDU(odr_encode(), &apdu, 0, "encode");
2999     char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
3000     m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
3001     memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
3002     odr_reset(odr_encode());
3003
3004     inc_ref();
3005     m_my_thread->put(m);
3006 #else
3007     int ret = handle_authentication(apdu);
3008     result_authentication(apdu, ret);
3009 #endif
3010 }
3011
3012 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
3013 {
3014     Z_ReferenceId **refid = get_referenceIdP(apdu);
3015     nmem_reset(m_referenceId_mem);
3016     if (refid && *refid)
3017     {
3018         m_referenceId = (Z_ReferenceId *)
3019             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
3020         m_referenceId->len = m_referenceId->size = (*refid)->len;
3021         m_referenceId->buf = (unsigned char *)
3022             nmem_malloc(m_referenceId_mem, (*refid)->len);
3023         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
3024     }
3025     else
3026         m_referenceId = 0;
3027
3028     if (!m_client && m_flag_invalid_session)
3029     {
3030         // Got request for a session that is invalid..
3031         m_apdu_invalid_session = apdu; // save package
3032         m_mem_invalid_session = odr_extract_mem(odr_decode());
3033         apdu = m_initRequest_apdu;     // but throw an init to the target
3034     }
3035
3036     // Determine our client.
3037     Z_OtherInformation **oi;
3038     get_otherInfoAPDU(apdu, &oi);
3039     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
3040     if (!m_client)
3041     {
3042         if (m_http_version)
3043         {   // HTTP. Send not found
3044             send_http_response(404);
3045             return;
3046         }
3047         else
3048         {
3049             // Z39.50 just shutdown
3050             delete this;
3051             return;
3052         }
3053     }
3054
3055     m_client->m_server = this;
3056
3057     if (apdu->which == Z_APDU_initRequest)
3058         handle_init(apdu);
3059     else
3060         handle_incoming_Z_PDU_2(apdu);
3061 }
3062
3063 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
3064 {
3065     handle_max_record_retrieve(apdu);
3066
3067     if (apdu)
3068         apdu = handle_syntax_validation(apdu);
3069
3070     if (apdu)
3071         apdu = handle_query_transformation(apdu);
3072
3073     if (apdu)
3074         apdu = handle_target_charset_conversion(apdu);
3075
3076     if (apdu)
3077         apdu = handle_query_validation(apdu);
3078
3079     if (apdu)
3080         apdu = result_set_optimize(apdu);
3081
3082     if (!apdu)
3083     {
3084         m_client->timeout(m_target_idletime);  // mark it active even
3085         recv_GDU_more(true);
3086         // though we didn't use it
3087         return;
3088     }
3089
3090     // delete other info construct completely if 0 elements
3091     Z_OtherInformation **oi;
3092     get_otherInfoAPDU(apdu, &oi);
3093     if (oi && *oi && (*oi)->num_elements == 0)
3094         *oi = 0;
3095
3096     if (apdu->which == Z_APDU_presentRequest &&
3097         m_client->m_resultSetStartPoint == 0)
3098     {
3099         Z_PresentRequest *pr = apdu->u.presentRequest;
3100         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3101         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3102     } else {
3103         m_client->m_resultSetStartPoint = 0;
3104     }
3105     if (m_client->send_to_target(apdu) < 0)
3106     {
3107         m_client->shutdown();
3108     }
3109     else
3110         m_client->m_waiting = 1;
3111 }
3112
3113 void Yaz_Proxy::connectNotify()
3114 {
3115 }
3116
3117 void Yaz_Proxy::releaseClient()
3118 {
3119     xfree(m_proxyTarget);
3120     m_proxyTarget = 0;
3121     m_flag_invalid_session = 0;
3122     // only keep if keep_alive flag is set...
3123     if (m_client &&
3124         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3125         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3126         m_client->m_waiting == 0)
3127     {
3128         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3129                  m_session_str,
3130                  m_client->get_hostname());
3131         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3132                 m_session_str, m_client->m_pdu_recv,
3133                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3134                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3135         assert (m_client->m_waiting != 2);
3136         // Tell client (if any) that no server connection is there..
3137         m_client->m_server = 0;
3138         m_client = 0;
3139     }
3140     else if (m_client)
3141     {
3142         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
3143                  m_session_str,
3144                  m_client->get_hostname());
3145         assert (m_client->m_waiting != 2);
3146         delete m_client;
3147         m_client = 0;
3148     }
3149     else if (!m_parent)
3150     {
3151         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
3152                  m_session_str);
3153         assert (m_parent);
3154     }
3155     else
3156     {
3157         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
3158                  m_session_str);
3159     }
3160     if (m_parent)
3161         m_parent->pre_init();
3162 }
3163
3164 bool Yaz_Proxy::dec_ref(bool main_ptr)
3165 {
3166     assert(m_ref_count > 0);
3167     if (main_ptr)
3168     {
3169         if (m_main_ptr_dec)
3170             return false;
3171         m_main_ptr_dec = true;
3172     }
3173
3174     m_http_keepalive = 0;
3175
3176     --m_ref_count;
3177     if (m_ref_count > 0)
3178         return false;
3179
3180     releaseClient();
3181
3182     delete this;
3183     return true;
3184 }
3185
3186 const char *Yaz_ProxyClient::get_session_str()
3187 {
3188     if (!m_server)
3189         return "0 ";
3190     return m_server->get_session_str();
3191 }
3192
3193 void Yaz_ProxyClient::shutdown()
3194 {
3195     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3196              get_hostname());
3197
3198     if (m_server)
3199     {
3200         m_waiting = 1;      // ensure it's released from Proxy in releaseClient
3201         m_server->dec_ref(true);
3202     }
3203     else
3204         delete this;
3205 }
3206
3207 void Yaz_Proxy::failNotify()
3208 {
3209     inc_request_no();
3210     yaz_log (YLOG_LOG, "%sConnection closed by client",
3211              get_session_str());
3212     dec_ref(true);
3213 }
3214
3215 void Yaz_Proxy::send_response_fail_client(const char *addr)
3216 {
3217     if (m_http_version)
3218     {
3219         Z_SRW_diagnostic *diagnostic = 0;
3220         int num_diagnostic = 0;
3221
3222         yaz_add_srw_diagnostic(odr_encode(),
3223                                &diagnostic, &num_diagnostic,
3224                                YAZ_SRW_SYSTEM_TEMPORARILY_UNAVAILABLE, addr);
3225         if (m_s2z_search_apdu)
3226             send_srw_search_response(diagnostic, num_diagnostic);
3227         else
3228             send_srw_explain_response(diagnostic, num_diagnostic);
3229     }
3230 }
3231 void Yaz_ProxyClient::failNotify()
3232 {
3233     if (m_server)
3234         m_server->inc_request_no();
3235     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
3236              get_session_str(), get_hostname());
3237
3238     if (m_server)
3239         m_server->send_response_fail_client(get_hostname());
3240     shutdown();
3241 }
3242
3243 void Yaz_ProxyClient::connectNotify()
3244 {
3245     const char *s = get_session_str();
3246     const char *h = get_hostname();
3247     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3248              m_target_idletime);
3249     timeout(m_target_idletime);
3250     if (!m_server)
3251         pre_init_client();
3252 }
3253
3254 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3255                                               *the_PDU_Observable, int fd)
3256 {
3257     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3258 }
3259
3260 Yaz_ProxyClient::~Yaz_ProxyClient()
3261 {
3262     if (m_prev)
3263         *m_prev = m_next;
3264     if (m_next)
3265         m_next->m_prev = m_prev;
3266     m_waiting = 2;     // for debugging purposes only.
3267     odr_destroy(m_init_odr);
3268     odr_destroy(m_idAuthentication_odr);
3269     delete m_last_query;
3270     xfree (m_last_resultSetId);
3271     xfree (m_cookie);
3272 }
3273
3274 void Yaz_ProxyClient::pre_init_client()
3275 {
3276     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3277     Z_InitRequest *req = apdu->u.initRequest;
3278
3279     int i;
3280     for (i = 0; i<= 24; i++)
3281         ODR_MASK_SET(req->options, i);
3282     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3283                    Z_Options_negotiationModel);
3284     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3285                    Z_Options_concurrentOperations);
3286     for (i = 0; i<= 10; i++)
3287         ODR_MASK_SET(req->protocolVersion, i);
3288
3289     if (send_to_target(apdu) < 0)
3290     {
3291         delete this;
3292     }
3293     else
3294     {
3295         m_waiting = 1;
3296         m_init_flag = 1;
3297     }
3298 }
3299
3300 void Yaz_Proxy::pre_init()
3301 {
3302     int i;
3303     const char *name = 0;
3304     const char *zurl_in_use[MAX_ZURL_PLEX];
3305     int limit_bw, limit_pdu, limit_req, limit_search;
3306     int target_idletime, client_idletime;
3307     int max_clients;
3308     int keepalive_limit_bw, keepalive_limit_pdu;
3309     int pre_init;
3310     const char *cql2rpn = 0;
3311     const char *authentication = 0;
3312     const char *negotiation_charset = 0;
3313     const char *negotiation_lang = 0;
3314
3315     Yaz_ProxyConfig *cfg = check_reconfigure();
3316
3317     zurl_in_use[0] = 0;
3318
3319     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3320         set_APDU_yazlog(1);
3321     else
3322         set_APDU_yazlog(0);
3323
3324     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3325                                           &limit_bw, &limit_pdu, &limit_req,
3326                                           &limit_search,
3327                                           &target_idletime, &client_idletime,
3328                                           &max_clients,
3329                                           &keepalive_limit_bw,
3330                                           &keepalive_limit_pdu,
3331                                           &pre_init,
3332                                           &cql2rpn,
3333                                           &authentication,
3334                                           &negotiation_charset,
3335                                           &negotiation_lang,
3336                                           0,
3337                                           0) ; i++)
3338     {
3339         if (pre_init)
3340         {
3341             int j;
3342             for (j = 0; zurl_in_use[j]; j++)
3343             {
3344                 Yaz_ProxyClient *c;
3345                 int spare = 0;
3346                 int spare_waiting = 0;
3347                 int in_use = 0;
3348                 int other = 0;
3349                 for (c = m_clientPool; c; c = c->m_next)
3350                 {
3351                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3352                     {
3353                         if (c->m_cookie == 0)
3354                         {
3355                             if (c->m_server == 0)
3356                                 if (c->m_waiting)
3357                                     spare_waiting++;
3358                                 else
3359                                     spare++;
3360                             else
3361                                 in_use++;
3362                         }
3363                         else
3364                             other++;
3365                     }
3366                 }
3367                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3368                         "sparew=%d preinit=%d",m_session_str,
3369                         name, zurl_in_use[j], in_use, other,
3370                         spare, spare_waiting, pre_init);
3371                 if (spare + spare_waiting < pre_init)
3372                 {
3373                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3374                     c->m_next = m_clientPool;
3375                     if (c->m_next)
3376                         c->m_next->m_prev = &c->m_next;
3377                     m_clientPool = c;
3378                     c->m_prev = &m_clientPool;
3379
3380                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3381                         c->set_APDU_yazlog(1);
3382                     else
3383                         c->set_APDU_yazlog(0);
3384
3385                     if (c->client(zurl_in_use[j]))
3386                     {
3387                         timeout(60);
3388                         delete c;
3389                         return;
3390                     }
3391                     c->timeout(30);
3392                     c->m_waiting = 1;
3393                     c->m_target_idletime = target_idletime;
3394                     c->m_seqno = m_seqno++;
3395                 }
3396             }
3397         }
3398     }
3399 }
3400
3401 void Yaz_Proxy::timeoutNotify()
3402 {
3403     if (m_parent)
3404     {
3405         GDU *gdu;
3406         switch(m_timeout_mode)
3407         {
3408         case timeout_normal:
3409         case timeout_busy:
3410             inc_request_no();
3411             m_in_queue.clear();
3412             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3413             dec_ref(true);
3414             break;
3415         case timeout_reduce:
3416             timeout(m_client_idletime);
3417             m_timeout_mode = timeout_busy;
3418             gdu = m_timeout_gdu;
3419             m_timeout_gdu = 0;
3420             recv_GDU_normal(gdu);
3421             break;
3422         case timeout_xsl:
3423             assert(m_stylesheet_nprl);
3424             convert_xsl_delay();
3425             recv_GDU_more(true);
3426         }
3427     }
3428     else
3429     {
3430         timeout(600);
3431         pre_init();
3432     }
3433 }
3434
3435 void Yaz_Proxy::markInvalid()
3436 {
3437     m_client = 0;
3438     m_flag_invalid_session = 1;
3439 }
3440
3441 void Yaz_ProxyClient::timeoutNotify()
3442 {
3443     if (m_server)
3444         m_server->inc_request_no();
3445
3446     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3447              get_hostname());
3448
3449     if (m_server)
3450         m_server->send_response_fail_client(get_hostname());
3451     shutdown();
3452
3453     m_root->pre_init();
3454 }
3455
3456 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
3457                                  Yaz_Proxy *parent) :
3458     Z_Assoc (the_PDU_Observable)
3459 {
3460     m_cookie = 0;
3461     m_next = 0;
3462     m_prev = 0;
3463     m_init_flag = 0;
3464     m_last_query = 0;
3465     m_last_resultSetId = 0;
3466     m_last_resultCount = 0;
3467     m_last_ok = 0;
3468     m_sr_transform = 0;
3469     m_waiting = 0;
3470     m_init_odr = odr_createmem (ODR_DECODE);
3471     m_initResponse = 0;
3472     m_initResponse_options = 0;
3473     m_initResponse_version = 0;
3474     m_initResponse_preferredMessageSize = 0;
3475     m_initResponse_maximumRecordSize = 0;
3476     m_resultSetStartPoint = 0;
3477     m_bytes_sent = m_bytes_recv = 0;
3478     m_pdu_recv = 0;
3479     m_server = 0;
3480     m_seqno = 0;
3481     m_target_idletime = 600;
3482     m_root = parent;
3483     m_idAuthentication_odr = odr_createmem(ODR_ENCODE);
3484     m_idAuthentication_ber_buf = 0;
3485     m_idAuthentication_ber_size = 0;
3486 }
3487
3488 const char *Yaz_Proxy::option(const char *name, const char *value)
3489 {
3490     if (!strcmp (name, "optimize")) {
3491         if (value) {
3492             xfree (m_optimize);
3493             m_optimize = xstrdup (value);
3494         }
3495         return m_optimize;
3496     }
3497     return 0;
3498 }
3499
3500 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
3501 {
3502
3503 }
3504
3505 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
3506 {
3507     if (apdu->which == Z_GDU_Z3950)
3508         recv_Z_PDU(apdu->u.z3950, len);
3509     else if (apdu->which == Z_GDU_HTTP_Response)
3510         recv_HTTP_response(apdu->u.HTTP_Response, len);
3511     else
3512         shutdown();
3513 }
3514
3515 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
3516 {
3517     if (!m_flag_invalid_session)
3518         return 0;
3519     m_flag_invalid_session = 0;
3520     handle_incoming_Z_PDU(m_apdu_invalid_session);
3521     assert (m_mem_invalid_session);
3522     nmem_destroy(m_mem_invalid_session);
3523     m_mem_invalid_session = 0;
3524     return 1;
3525 }
3526
3527 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
3528 {
3529     m_bytes_recv += len;
3530
3531     m_pdu_recv++;
3532     m_waiting = 0;
3533     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
3534         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
3535                  apdu_name(apdu), get_hostname(), len);
3536     if (apdu->which == Z_APDU_initResponse)
3537     {
3538         if (!m_server)  // if this is a pre init session , check for more
3539             m_root->pre_init();
3540         NMEM nmem = odr_extract_mem (odr_decode());
3541         odr_reset (m_init_odr);
3542         nmem_transfer (m_init_odr->mem, nmem);
3543         m_initResponse = apdu;
3544         m_initResponse_options = apdu->u.initResponse->options;
3545         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3546         m_initResponse_preferredMessageSize =
3547             *apdu->u.initResponse->preferredMessageSize;
3548         m_initResponse_maximumRecordSize =
3549             *apdu->u.initResponse->maximumRecordSize;
3550
3551         Z_InitResponse *ir = apdu->u.initResponse;
3552         char *im0 = ir->implementationName;
3553
3554         char *im1 = (char*)
3555             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3556         *im1 = '\0';
3557         if (im0)
3558         {
3559             strcat(im1, im0);
3560             strcat(im1, " ");
3561         }
3562         strcat(im1, "(YAZ Proxy)");
3563         ir->implementationName = im1;
3564
3565         nmem_destroy (nmem);
3566
3567         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3568             return;
3569     }
3570     if (apdu->which == Z_APDU_searchResponse)
3571     {
3572         Z_SearchResponse *sr = apdu->u.searchResponse;
3573         m_last_resultCount = *sr->resultCount;
3574         int status = *sr->searchStatus;
3575         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3576         {
3577             m_last_ok = 1;
3578
3579             if (sr->records && sr->records->which == Z_Records_DBOSD)
3580             {
3581                 m_cache.add(odr_decode(),
3582                             sr->records->u.databaseOrSurDiagnostics, 1,
3583                             *sr->resultCount);
3584             }
3585         }
3586     }
3587     if (apdu->which == Z_APDU_presentResponse)
3588     {
3589         Z_PresentResponse *pr = apdu->u.presentResponse;
3590         if (m_sr_transform)
3591         {
3592             m_sr_transform = 0;
3593             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3594             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3595             sr->referenceId = pr->referenceId;
3596             *sr->resultCount = m_last_resultCount;
3597             sr->records = pr->records;
3598             sr->nextResultSetPosition = pr->nextResultSetPosition;
3599             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3600             apdu = new_apdu;
3601         }
3602         if (pr->records &&
3603             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3604         {
3605             m_cache.add(odr_decode(),
3606                         pr->records->u.databaseOrSurDiagnostics,
3607                         m_resultSetStartPoint, -1);
3608             m_resultSetStartPoint = 0;
3609         }
3610     }
3611     if (m_cookie)
3612         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3613
3614     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
3615
3616     if (server)
3617         server->send_to_client(apdu);
3618     if (apdu->which == Z_APDU_close)
3619         shutdown();
3620     else if (server)
3621         server->recv_GDU_more(true);
3622 }
3623
3624 void Yaz_Proxy::low_socket_close()
3625 {
3626 #if WIN32
3627 #else
3628     int i;
3629     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3630         if  (m_lo_fd[i] >= 0)
3631             ::close(m_lo_fd[i]);
3632 #endif
3633 }
3634
3635 void Yaz_Proxy::low_socket_open()
3636 {
3637 #if WIN32
3638 #else
3639     int i;
3640     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3641         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3642 #endif
3643 }
3644
3645 int Yaz_Proxy::server(const char *addr)
3646 {
3647     int r = Z_Assoc::server(addr);
3648     if (!r)
3649     {
3650         yaz_log(YLOG_LOG, "%sStarted proxy "
3651 #ifdef VERSION
3652             VERSION
3653 #endif
3654             " on %s", m_session_str, addr);
3655         timeout(1);
3656     }
3657     return r;
3658 }
3659
3660 void Yaz_Proxy::base64_decode(const char *base64, char *buf, int buf_len)
3661 {
3662     const char *base64_chars =
3663         "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
3664     int len = strlen(base64);
3665     int buf_pos = 0;
3666     int index = 1;
3667
3668     for (int pos = 0; pos <= len; pos++)
3669     {
3670         if (base64[pos] == '=' || buf_pos + 1 >= buf_len)
3671             break;
3672
3673         const char *ch_ptr = strchr(base64_chars, base64[pos]);
3674         if (!ch_ptr)
3675             break;
3676         char ch = (char) (ch_ptr - base64_chars);
3677         switch (index)
3678         {
3679             case 1:
3680                 buf[buf_pos] = ch << 2;
3681                 break;
3682             case 2:
3683                 buf[buf_pos++] += (ch & 0x30) >> 4;
3684                 buf[buf_pos] = (ch & 0x0f) << 4;
3685                 break;
3686             case 3:
3687                 buf[buf_pos++] += (ch & 0x3c) >> 2;
3688                 buf[buf_pos] = (ch & 0x03) << 6;
3689                 break;
3690             case 4:
3691                 buf[buf_pos++] += ch;
3692         }
3693         if (index < 4)
3694             index++;
3695         else
3696             index = 1;
3697     }
3698     buf[buf_pos] = '\0';
3699 }
3700
3701 /*
3702  * Local variables:
3703  * c-basic-offset: 4
3704  * indent-tabs-mode: nil
3705  * End:
3706  * vim: shiftwidth=4 tabstop=8 expandtab
3707  */
3708