src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.44 2006-03-25 10:59:14 adam Exp $
   2    Copyright (c) 1998-2005, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <stdlib.h>
  42 #include <time.h>
  43 #include <fcntl.h>
  44
  45 #include <yaz/srw.h>
  46 #include <yaz/marcdisp.h>
  47 #include <yaz/yaz-iconv.h>
  48 #include <yaz/log.h>
  49 #include <yaz/diagbib1.h>
  50 #include "proxyp.h"
  51 #include <yaz/pquery.h>
  52 #include <yaz/otherinfo.h>
  53 #include <yaz/charneg.h>
  54 #include "msg-thread.h"
  55
  56 using namespace yazpp_1;
  57
  58 #ifdef WIN32
  59 #define strncasecmp _strnicmp
  60 #endif
  61
  62 #define USE_AUTH_MSG 1
  63
  64 #if USE_AUTH_MSG
  65 class Auth_Msg : public IMsg_Thread {
  66 public:
  67     int m_ret;
  68     IMsg_Thread *handle();
  69     void result();
  70     Yaz_Proxy *m_proxy;
  71     NMEM m_nmem;
  72     char *m_apdu_buf;
  73     int m_apdu_len;
  74     Auth_Msg();
  75     virtual ~Auth_Msg();
  76 };
  77
  78 Auth_Msg::Auth_Msg()
  79 {
  80     m_nmem = nmem_create();
  81 }
  82
  83 Auth_Msg::~Auth_Msg()
  84 {
  85     nmem_destroy(m_nmem);
  86 }
  87
  88 IMsg_Thread *Auth_Msg::handle()
  89 {
  90     yaz_log(YLOG_LOG, "Auth_Msg:handle begin");
  91     ODR decode = odr_createmem(ODR_DECODE);
  92     Z_APDU *apdu;
  93
  94     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
  95     int r = z_APDU(decode, &apdu, 0, 0);
  96     if (!r)
  97     {
  98         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
  99     }
 100     else
 101     {
 102         m_ret = m_proxy->handle_authentication(apdu);
 103     }
 104     yaz_log(YLOG_LOG, "Auth_Msg:handle end");
 105     odr_destroy(decode);
 106     return this;
 107 }
 108
 109 void Auth_Msg::result()
 110 {
 111     yaz_log(YLOG_LOG, "Auth_Msg:result proxy ok buf=%p len=%d",
 112             m_apdu_buf, m_apdu_len);
 113     if (m_proxy->dec_ref(false))
 114         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
 115     else
 116     {
 117         yaz_log(YLOG_LOG, "Auth_Msg::proxy still alive");
 118         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
 119         Z_APDU *apdu = 0;
 120         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
 121         if (r)
 122             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU OK");
 123         else
 124             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
 125         m_proxy->result_authentication(apdu, m_ret);
 126     }
 127     delete this;
 128 }
 129
 130 #endif
 131
 132 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
 133 {
 134     if (apdu == 0 || ret == 0)
 135     {
 136         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
 137         *apdu_reject->u.initResponse->result = 0;
 138         send_to_client(apdu_reject);
 139         dec_ref(false);
 140     }
 141     else
 142     {
 143         if (apdu->which == Z_APDU_initRequest)
 144         {
 145             Yaz_ProxyConfig *cfg = check_reconfigure();
 146             if (cfg)
 147                 cfg->target_authentication(m_default_target, odr_encode(),
 148                                            apdu->u.initRequest);
 149         }
 150         handle_incoming_Z_PDU_2(apdu);
 151     }
 152 }
 153
 154 static const char *apdu_name(Z_APDU *apdu)
 155 {
 156     switch (apdu->which)
 157     {
 158     case Z_APDU_initRequest:
 159         return "initRequest";
 160     case Z_APDU_initResponse:
 161         return "initResponse";
 162     case Z_APDU_searchRequest:
 163         return "searchRequest";
 164     case Z_APDU_searchResponse:
 165         return "searchResponse";
 166     case Z_APDU_presentRequest:
 167         return "presentRequest";
 168     case Z_APDU_presentResponse:
 169         return "presentResponse";
 170     case Z_APDU_deleteResultSetRequest:
 171         return "deleteResultSetRequest";
 172     case Z_APDU_deleteResultSetResponse:
 173         return "deleteResultSetResponse";
 174     case Z_APDU_scanRequest:
 175         return "scanRequest";
 176     case Z_APDU_scanResponse:
 177         return "scanResponse";
 178     case Z_APDU_sortRequest:
 179         return "sortRequest";
 180     case Z_APDU_sortResponse:
 181         return "sortResponse";
 182     case Z_APDU_extendedServicesRequest:
 183         return "extendedServicesRequest";
 184     case Z_APDU_extendedServicesResponse:
 185         return "extendedServicesResponse";
 186     case Z_APDU_close:
 187         return "close";
 188     }
 189     return "other";
 190 }
 191
 192 static const char *gdu_name(Z_GDU *gdu)
 193 {
 194     switch(gdu->which)
 195     {
 196     case Z_GDU_Z3950:
 197         return apdu_name(gdu->u.z3950);
 198     case Z_GDU_HTTP_Request:
 199         return "HTTP Request";
 200     case Z_GDU_HTTP_Response:
 201         return "HTTP Response";
 202     }
 203     return "Unknown request/response";
 204 }
 205
 206 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
 207                      ISocketObservable *the_socket_observable,
 208                      Yaz_Proxy *parent)
 209     :
 210     Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 211 {
 212     m_PDU_Observable = the_PDU_Observable;
 213     m_socket_observable = the_socket_observable;
 214     m_client = 0;
 215     m_parent = parent;
 216     m_clientPool = 0;
 217     m_seqno = 1;
 218     m_keepalive_limit_bw = 500000;
 219     m_keepalive_limit_pdu = 1000;
 220     m_proxyTarget = 0;
 221     m_default_target = 0;
 222     m_proxy_negotiation_charset = 0;
 223     m_proxy_negotiation_lang = 0;
 224     m_proxy_negotiation_default_charset = 0;
 225     m_charset_converter = new Yaz_CharsetConverter;
 226     m_max_clients = 150;
 227     m_log_mask = 0;
 228     m_seed = time(0);
 229     m_client_idletime = 600;
 230     m_target_idletime = 600;
 231     m_optimize = xstrdup ("1");
 232     strcpy(m_session_str, "0 ");
 233     m_session_no = 0;
 234     m_bytes_sent = 0;
 235     m_bytes_recv = 0;
 236     m_bw_max = 0;
 237     m_pdu_max = 0;
 238     m_search_max = 0;
 239     m_connect_max = 0;
 240     m_timeout_mode = timeout_normal;
 241     m_timeout_gdu = 0;
 242     m_max_record_retrieve = 0;
 243     m_reconfig_flag = 0;
 244     m_config_fname = 0;
 245     m_request_no = 0;
 246     m_flag_invalid_session = 0;
 247     m_referenceId = 0;
 248     m_referenceId_mem = nmem_create();
 249     m_config = 0;
 250     m_marcxml_mode = none;
 251     m_stylesheet_xsp = 0;
 252     m_stylesheet_nprl = 0;
 253     m_stylesheet_apdu = 0;
 254     m_s2z_stylesheet = 0;
 255     m_s2z_database = 0;
 256     m_schema = 0;
 257     m_backend_type = 0;
 258     m_backend_charset = 0;
 259     m_frontend_type = 0;
 260     m_initRequest_apdu = 0;
 261     m_initRequest_mem = 0;
 262     m_initRequest_preferredMessageSize = 0;
 263     m_initRequest_maximumRecordSize = 0;
 264     m_initRequest_options = 0;
 265     m_initRequest_version = 0;
 266     m_initRequest_oi_negotiation_charsets = 0;
 267     m_initRequest_oi_negotiation_num_charsets = 0;
 268     m_initRequest_oi_negotiation_langs = 0;
 269     m_initRequest_oi_negotiation_num_langs = 0;
 270     m_initRequest_oi_negotiation_selected = 0;
 271     m_apdu_invalid_session = 0;
 272     m_mem_invalid_session = 0;
 273     m_s2z_odr_init = 0;
 274     m_s2z_odr_search = 0;
 275     m_s2z_init_apdu = 0;
 276     m_s2z_search_apdu = 0;
 277     m_s2z_present_apdu = 0;
 278     m_http_keepalive = 0;
 279     m_http_version = 0;
 280     m_soap_ns = 0;
 281     m_s2z_packing = Z_SRW_recordPacking_string;
 282 #if HAVE_GETTIMEOFDAY
 283     m_time_tv = xmalloc(sizeof(struct timeval));
 284     struct timeval *tv = (struct timeval *) m_time_tv;
 285     tv->tv_sec = 0;
 286     tv->tv_usec = 0;
 287 #else
 288     m_time_tv = 0;
 289 #endif
 290     m_usemarcon_ini_stage1 = 0;
 291     m_usemarcon_ini_stage2 = 0;
 292     m_usemarcon = new Yaz_usemarcon();
 293     if (!m_parent)
 294         low_socket_open();
 295     m_my_thread = 0;
 296     m_ref_count = 1;
 297     m_main_ptr_dec = false;
 298     m_peername = 0;
 299 }
 300
 301 void Yaz_Proxy::inc_ref()
 302 {
 303     m_ref_count++;
 304 }
 305
 306 Yaz_Proxy::~Yaz_Proxy()
 307 {
 308     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 309             m_bytes_sent, m_bytes_recv);
 310     nmem_destroy(m_initRequest_mem);
 311     nmem_destroy(m_mem_invalid_session);
 312     nmem_destroy(m_referenceId_mem);
 313
 314     xfree(m_proxyTarget);
 315     xfree(m_default_target);
 316     xfree(m_proxy_negotiation_charset);
 317     xfree(m_proxy_negotiation_lang);
 318     xfree(m_proxy_negotiation_default_charset);
 319     delete m_charset_converter;
 320     xfree(m_optimize);
 321
 322 #if HAVE_XSLT
 323     if (m_stylesheet_xsp)
 324         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 325 #endif
 326     xfree (m_time_tv);
 327
 328     xfree (m_peername);
 329     xfree (m_schema);
 330     xfree (m_backend_type);
 331     xfree (m_backend_charset);
 332     xfree (m_usemarcon_ini_stage1);
 333     xfree (m_usemarcon_ini_stage2);
 334     delete m_usemarcon;
 335     if (m_s2z_odr_init)
 336         odr_destroy(m_s2z_odr_init);
 337     if (m_s2z_odr_search)
 338         odr_destroy(m_s2z_odr_search);
 339     if (!m_parent)
 340         low_socket_close();
 341     if (!m_parent)
 342         delete m_my_thread;
 343     delete m_config;
 344 }
 345
 346 void Yaz_Proxy::set_debug_mode(int mode)
 347 {
 348     m_debug_mode = mode;
 349 }
 350
 351 int Yaz_Proxy::set_config(const char *config)
 352 {
 353     delete m_config;
 354     m_config = new Yaz_ProxyConfig();
 355     xfree(m_config_fname);
 356     m_config_fname = xstrdup(config);
 357     int r = m_config->read_xml(config);
 358     if (!r)
 359         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 360     return r;
 361 }
 362
 363 void Yaz_Proxy::set_default_target(const char *target)
 364 {
 365     xfree (m_default_target);
 366     m_default_target = 0;
 367     if (target)
 368         m_default_target = (char *) xstrdup (target);
 369 }
 370
 371 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang,
 372                                        const char *default_charset)
 373 {
 374     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
 375         "default charset to '%s', language to '%s'", m_session_str,
 376         charset?charset:"none",
 377         default_charset?default_charset:"none",
 378         lang?lang:"none");
 379     xfree (m_proxy_negotiation_charset);
 380     xfree (m_proxy_negotiation_lang);
 381     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 382     if (charset)
 383         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 384     if (lang)
 385         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 386     if (default_charset)
 387         m_proxy_negotiation_default_charset =
 388             (char *) xstrdup (default_charset);
 389 }
 390
 391 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 392 {
 393     if (m_parent)
 394         return m_parent->check_reconfigure();
 395
 396     Yaz_ProxyConfig *cfg = m_config;
 397     if (m_reconfig_flag)
 398     {
 399         yaz_log(YLOG_LOG, "reconfigure");
 400         yaz_log_reopen();
 401         if (m_config_fname && cfg)
 402         {
 403             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 404             int r = cfg->read_xml(m_config_fname);
 405             if (r)
 406                 yaz_log(YLOG_WARN, "reconfigure failed");
 407             else
 408             {
 409                 m_log_mask = 0;
 410                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 411             }
 412         }
 413         else
 414             yaz_log(YLOG_LOG, "reconfigure");
 415         m_reconfig_flag = 0;
 416     }
 417     return cfg;
 418 }
 419
 420 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 421                                         *the_PDU_Observable, int fd)
 422 {
 423     check_reconfigure();
 424     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 425                                          m_socket_observable, this);
 426     new_proxy->m_config = 0;
 427     new_proxy->m_config_fname = 0;
 428     new_proxy->timeout(m_client_idletime);
 429     new_proxy->m_target_idletime = m_target_idletime;
 430     new_proxy->set_default_target(m_default_target);
 431     new_proxy->m_max_clients = m_max_clients;
 432     new_proxy->m_log_mask = m_log_mask;
 433     new_proxy->set_APDU_log(get_APDU_log());
 434     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 435         new_proxy->set_APDU_yazlog(1);
 436     else
 437         new_proxy->set_APDU_yazlog(0);
 438     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 439     m_session_no++;
 440     new_proxy->m_peername = xstrdup(the_PDU_Observable->getpeername());
 441     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 442              new_proxy->m_peername);
 443     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 444         m_proxy_negotiation_lang, m_proxy_negotiation_default_charset);
 445     // create thread object the first time we get an incoming connection
 446     if (!m_my_thread)
 447         m_my_thread = new Msg_Thread(m_socket_observable, 1);
 448     new_proxy->m_my_thread = m_my_thread;
 449     return new_proxy;
 450 }
 451
 452 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 453 {
 454     int oid[OID_SIZE];
 455     Z_OtherInformationUnit *oi;
 456     struct oident ent;
 457     ent.proto = PROTO_Z3950;
 458     ent.oclass = CLASS_USERINFO;
 459     ent.value = (oid_value) VAL_COOKIE;
 460     assert (oid_ent_to_oid (&ent, oid));
 461
 462     if (oid_ent_to_oid (&ent, oid) &&
 463         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 464         oi->which == Z_OtherInfo_characterInfo)
 465         return oi->information.characterInfo;
 466     return 0;
 467 }
 468 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 469 {
 470     int oid[OID_SIZE];
 471     Z_OtherInformationUnit *oi;
 472     struct oident ent;
 473     ent.proto = PROTO_Z3950;
 474     ent.oclass = CLASS_USERINFO;
 475     ent.value = (oid_value) VAL_PROXY;
 476     if (oid_ent_to_oid (&ent, oid) &&
 477         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 478         oi->which == Z_OtherInfo_characterInfo)
 479         return oi->information.characterInfo;
 480     return 0;
 481 }
 482 const char *Yaz_Proxy::load_balance(const char **url)
 483 {
 484     int zurl_in_use[MAX_ZURL_PLEX];
 485     int zurl_in_spare[MAX_ZURL_PLEX];
 486     Yaz_ProxyClient *c;
 487     int i;
 488
 489     for (i = 0; i<MAX_ZURL_PLEX; i++)
 490     {
 491         zurl_in_use[i] = 0;
 492         zurl_in_spare[i] = 0;
 493     }
 494     for (c = m_parent->m_clientPool; c; c = c->m_next)
 495     {
 496         for (i = 0; url[i]; i++)
 497             if (!strcmp(url[i], c->get_hostname()))
 498             {
 499                 zurl_in_use[i]++;
 500                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 501                     zurl_in_spare[i]++;
 502             }
 503     }
 504     int min_use = 100000;
 505     int spare_for_min = 0;
 506     int max_spare = 0;
 507     const char *ret_min = 0;
 508     const char *ret_spare = 0;
 509     for (i = 0; url[i]; i++)
 510     {
 511         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 512                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 513         if (min_use > zurl_in_use[i])
 514         {
 515             ret_min = url[i];
 516             min_use = zurl_in_use[i];
 517             spare_for_min = zurl_in_spare[i];
 518         }
 519         if (max_spare < zurl_in_spare[i])
 520         {
 521             ret_spare = url[i];
 522             max_spare = zurl_in_spare[i];
 523         }
 524     }
 525     return ret_min;
 526 }
 527
 528 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 529                                        const char *proxy_host)
 530 {
 531     assert (m_parent);
 532     Yaz_Proxy *parent = m_parent;
 533     Yaz_ProxyClient *c = m_client;
 534
 535     if (!m_proxyTarget)
 536     {
 537         const char *url[MAX_ZURL_PLEX];
 538         Yaz_ProxyConfig *cfg = check_reconfigure();
 539         if (proxy_host)
 540         {
 541             if (parent && parent->m_debug_mode)
 542             {
 543                 // only to be enabled for debugging...
 544                 if (!strcmp(proxy_host, "stop"))
 545                     exit(0);
 546             }
 547             xfree(m_default_target);
 548             m_default_target = xstrdup(proxy_host);
 549         }
 550         proxy_host = m_default_target;
 551         int client_idletime = -1;
 552         const char *cql2rpn_fname = 0;
 553         const char *negotiation_charset = 0;
 554         const char *negotiation_lang = 0;
 555         const char *query_charset = 0;
 556         const char *default_client_query_charset = 0;
 557         url[0] = m_default_target;
 558         url[1] = 0;
 559         if (cfg)
 560         {
 561             int pre_init = 0;
 562             cfg->get_target_info(proxy_host, url, &m_bw_max,
 563                                  &m_pdu_max, &m_max_record_retrieve,
 564                                  &m_search_max, &m_connect_max,
 565                                  &m_target_idletime, &client_idletime,
 566                                  &parent->m_max_clients,
 567                                  &m_keepalive_limit_bw,
 568                                  &m_keepalive_limit_pdu,
 569                                  &pre_init,
 570                                  &cql2rpn_fname,
 571                                  &negotiation_charset,
 572                                  &negotiation_lang,
 573                                  &query_charset,
 574                                  &default_client_query_charset);
 575         }
 576         if (client_idletime != -1)
 577         {
 578             m_client_idletime = client_idletime;
 579             timeout(m_client_idletime);
 580         }
 581         if (cql2rpn_fname)
 582             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 583         if (negotiation_charset || negotiation_lang || default_client_query_charset)
 584         {
 585             set_proxy_negotiation(negotiation_charset,
 586                 negotiation_lang, default_client_query_charset);
 587         }
 588         m_charset_converter->set_target_query_charset(query_charset);
 589         if (!url[0])
 590         {
 591             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 592             return 0;
 593         }
 594         // we don't handle multiplexing for cookie session, so we just
 595         // pick the first one in this case (anonymous users will be able
 596         // to use any backend)
 597         if (cookie && *cookie)
 598             m_proxyTarget = (char*) xstrdup(url[0]);
 599         else
 600             m_proxyTarget = (char*) xstrdup(load_balance(url));
 601     }
 602     if (cookie && *cookie)
 603     {   // search in sessions with a cookie
 604         for (c = parent->m_clientPool; c; c = c->m_next)
 605         {
 606             assert (c->m_prev);
 607             assert (*c->m_prev == c);
 608             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 609                 !strcmp(m_proxyTarget, c->get_hostname()))
 610             {
 611                 // Found it in cache
 612                 // The following handles "cancel"
 613                 // If connection is busy (waiting for PDU) and
 614                 // we have an initRequest we can safely do re-open
 615                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 616                 {
 617                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 618                              c->get_hostname());
 619                     c->close();
 620                     c->m_init_flag = 0;
 621
 622                     c->m_last_ok = 0;
 623                     c->m_cache.clear();
 624                     c->m_last_resultCount = 0;
 625                     c->m_sr_transform = 0;
 626                     c->m_waiting = 0;
 627                     c->m_resultSetStartPoint = 0;
 628                     c->m_target_idletime = m_target_idletime;
 629                     if (c->client(m_proxyTarget))
 630                     {
 631                         delete c;
 632                         return 0;
 633                     }
 634                     c->timeout(30);
 635                 }
 636                 c->m_seqno = parent->m_seqno;
 637                 if (c->m_server && c->m_server != this)
 638                     c->m_server->m_client = 0;
 639                 c->m_server = this;
 640                 (parent->m_seqno)++;
 641                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 642                 return c;
 643             }
 644         }
 645     }
 646     else if (!c && apdu->which == Z_APDU_initRequest )
 647     {
 648         // anonymous sessions without cookie.
 649         // if authentication is set it is NOT anonymous se we can't share them.
 650         // If charset and lang negotiation is use it is NOT anonymous session too.
 651         for (c = parent->m_clientPool; c; c = c->m_next)
 652         {
 653             assert(c->m_prev);
 654             assert(*c->m_prev == c);
 655             if (c->m_server == 0 && c->m_cookie == 0 &&  c->m_waiting == 0
 656                 && c->compare_idAuthentication(apdu)
 657                 && c->compare_charset(apdu)
 658                 && !strcmp(m_proxyTarget, c->get_hostname()))
 659             {
 660                 // found it in cache
 661                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 662                          m_session_str, parent->m_seqno, c->get_hostname());
 663
 664                 c->m_seqno = parent->m_seqno;
 665                 assert(c->m_server == 0);
 666                 c->m_server = this;
 667
 668                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 669                     c->set_APDU_yazlog(1);
 670                 else
 671                     c->set_APDU_yazlog(0);
 672
 673                 (parent->m_seqno)++;
 674
 675                 parent->pre_init();
 676
 677                 return c;
 678             }
 679         }
 680     }
 681     if (!m_client)
 682     {
 683         if (apdu->which != Z_APDU_initRequest)
 684         {
 685             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 686             return 0;
 687         }
 688         // go through list of clients - and find the lowest/oldest one.
 689         Yaz_ProxyClient *c_min = 0;
 690         int min_seq = -1;
 691         int no_of_clients = 0;
 692         if (parent->m_clientPool)
 693             yaz_log (YLOG_DEBUG, "Existing sessions");
 694         for (c = parent->m_clientPool; c; c = c->m_next)
 695         {
 696             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 697                                c->m_waiting, c->get_hostname(),
 698                                c->m_cookie ? c->m_cookie : "");
 699             no_of_clients++;
 700             if (min_seq < 0 || c->m_seqno < min_seq)
 701             {
 702                 min_seq = c->m_seqno;
 703                 c_min = c;
 704             }
 705         }
 706         if (no_of_clients >= parent->m_max_clients)
 707         {
 708             c = c_min;
 709             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 710             {
 711                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 712                          m_session_str, parent->m_max_clients, c->m_seqno);
 713                 if (c->m_server && c->m_server != this)
 714                     delete c->m_server;   // PROBLEM: m_ref_count!
 715                 c->m_server = 0;
 716             }
 717             else
 718             {
 719                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 720                          m_session_str, parent->m_max_clients,
 721                          c->m_seqno, parent->m_seqno, c->get_hostname());
 722                 xfree (c->m_cookie);
 723                 c->m_cookie = 0;
 724                 if (cookie)
 725                     c->m_cookie = xstrdup(cookie);
 726                 c->m_seqno = parent->m_seqno;
 727                 if (c->m_server && c->m_server != this)
 728                 {
 729                     c->m_server->m_client = 0;
 730                     delete c->m_server;   // PROBLEM: m_ref_count!
 731                 }
 732                 (parent->m_seqno)++;
 733                 c->m_target_idletime = m_target_idletime;
 734                 c->timeout(m_target_idletime);
 735
 736                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 737                     c->set_APDU_yazlog(1);
 738                 else
 739                     c->set_APDU_yazlog(0);
 740
 741                 return c;
 742             }
 743         }
 744         else
 745         {
 746             yaz_log (YLOG_LOG, "%sNEW %d %s",
 747                      m_session_str, parent->m_seqno, m_proxyTarget);
 748             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 749             c->m_next = parent->m_clientPool;
 750             if (c->m_next)
 751                 c->m_next->m_prev = &c->m_next;
 752             parent->m_clientPool = c;
 753             c->m_prev = &parent->m_clientPool;
 754         }
 755
 756         xfree (c->m_cookie);
 757         c->m_cookie = 0;
 758         if (cookie)
 759             c->m_cookie = xstrdup(cookie);
 760
 761         c->m_seqno = parent->m_seqno;
 762         c->m_init_flag = 0;
 763         c->m_last_resultCount = 0;
 764         c->m_last_ok = 0;
 765         c->m_cache.clear();
 766         c->m_sr_transform = 0;
 767         c->m_waiting = 0;
 768         c->m_resultSetStartPoint = 0;
 769         (parent->m_seqno)++;
 770         if (c->client(m_proxyTarget))
 771         {
 772             delete c;
 773             return 0;
 774         }
 775         c->m_target_idletime = m_target_idletime;
 776         c->timeout(30);
 777
 778         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 779             c->set_APDU_yazlog(1);
 780         else
 781             c->set_APDU_yazlog(0);
 782
 783         c->set_idAuthentication(apdu);
 784     }
 785     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 786     return c;
 787 }
 788
 789 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 790 {
 791     int i;
 792     for (i = 0; i<num; i++)
 793     {
 794         oident *ent;
 795         Z_DefaultDiagFormat *r;
 796         Z_DiagRec *p = pp[i];
 797         if (p->which != Z_DiagRec_defaultFormat)
 798         {
 799             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 800             return;
 801         }
 802         else
 803             r = p->u.defaultFormat;
 804         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 805             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 806             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 807         switch (r->which)
 808         {
 809         case Z_DefaultDiagFormat_v2Addinfo:
 810             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 811                     m_session_str,
 812                     *r->condition, diagbib1_str(*r->condition),
 813                     r->u.v2Addinfo);
 814             break;
 815         case Z_DefaultDiagFormat_v3Addinfo:
 816             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 817                     m_session_str,
 818                     *r->condition, diagbib1_str(*r->condition),
 819                     r->u.v3Addinfo);
 820             break;
 821         }
 822     }
 823 }
 824
 825 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 826 {
 827     if (!m_stylesheet_xsp || p->num_records <= 0)
 828     {
 829         return 0;  /* no XSLT to be done ... */
 830     }
 831
 832     m_stylesheet_offset = 0;
 833     m_stylesheet_nprl = p;
 834     m_stylesheet_apdu = apdu;
 835     m_timeout_mode = timeout_xsl;
 836
 837     timeout(0);
 838     return 1;
 839 }
 840
 841 void Yaz_Proxy::convert_xsl_delay()
 842 {
 843 #if HAVE_XSLT
 844     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 845     if (npr->which == Z_NamePlusRecord_databaseRecord)
 846     {
 847         Z_External *r = npr->u.databaseRecord;
 848         if (r->which == Z_External_octet)
 849         {
 850 #if 0
 851             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 852 #endif
 853             xmlDocPtr res, doc = xmlParseMemory(
 854                 (char*) r->u.octet_aligned->buf,
 855                 r->u.octet_aligned->len);
 856
 857
 858             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 859                     m_session_str, m_stylesheet_offset);
 860             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 861                                       doc, 0);
 862
 863             if (res)
 864             {
 865                 xmlChar *out_buf;
 866                 int out_len;
 867                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 868
 869                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 870                     u.databaseRecord =
 871                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 872                                  (char*) out_buf, out_len);
 873                 xmlFree(out_buf);
 874                 xmlFreeDoc(res);
 875             }
 876
 877             xmlFreeDoc(doc);
 878         }
 879     }
 880 #endif
 881     m_stylesheet_offset++;
 882     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 883     {
 884         m_timeout_mode = timeout_normal;
 885         m_stylesheet_nprl = 0;
 886 #if HAVE_XSLT
 887         if (m_stylesheet_xsp)
 888             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 889 #endif
 890         m_stylesheet_xsp = 0;
 891         timeout(m_client_idletime);
 892         send_PDU_convert(m_stylesheet_apdu);
 893     }
 894     else
 895         timeout(0);
 896 }
 897
 898 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 899 {
 900     if (m_frontend_type != VAL_NONE)
 901     {
 902         int i;
 903         for (i = 0; i < p->num_records; i++)
 904         {
 905             Z_NamePlusRecord *npr = p->records[i];
 906             if (npr->which == Z_NamePlusRecord_databaseRecord)
 907             {
 908                 Z_External *r = npr->u.databaseRecord;
 909                 if (r->which == Z_External_octet)
 910                 {
 911 #if HAVE_USEMARCON
 912                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 913                     {
 914                         if (!m_usemarcon->m_stage1)
 915                         {
 916                             m_usemarcon->m_stage1 = new CDetails();
 917                         }
 918                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 919                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 920                         int res = m_usemarcon->m_stage1->Start();
 921                         if (res == 0)
 922                         {
 923                             char *converted;
 924                             int convlen;
 925                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 926                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 927                             {
 928                                 if (!m_usemarcon->m_stage2)
 929                                 {
 930                                     m_usemarcon->m_stage2 = new CDetails();
 931                                 }
 932                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 933                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 934                                 res = m_usemarcon->m_stage2->Start();
 935                                 if (res == 0)
 936                                 {
 937                                     free(converted);
 938                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 939                                 }
 940                                 else
 941                                 {
 942                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 943                                 }
 944                             }
 945                             npr->u.databaseRecord =
 946                                 z_ext_record(odr_encode(),
 947                                              m_frontend_type,
 948                                              converted,
 949                                              strlen(converted));
 950                             free(converted);
 951                         }
 952                         else
 953                         {
 954                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 955                         }
 956                         continue;
 957                     }
 958 #endif
 959 /* HAVE_USEMARCON */
 960                     npr->u.databaseRecord =
 961                         z_ext_record(odr_encode(),
 962                                      m_frontend_type,
 963                                      (char*) r->u.octet_aligned->buf,
 964                                      r->u.octet_aligned->len);
 965                 }
 966             }
 967         }
 968     }
 969 }
 970
 971 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
 972                                         const char *backend_charset)
 973 {
 974     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
 975     int sel =   m_charset_converter->get_client_charset_selected();
 976     const char *client_record_charset =
 977         m_charset_converter->get_client_query_charset();
 978     if (sel && backend_charset && client_record_charset &&
 979         strcmp(backend_charset, client_record_charset))
 980     {
 981         int i;
 982         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
 983                                         backend_charset);
 984         yaz_marc_t mt = yaz_marc_create();
 985         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
 986         yaz_marc_iconv(mt, cd);
 987         for (i = 0; i < p->num_records; i++)
 988         {
 989             Z_NamePlusRecord *npr = p->records[i];
 990             if (npr->which == Z_NamePlusRecord_databaseRecord)
 991             {
 992                 Z_External *r = npr->u.databaseRecord;
 993                 oident *ent = oid_getentbyoid(r->direct_reference);
 994                 if (!ent || ent->value == VAL_NONE)
 995                     continue;
 996
 997                 if (ent->value == VAL_SUTRS)
 998                 {
 999                     WRBUF w = wrbuf_alloc();
1000
1001                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
1002                                       r->u.octet_aligned->len);
1003                     npr->u.databaseRecord =
1004                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
1005                                      wrbuf_len(w));
1006                     wrbuf_free(w, 1);
1007                 }
1008                 else if (ent->value == VAL_TEXT_XML)
1009                 {
1010                     ;
1011                 }
1012                 else if (r->which == Z_External_octet)
1013                 {
1014                     int rlen;
1015                     char *result;
1016                     if (yaz_marc_decode_buf(mt,
1017                                             (char*) r->u.octet_aligned->buf,
1018                                             r->u.octet_aligned->len,
1019                                             &result, &rlen))
1020                     {
1021                         npr->u.databaseRecord =
1022                             z_ext_record(odr_encode(), ent->value, result, rlen);
1023                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1024                                 m_session_str);
1025                     }
1026                 }
1027             }
1028         }
1029         if (cd)
1030             yaz_iconv_close(cd);
1031         yaz_marc_destroy(mt);
1032     }
1033     else
1034     {
1035         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
1036     }
1037 }
1038
1039 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1040                                    const char *backend_charset)
1041 {
1042     int i;
1043     if (!backend_charset)
1044         backend_charset = "MARC-8";
1045     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1046     yaz_marc_t mt = yaz_marc_create();
1047     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1048     yaz_marc_iconv(mt, cd);
1049     for (i = 0; i < p->num_records; i++)
1050     {
1051         Z_NamePlusRecord *npr = p->records[i];
1052         if (npr->which == Z_NamePlusRecord_databaseRecord)
1053         {
1054             Z_External *r = npr->u.databaseRecord;
1055             if (r->which == Z_External_OPAC)
1056             {
1057                 WRBUF w = wrbuf_alloc();
1058
1059                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
1060                 npr->u.databaseRecord = z_ext_record(
1061                     odr_encode(), VAL_TEXT_XML,
1062                     wrbuf_buf(w), wrbuf_len(w)
1063                     );
1064                 wrbuf_free(w, 1);
1065             }
1066             else if (r->which == Z_External_octet)
1067             {
1068                 int rlen;
1069                 char *result;
1070                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1071                                         r->u.octet_aligned->len,
1072                                         &result, &rlen))
1073                 {
1074                     npr->u.databaseRecord =
1075                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
1076                 }
1077             }
1078         }
1079     }
1080     if (cd)
1081         yaz_iconv_close(cd);
1082     yaz_marc_destroy(mt);
1083 }
1084
1085 void Yaz_Proxy::logtime()
1086 {
1087 #if HAVE_GETTIMEOFDAY
1088     struct timeval *tv = (struct timeval*) m_time_tv;
1089     if (tv->tv_sec)
1090     {
1091         struct timeval tv1;
1092         gettimeofday(&tv1, 0);
1093         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1094             (tv1.tv_usec - tv->tv_usec);
1095         if (diff >= 0)
1096             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1097                     diff/1000000, (diff/1000)%1000);
1098     }
1099     tv->tv_sec = 0;
1100     tv->tv_usec = 0;
1101 #endif
1102 }
1103
1104 int Yaz_Proxy::send_http_response(int code)
1105 {
1106     ODR o = odr_encode();
1107     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1108     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1109     if (m_http_version)
1110         hres->version = odr_strdup(o, m_http_version);
1111     if (m_http_keepalive)
1112         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1113     else
1114         timeout(0);
1115
1116     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1117     {
1118         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1119                  gdu_name(gdu));
1120     }
1121     int len;
1122     int r = send_GDU(gdu, &len);
1123     m_bytes_sent += len;
1124     m_bw_stat.add_bytes(len);
1125     logtime();
1126
1127     recv_GDU_more(true);
1128
1129     return r;
1130 }
1131
1132 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */)
1133 {
1134     ODR o = odr_encode();
1135     const char *ctype = "text/xml";
1136     Z_GDU *gdu = z_get_HTTP_Response(o, http_code);
1137     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1138     if (m_http_version)
1139         hres->version = odr_strdup(o, m_http_version);
1140     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1141     if (m_http_keepalive)
1142         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1143     else
1144         timeout(0);
1145     if (http_code == 401)
1146         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\"");
1147
1148     static Z_SOAP_Handler soap_handlers[2] = {
1149 #if HAVE_XSLT
1150         {"http://www.loc.gov/zing/srw/", 0,
1151          (Z_SOAP_fun) yaz_srw_codec},
1152 #endif
1153         {0, 0, 0}
1154     };
1155
1156     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1157     soap_package->which = Z_SOAP_generic;
1158     soap_package->u.generic =
1159         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1160     soap_package->u.generic->no = 0;
1161     soap_package->u.generic->ns = soap_handlers[0].ns;
1162     soap_package->u.generic->p = (void *) srw_pdu;
1163     soap_package->ns = m_soap_ns;
1164     z_soap_codec_enc_xsl(o, &soap_package,
1165                          &hres->content_buf, &hres->content_len,
1166                          soap_handlers, 0, m_s2z_stylesheet);
1167     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1168     {
1169         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1170                  gdu_name(gdu));
1171     }
1172     int len;
1173     int r = send_GDU(gdu, &len);
1174     m_bytes_sent += len;
1175     m_bw_stat.add_bytes(len);
1176     logtime();
1177
1178     recv_GDU_more(true);
1179
1180     return r;
1181 }
1182
1183 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1184 {
1185     ODR o = odr_encode();
1186     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1187         odr_malloc(o, sizeof(*diagnostic));
1188     int num_diagnostic = 1;
1189     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1190     return send_srw_search_response(diagnostic, num_diagnostic, srw_error == 3 ? 401 : 200);
1191 }
1192
1193 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1194                              Z_DefaultDiagFormat *ddf)
1195 {
1196     int bib1_code = *ddf->condition;
1197     if (bib1_code == 109)
1198         return 404;
1199     srw_res->num_diagnostics = 1;
1200     srw_res->diagnostics = (Z_SRW_diagnostic *)
1201         odr_malloc(o, sizeof(*srw_res->diagnostics));
1202     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1203                           yaz_diag_bib1_to_srw(*ddf->condition),
1204                           ddf->u.v2Addinfo);
1205     return 0;
1206 }
1207
1208 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1209 {
1210     ODR o = odr_encode();
1211     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1212     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1213
1214     srw_res->numberOfRecords = odr_intdup (o, hits);
1215     if (records && records->which == Z_Records_DBOSD)
1216     {
1217         srw_res->num_records =
1218             records->u.databaseOrSurDiagnostics->num_records;
1219         int i;
1220         srw_res->records = (Z_SRW_record *)
1221             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1222         for (i = 0; i < srw_res->num_records; i++)
1223         {
1224             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1225             if (npr->which != Z_NamePlusRecord_databaseRecord)
1226             {
1227                 srw_res->records[i].recordSchema = "diagnostic";
1228                 srw_res->records[i].recordPacking = m_s2z_packing;
1229                 srw_res->records[i].recordData_buf = "67";
1230                 srw_res->records[i].recordData_len = 2;
1231                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1232                 continue;
1233             }
1234             Z_External *r = npr->u.databaseRecord;
1235             oident *ent = oid_getentbyoid(r->direct_reference);
1236             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1237             {
1238                 srw_res->records[i].recordSchema = m_schema;
1239                 srw_res->records[i].recordPacking = m_s2z_packing;
1240                 srw_res->records[i].recordData_buf = (char*)
1241                     r->u.octet_aligned->buf;
1242                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1243                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1244             }
1245             else
1246             {
1247                 srw_res->records[i].recordSchema = "diagnostic";
1248                 srw_res->records[i].recordPacking = m_s2z_packing;
1249                 srw_res->records[i].recordData_buf = "67";
1250                 srw_res->records[i].recordData_len = 2;
1251                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1252             }
1253         }
1254     }
1255     if (records && records->which == Z_Records_NSD)
1256     {
1257         int http_code;
1258         http_code = z_to_srw_diag(odr_encode(), srw_res,
1259                                    records->u.nonSurrogateDiagnostic);
1260         if (http_code)
1261             return send_http_response(http_code);
1262     }
1263     return send_srw_response(srw_pdu);
1264
1265 }
1266
1267 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1268                                         int num_diagnostics, int http_code /* = 200 */)
1269 {
1270     ODR o = odr_encode();
1271     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1272     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1273
1274     srw_res->num_diagnostics = num_diagnostics;
1275     srw_res->diagnostics = diagnostics;
1276     return send_srw_response(srw_pdu, http_code);
1277 }
1278
1279 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1280                                         int num_diagnostics)
1281 {
1282     Yaz_ProxyConfig *cfg = check_reconfigure();
1283     if (cfg)
1284     {
1285         int len;
1286         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1287                                        m_s2z_database, &len);
1288         if (b)
1289         {
1290             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1291             Z_SRW_explainResponse *er = res->u.explain_response;
1292
1293             er->record.recordData_buf = b;
1294             er->record.recordData_len = len;
1295             er->record.recordPacking = m_s2z_packing;
1296             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1297
1298             er->diagnostics = diagnostics;
1299             er->num_diagnostics = num_diagnostics;
1300             return send_srw_response(res);
1301         }
1302     }
1303     return send_http_response(404);
1304 }
1305
1306 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1307 {
1308     if (m_http_version)
1309     {
1310         if (apdu->which == Z_APDU_initResponse)
1311         {
1312             Z_InitResponse *res = apdu->u.initResponse;
1313             if (*res->result == 0)
1314             {
1315                 send_to_srw_client_error(3, 0);
1316             }
1317             else if (!m_s2z_search_apdu)
1318             {
1319                 send_srw_explain_response(0, 0);
1320             }
1321             else
1322             {
1323                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1324             }
1325         }
1326         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1327         {
1328             m_s2z_search_apdu = 0;
1329             Z_SearchResponse *res = apdu->u.searchResponse;
1330             m_s2z_hit_count = *res->resultCount;
1331             if (res->records && res->records->which == Z_Records_NSD)
1332             {
1333                 send_to_srw_client_ok(0, res->records, 1);
1334             }
1335             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1336             {
1337                 // adjust
1338                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1339
1340                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1341                 {
1342                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1343                         > m_s2z_hit_count)
1344                         *pr->numberOfRecordsRequested =
1345                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1346                 }
1347                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1348             }
1349             else
1350             {
1351                 m_s2z_present_apdu = 0;
1352                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1353             }
1354         }
1355         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1356         {
1357             int start =
1358                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1359
1360             m_s2z_present_apdu = 0;
1361             Z_PresentResponse *res = apdu->u.presentResponse;
1362             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1363         }
1364     }
1365     else
1366     {
1367         int len = 0;
1368         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1369             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1370                      apdu_name(apdu));
1371         int r = send_Z_PDU(apdu, &len);
1372         m_bytes_sent += len;
1373         m_bw_stat.add_bytes(len);
1374         logtime();
1375         return r;
1376     }
1377     return 0;
1378 }
1379
1380 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1381 {
1382     int kill_session = 0;
1383     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1384
1385     if (new_id)
1386         *new_id = m_referenceId;
1387
1388     if (apdu->which == Z_APDU_searchResponse)
1389     {
1390         Z_SearchResponse *sr = apdu->u.searchResponse;
1391         Z_Records *p = sr->records;
1392         if (p && p->which == Z_Records_NSD)
1393         {
1394             Z_DiagRec dr, *dr_p = &dr;
1395             dr.which = Z_DiagRec_defaultFormat;
1396             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1397
1398             *sr->searchStatus = 0;
1399             display_diagrecs(&dr_p, 1);
1400         }
1401         else
1402         {
1403             if (p && p->which == Z_Records_DBOSD)
1404             {
1405                 if (m_backend_type
1406 #if HAVE_USEMARCON
1407                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1408 #endif
1409                     )
1410                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1411                 if (m_marcxml_mode == marcxml)
1412                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1413                                        m_backend_charset);
1414                 else
1415                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1416                                             m_backend_charset);
1417                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1418                     return 0;
1419
1420             }
1421             if (sr->resultCount)
1422             {
1423                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1424                         *sr->resultCount);
1425                 if (*sr->resultCount < 0)
1426                 {
1427                     m_flag_invalid_session = 1;
1428                     kill_session = 1;
1429
1430                     *sr->searchStatus = 0;
1431                     sr->records =
1432                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1433                     *sr->resultCount = 0;
1434                 }
1435             }
1436         }
1437     }
1438     else if (apdu->which == Z_APDU_presentResponse)
1439     {
1440         Z_PresentResponse *sr = apdu->u.presentResponse;
1441         Z_Records *p = sr->records;
1442         if (p && p->which == Z_Records_NSD)
1443         {
1444             Z_DiagRec dr, *dr_p = &dr;
1445             dr.which = Z_DiagRec_defaultFormat;
1446             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1447             if (*sr->presentStatus == Z_PresentStatus_success)
1448                 *sr->presentStatus = Z_PresentStatus_failure;
1449             display_diagrecs(&dr_p, 1);
1450         }
1451         if (p && p->which == Z_Records_DBOSD)
1452         {
1453             if (m_backend_type
1454 #if HAVE_USEMARCON
1455                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1456 #endif
1457                 )
1458                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1459             if (m_marcxml_mode == marcxml)
1460                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1461                                    m_backend_charset);
1462             else
1463                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1464                                         m_backend_charset);
1465             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1466                 return 0;
1467         }
1468     }
1469     else if (apdu->which == Z_APDU_initResponse)
1470     {
1471         //Get and check negotiation record
1472         //from init response.
1473         handle_charset_lang_negotiation(apdu);
1474
1475         if (m_initRequest_options)
1476         {
1477             Z_Options *nopt =
1478                 (Odr_bitmask *)odr_malloc(odr_encode(),
1479                                           sizeof(Odr_bitmask));
1480             ODR_MASK_ZERO(nopt);
1481
1482             int i;
1483             for (i = 0; i<24; i++)
1484                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1485                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1486                     ODR_MASK_SET(nopt, i);
1487             apdu->u.initResponse->options = nopt;
1488         }
1489         if (m_initRequest_version)
1490         {
1491             Z_ProtocolVersion *nopt =
1492                 (Odr_bitmask *)odr_malloc(odr_encode(),
1493                                           sizeof(Odr_bitmask));
1494             ODR_MASK_ZERO(nopt);
1495
1496             int i;
1497             for (i = 0; i<8; i++)
1498                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1499                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1500                     ODR_MASK_SET(nopt, i);
1501             apdu->u.initResponse->protocolVersion = nopt;
1502         }
1503         apdu->u.initResponse->preferredMessageSize =
1504             odr_intdup(odr_encode(),
1505                        m_client->m_initResponse_preferredMessageSize >
1506                        m_initRequest_preferredMessageSize ?
1507                        m_initRequest_preferredMessageSize :
1508                        m_client->m_initResponse_preferredMessageSize);
1509         apdu->u.initResponse->maximumRecordSize =
1510             odr_intdup(odr_encode(),
1511                        m_client->m_initResponse_maximumRecordSize >
1512                        m_initRequest_maximumRecordSize ?
1513                        m_initRequest_maximumRecordSize :
1514                        m_client->m_initResponse_maximumRecordSize);
1515     }
1516
1517     int r = send_PDU_convert(apdu);
1518     if (r)
1519         return r;
1520     if (kill_session)
1521     {
1522         delete m_client;
1523         m_client = 0;
1524         m_parent->pre_init();
1525     }
1526     return r;
1527 }
1528
1529 void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu)
1530 {
1531     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1532
1533     odr_reset(m_idAuthentication_odr);
1534     z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0);
1535     m_idAuthentication_ber_buf =
1536         odr_getbuf(m_idAuthentication_odr,
1537                    &m_idAuthentication_ber_size, 0);
1538 }
1539
1540 bool Yaz_ProxyClient::compare_charset(Z_APDU *apdu)
1541 {
1542     return true;
1543 }
1544
1545 bool Yaz_ProxyClient::compare_idAuthentication(Z_APDU *apdu)
1546 {
1547     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1548     ODR odr = odr_createmem(ODR_ENCODE);
1549
1550     z_IdAuthentication(odr, &t, 1, 0);
1551     int sz;
1552     char *buf = odr_getbuf(odr, &sz, 0);
1553     if (buf && m_idAuthentication_ber_buf
1554         && sz == m_idAuthentication_ber_size
1555         && !memcmp(m_idAuthentication_ber_buf, buf, sz))
1556     {
1557         odr_destroy(odr);
1558         return true;
1559     }
1560     odr_destroy(odr);
1561     if (!buf && !m_idAuthentication_ber_buf)
1562         return true;
1563     return false;
1564 }
1565
1566 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1567 {
1568     int len = 0;
1569     const char *apdu_name_tmp = apdu_name(apdu);
1570     int r = send_Z_PDU(apdu, &len);
1571     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1572         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1573                  get_session_str(),
1574                  apdu_name_tmp, get_hostname(), len);
1575     m_bytes_sent += len;
1576     return r;
1577 }
1578
1579 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1580 {
1581     if (apdu->which == Z_APDU_presentRequest)
1582     {
1583         Z_PresentRequest *pr = apdu->u.presentRequest;
1584         int toget = *pr->numberOfRecordsRequested;
1585         int start = *pr->resultSetStartPoint;
1586
1587         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1588                 pr->resultSetId, start, toget);
1589
1590         if (*m_parent->m_optimize == '0')
1591             return apdu;
1592
1593         if (!m_client->m_last_resultSetId)
1594         {
1595             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1596             new_apdu->u.presentResponse->records =
1597                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1598                                                pr->resultSetId);
1599             send_to_client(new_apdu);
1600             return 0;
1601         }
1602         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1603         {
1604             if (start+toget-1 > m_client->m_last_resultCount)
1605             {
1606                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1607                 new_apdu->u.presentResponse->records =
1608                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1609                 send_to_client(new_apdu);
1610                 return 0;
1611             }
1612             Z_NamePlusRecordList *npr;
1613 #if 0
1614             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1615                     m_session_str, start, toget, yaz_z3950oid_to_str(
1616                         pr->preferredRecordSyntax, &oclass));
1617 #endif
1618             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1619                                           pr->preferredRecordSyntax,
1620                                           pr->recordComposition))
1621             {
1622                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1623                          m_session_str);
1624                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1625                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1626
1627                 new_apdu->u.presentResponse->numberOfRecordsReturned
1628                     = odr_intdup(odr_encode(), toget);
1629
1630                 new_apdu->u.presentResponse->records = (Z_Records*)
1631                     odr_malloc(odr_encode(), sizeof(Z_Records));
1632                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1633                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1634                 new_apdu->u.presentResponse->nextResultSetPosition =
1635                     odr_intdup(odr_encode(), start+toget);
1636
1637                 send_to_client(new_apdu);
1638                 return 0;
1639             }
1640         }
1641     }
1642
1643     if (apdu->which != Z_APDU_searchRequest)
1644         return apdu;
1645     Z_SearchRequest *sr = apdu->u.searchRequest;
1646     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1647     Yaz_Z_Databases this_databases;
1648
1649     this_databases.set(sr->num_databaseNames, (const char **)
1650                        sr->databaseNames);
1651
1652     this_query->set_Z_Query(sr->query);
1653
1654     char query_str[120];
1655     this_query->print(query_str, sizeof(query_str)-1);
1656     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1657
1658     if (*m_parent->m_optimize != '0' &&
1659         m_client->m_last_ok && m_client->m_last_query &&
1660         m_client->m_last_query->match(this_query) &&
1661         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1662         m_client->m_last_databases.match(this_databases))
1663     {
1664         delete this_query;
1665         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1666             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1667         {
1668             Z_NamePlusRecordList *npr;
1669             int toget = *sr->mediumSetPresentNumber;
1670             Z_RecordComposition *comp = 0;
1671
1672             if (toget > m_client->m_last_resultCount)
1673                 toget = m_client->m_last_resultCount;
1674
1675             if (sr->mediumSetElementSetNames)
1676             {
1677                 comp = (Z_RecordComposition *)
1678                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1679                 comp->which = Z_RecordComp_simple;
1680                 comp->u.simple = sr->mediumSetElementSetNames;
1681             }
1682
1683             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1684                                           sr->preferredRecordSyntax, comp))
1685             {
1686                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1687                          m_session_str);
1688                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1689                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1690                 new_apdu->u.searchResponse->resultCount =
1691                     &m_client->m_last_resultCount;
1692
1693                 new_apdu->u.searchResponse->numberOfRecordsReturned
1694                     = odr_intdup(odr_encode(), toget);
1695
1696                 new_apdu->u.searchResponse->presentStatus =
1697                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1698                 new_apdu->u.searchResponse->records = (Z_Records*)
1699                     odr_malloc(odr_encode(), sizeof(Z_Records));
1700                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1701                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1702                 new_apdu->u.searchResponse->nextResultSetPosition =
1703                     odr_intdup(odr_encode(), toget+1);
1704                 send_to_client(new_apdu);
1705                 return 0;
1706             }
1707             else
1708             {
1709                 // medium Set
1710                 // send present request (medium size)
1711                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1712                          m_session_str);
1713
1714                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1715                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1716                 pr->referenceId = sr->referenceId;
1717                 pr->resultSetId = sr->resultSetName;
1718                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1719                 *pr->numberOfRecordsRequested = toget;
1720                 pr->recordComposition = comp;
1721                 m_client->m_sr_transform = 1;
1722                 return new_apdu;
1723             }
1724         }
1725         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1726             m_client->m_last_resultCount <= 0)
1727         {
1728             // large set. Return pseudo-search response immediately
1729             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1730                      m_session_str);
1731             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1732             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1733             new_apdu->u.searchResponse->resultCount =
1734                 &m_client->m_last_resultCount;
1735             send_to_client(new_apdu);
1736             return 0;
1737         }
1738         else
1739         {
1740             Z_NamePlusRecordList *npr;
1741             int toget = m_client->m_last_resultCount;
1742             Z_RecordComposition *comp = 0;
1743             // small set
1744             // send a present request (small set)
1745
1746             if (sr->smallSetElementSetNames)
1747             {
1748                 comp = (Z_RecordComposition *)
1749                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1750                 comp->which = Z_RecordComp_simple;
1751                 comp->u.simple = sr->smallSetElementSetNames;
1752             }
1753
1754             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1755                                           sr->preferredRecordSyntax, comp))
1756             {
1757                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1758                          m_session_str);
1759                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1760                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1761                 new_apdu->u.searchResponse->resultCount =
1762                     &m_client->m_last_resultCount;
1763
1764                 new_apdu->u.searchResponse->numberOfRecordsReturned
1765                     = odr_intdup(odr_encode(), toget);
1766
1767                 new_apdu->u.searchResponse->presentStatus =
1768                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1769                 new_apdu->u.searchResponse->records = (Z_Records*)
1770                     odr_malloc(odr_encode(), sizeof(Z_Records));
1771                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1772                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1773                 new_apdu->u.searchResponse->nextResultSetPosition =
1774                     odr_intdup(odr_encode(), toget+1);
1775                 send_to_client(new_apdu);
1776                 return 0;
1777             }
1778             else
1779             {
1780                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1781                          m_session_str);
1782                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1783                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1784                 pr->referenceId = sr->referenceId;
1785                 pr->resultSetId = sr->resultSetName;
1786                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1787                 *pr->numberOfRecordsRequested = toget;
1788                 pr->recordComposition = comp;
1789                 m_client->m_sr_transform = 1;
1790                 return new_apdu;
1791             }
1792         }
1793     }
1794     else  // query doesn't match
1795     {
1796         delete m_client->m_last_query;
1797         m_client->m_last_query = this_query;
1798         m_client->m_last_ok = 0;
1799         m_client->m_cache.clear();
1800         m_client->m_resultSetStartPoint = 0;
1801
1802         xfree (m_client->m_last_resultSetId);
1803         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1804
1805         m_client->m_last_databases.set(sr->num_databaseNames,
1806                                        (const char **) sr->databaseNames);
1807     }
1808     return apdu;
1809 }
1810
1811
1812 void Yaz_Proxy::inc_request_no()
1813 {
1814     char *cp = strchr(m_session_str, ' ');
1815     m_request_no++;
1816     if (cp)
1817         sprintf(cp+1, "%d ", m_request_no);
1818 }
1819
1820 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1821 {
1822     inc_request_no();
1823
1824     m_bytes_recv += len;
1825
1826     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1827         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1828                  m_session_str, gdu_name(apdu), len);
1829
1830 #if HAVE_GETTIMEOFDAY
1831     gettimeofday((struct timeval *) m_time_tv, 0);
1832 #endif
1833     m_bw_stat.add_bytes(len);
1834     m_pdu_stat.add_bytes(1);
1835
1836     GDU *gdu = new GDU(apdu);
1837     m_in_queue.enqueue(gdu);
1838
1839     recv_GDU_more(false);
1840 }
1841
1842 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
1843 {
1844     int bw_total = m_bw_stat.get_total();
1845     int pdu_total = m_pdu_stat.get_total();
1846     int reduce = 0;
1847
1848     assert(m_timeout_mode == timeout_busy);
1849     assert(m_timeout_gdu == 0);
1850
1851     if (m_bw_max)
1852     {
1853         if (bw_total > m_bw_max)
1854         {
1855             reduce = (bw_total/m_bw_max);
1856         }
1857     }
1858     if (m_pdu_max)
1859     {
1860         if (pdu_total > m_pdu_max)
1861         {
1862             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1863             reduce = (reduce > nreduce) ? reduce : nreduce;
1864         }
1865     }
1866     m_http_version = 0;
1867
1868 #if 0
1869     /* uncomment to force a big reduce */
1870     m_timeout_mode = timeout_reduce;
1871     m_timeout_gdu = gdu;
1872     timeout(3);       // call us reduce seconds later
1873     return;
1874 #endif
1875     if (reduce)
1876     {
1877         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1878                 m_session_str, reduce, bw_total, pdu_total,
1879                 m_bw_max, m_pdu_max);
1880
1881         m_timeout_mode = timeout_reduce;
1882         m_timeout_gdu = gdu;
1883         timeout(reduce);       // call us reduce seconds later
1884     }
1885     else
1886         recv_GDU_normal(gdu);
1887 }
1888
1889 void Yaz_Proxy::recv_GDU_more(bool normal)
1890 {
1891     GDU *g;
1892     if (normal && m_timeout_mode == timeout_busy)
1893         m_timeout_mode = timeout_normal;
1894     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
1895     {
1896         m_timeout_mode = timeout_busy;
1897         recv_GDU_reduce(g);
1898     }
1899 }
1900
1901 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
1902 {
1903     Z_GDU *apdu = 0;
1904     gdu->move_away_gdu(odr_decode(), &apdu);
1905     delete gdu;
1906
1907     if (apdu->which == Z_GDU_Z3950)
1908         handle_incoming_Z_PDU(apdu->u.z3950);
1909     else if (apdu->which == Z_GDU_HTTP_Request)
1910         handle_incoming_HTTP(apdu->u.HTTP_Request);
1911 }
1912
1913 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1914 {
1915     if (m_max_record_retrieve)
1916     {
1917         if (apdu->which == Z_APDU_presentRequest)
1918         {
1919             Z_PresentRequest *pr = apdu->u.presentRequest;
1920             if (pr->numberOfRecordsRequested &&
1921                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1922                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1923         }
1924     }
1925 }
1926
1927 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1928 {
1929     if (apdu->which == Z_APDU_initRequest)
1930     {
1931         yaz_log(YLOG_LOG, "%shandle_charset_lang_negotiation",
1932                 m_session_str);
1933         if (m_initRequest_options &&
1934             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1935             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1936         {
1937             // There is no negotiation proposal from
1938             // client's side. OK. The proxy negotiation
1939             // in use, only.
1940             Z_InitRequest *initRequest = apdu->u.initRequest;
1941             Z_OtherInformation **otherInfo;
1942             Z_OtherInformationUnit *oi;
1943             get_otherInfoAPDU(apdu, &otherInfo);
1944             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1945             if (oi)
1946             {
1947                 ODR_MASK_SET(initRequest->options,
1948                     Z_Options_negotiationModel);
1949                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1950                 oi->information.externallyDefinedInfo =
1951                 yaz_set_proposal_charneg(odr_encode(),
1952                     (const char**)&m_proxy_negotiation_charset,
1953                     m_proxy_negotiation_charset ? 1:0,
1954                     (const char**)&m_proxy_negotiation_lang,
1955                     m_proxy_negotiation_lang ? 1:0,
1956                     1);
1957             }
1958         }
1959         else if (m_initRequest_options &&
1960                  ODR_MASK_GET(m_initRequest_options,
1961                               Z_Options_negotiationModel) &&
1962                  m_charset_converter->get_target_query_charset())
1963         {
1964             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1965                     m_session_str,
1966                     m_charset_converter->get_target_query_charset());
1967             Z_InitRequest *initRequest = apdu->u.initRequest;
1968             Z_CharSetandLanguageNegotiation *negotiation =
1969                 yaz_get_charneg_record (initRequest->otherInfo);
1970             if (negotiation &&
1971                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1972             {
1973                 NMEM nmem = nmem_create();
1974                 char **charsets = 0;
1975                 int num_charsets = 0;
1976                 char **langs = 0;
1977                 int num_langs = 0;
1978                 int selected = 0;
1979                 yaz_get_proposal_charneg (nmem, negotiation,
1980                                           &charsets, &num_charsets,
1981                                           &langs, &num_langs, &selected);
1982                 int i;
1983                 for (i = 0; i<num_charsets; i++)
1984                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
1985                             charsets[i]);
1986                 for (i = 0; i<num_langs; i++)
1987                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
1988                             langs[i]);
1989
1990                 const char *t_charset =
1991                     m_charset_converter->get_target_query_charset();
1992                 // sweep through charsets and pick the first supported
1993                 // conversion
1994                 for (i = 0; i<num_charsets; i++)
1995                 {
1996                     const char *c_charset = charsets[i];
1997                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
1998                         break;
1999                 }
2000                 if (i != num_charsets)
2001                 {
2002                     // got one .. set up ODR for reverse direction
2003                     const char *c_charset = charsets[i];
2004                     odr_set_charset(odr_encode(), c_charset, t_charset);
2005                     m_charset_converter->set_client_query_charset(c_charset);
2006                     m_charset_converter->set_client_charset_selected(selected);
2007                 }
2008                 nmem_destroy(nmem);
2009                 ODR_MASK_CLEAR(m_initRequest_options,
2010                                Z_Options_negotiationModel);
2011                 yaz_del_charneg_record(&initRequest->otherInfo);
2012             }
2013             else
2014             {
2015                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
2016                         m_session_str);
2017             }
2018         }
2019         else if (m_charset_converter->get_target_query_charset() &&
2020             m_proxy_negotiation_default_charset)
2021         {
2022             m_charset_converter->
2023                 set_client_query_charset(m_proxy_negotiation_default_charset);
2024         }
2025     }
2026     else if (apdu->which == Z_APDU_initResponse)
2027     {
2028         Z_InitResponse *initResponse = apdu->u.initResponse;
2029         Z_OtherInformation **otherInfo;
2030         get_otherInfoAPDU(apdu, &otherInfo);
2031
2032         Z_CharSetandLanguageNegotiation *charneg = 0;
2033
2034         if (otherInfo && *otherInfo &&
2035             ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel)
2036             && (charneg = yaz_get_charneg_record(*otherInfo)))
2037         {
2038             char *charset = 0;
2039             char *lang = 0;
2040             int selected = 0;
2041
2042             yaz_get_response_charneg(m_referenceId_mem, charneg,
2043                 &charset, &lang, &selected);
2044
2045             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
2046                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
2047
2048             if (m_initRequest_options &&
2049                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
2050             {
2051                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
2052                     m_session_str);
2053             }
2054             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2055             {
2056                 // negotiation-charset, negotiation-lang
2057                 // elements of config file in use.
2058
2059                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2060                     m_session_str);
2061
2062                 // clear negotiation option.
2063                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2064
2065                 // Delete negotiation (charneg-3) entry.
2066                 yaz_del_charneg_record(otherInfo);
2067             }
2068         }
2069         else
2070         {
2071             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2072             {
2073                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2074                         m_session_str);
2075             }
2076             else if (m_charset_converter->get_client_query_charset())
2077             {
2078                 Z_OtherInformation **otherInfo;
2079                 Z_OtherInformationUnit *oi;
2080                 get_otherInfoAPDU(apdu, &otherInfo);
2081                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2082                 if (oi)
2083                 {
2084                     ODR_MASK_SET(initResponse->options,
2085                                  Z_Options_negotiationModel);
2086                     if (m_initRequest_options)
2087                         ODR_MASK_SET(m_initRequest_options,
2088                                      Z_Options_negotiationModel);
2089
2090                     oi->which = Z_OtherInfo_externallyDefinedInfo;
2091                     oi->information.externallyDefinedInfo =
2092                         yaz_set_response_charneg(
2093                             odr_encode(),
2094                             m_charset_converter->get_client_query_charset(),
2095                             0 /* no lang */,
2096                             m_charset_converter->get_client_charset_selected());
2097                 }
2098             }
2099         }
2100     }
2101 }
2102
2103 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2104                                                      int error,
2105                                                      const char *addinfo)
2106 {
2107     Z_Records *rec = (Z_Records *)
2108         odr_malloc (odr, sizeof(*rec));
2109     int *err = (int *)
2110         odr_malloc (odr, sizeof(*err));
2111     Z_DiagRec *drec = (Z_DiagRec *)
2112         odr_malloc (odr, sizeof(*drec));
2113     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2114         odr_malloc (odr, sizeof(*dr));
2115     *err = error;
2116     rec->which = Z_Records_NSD;
2117     rec->u.nonSurrogateDiagnostic = dr;
2118     dr->diagnosticSetId =
2119         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
2120     dr->condition = err;
2121     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2122     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2123     return rec;
2124 }
2125
2126 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2127 {
2128     if (apdu->which == Z_APDU_searchRequest &&
2129         apdu->u.searchRequest->query &&
2130         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2131         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2132     {
2133         Z_RPNQuery *rpnquery = 0;
2134         Z_SearchRequest *sr = apdu->u.searchRequest;
2135         char *addinfo = 0;
2136
2137         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2138                 sr->query->u.type_104->u.cql);
2139
2140         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2141                                           &rpnquery, odr_encode(),
2142                                           &addinfo);
2143         if (r == -3)
2144             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2145         else if (r)
2146         {
2147             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2148             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2149
2150             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2151             new_apdu->u.searchResponse->records =
2152                 create_nonSurrogateDiagnostics(odr_encode(),
2153                                                yaz_diag_srw_to_bib1(r),
2154                                                addinfo);
2155             *new_apdu->u.searchResponse->searchStatus = 0;
2156
2157             send_to_client(new_apdu);
2158
2159             return 0;
2160         }
2161         else
2162         {
2163             sr->query->which = Z_Query_type_1;
2164             sr->query->u.type_1 = rpnquery;
2165         }
2166         return apdu;
2167     }
2168     return apdu;
2169 }
2170
2171 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2172 {
2173     if (apdu->which == Z_APDU_searchRequest &&
2174         apdu->u.searchRequest->query)
2175     {
2176         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2177             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2178         {
2179             if (m_http_version)
2180                 m_charset_converter->set_client_query_charset("UTF-8");
2181             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2182             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2183         }
2184     }
2185     return apdu;
2186 }
2187
2188
2189 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2190 {
2191     if (apdu->which == Z_APDU_searchRequest)
2192     {
2193         Z_SearchRequest *sr = apdu->u.searchRequest;
2194         int err = 0;
2195         char *addinfo = 0;
2196
2197         Yaz_ProxyConfig *cfg = check_reconfigure();
2198         if (cfg)
2199             err = cfg->check_query(odr_encode(), m_default_target,
2200                                    sr->query, &addinfo);
2201         if (err)
2202         {
2203             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2204
2205             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2206             new_apdu->u.searchResponse->records =
2207                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2208             *new_apdu->u.searchResponse->searchStatus = 0;
2209
2210             send_to_client(new_apdu);
2211
2212             return 0;
2213         }
2214     }
2215     return apdu;
2216 }
2217
2218 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2219 {
2220     if (apdu->which != Z_APDU_initRequest)
2221         return 1;  // pass if no init request
2222     Z_InitRequest *req = apdu->u.initRequest;
2223
2224     Yaz_ProxyConfig *cfg = check_reconfigure();
2225     if (!cfg)
2226         return 1;  // pass if no config
2227
2228     int ret;
2229     if (req->idAuthentication == 0)
2230     {
2231         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2232                                          m_peername);
2233     }
2234     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2235     {
2236         ret = cfg->client_authentication(
2237             m_default_target,
2238             req->idAuthentication->u.idPass->userId,
2239             req->idAuthentication->u.idPass->groupId,
2240             req->idAuthentication->u.idPass->password,
2241             m_peername);
2242     }
2243     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2244     {
2245         char user[64], pass[64];
2246         *user = '\0';
2247         *pass = '\0';
2248         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2249         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2250                                          m_peername);
2251     }
2252     else
2253         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2254                                          m_peername);
2255     return ret;
2256 }
2257
2258 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2259 {
2260     m_marcxml_mode = none;
2261     if (apdu->which == Z_APDU_searchRequest)
2262     {
2263         Z_SearchRequest *sr = apdu->u.searchRequest;
2264         int err = 0;
2265         char *addinfo = 0;
2266         Yaz_ProxyConfig *cfg = check_reconfigure();
2267
2268         Z_RecordComposition rc_temp, *rc = 0;
2269         if (sr->smallSetElementSetNames)
2270         {
2271             rc_temp.which = Z_RecordComp_simple;
2272             rc_temp.u.simple = sr->smallSetElementSetNames;
2273             rc = &rc_temp;
2274         }
2275
2276         if (sr->preferredRecordSyntax)
2277         {
2278             struct oident *ent;
2279             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2280             m_frontend_type = ent->value;
2281         }
2282         else
2283             m_frontend_type = VAL_NONE;
2284
2285         char *stylesheet_name = 0;
2286         if (cfg)
2287             err = cfg->check_syntax(odr_encode(),
2288                                     m_default_target,
2289                                     sr->preferredRecordSyntax, rc,
2290                                     &addinfo, &stylesheet_name, &m_schema,
2291                                     &m_backend_type, &m_backend_charset,
2292                                     &m_usemarcon_ini_stage1,
2293                                     &m_usemarcon_ini_stage2);
2294         if (stylesheet_name)
2295         {
2296             m_parent->low_socket_close();
2297
2298 #if HAVE_XSLT
2299             if (m_stylesheet_xsp)
2300                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2301             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2302                                                        stylesheet_name);
2303 #endif
2304             m_stylesheet_offset = 0;
2305             xfree(stylesheet_name);
2306
2307             m_parent->low_socket_open();
2308         }
2309         if (err == -1)
2310         {
2311             sr->smallSetElementSetNames = 0;
2312             sr->mediumSetElementSetNames = 0;
2313             m_marcxml_mode = marcxml;
2314             if (m_backend_type)
2315             {
2316
2317                 sr->preferredRecordSyntax =
2318                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2319                                         m_backend_type);
2320             }
2321             else
2322                 sr->preferredRecordSyntax =
2323                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2324                                            VAL_USMARC);
2325         }
2326         else if (err)
2327         {
2328             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2329
2330             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2331             new_apdu->u.searchResponse->records =
2332                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2333             *new_apdu->u.searchResponse->searchStatus = 0;
2334
2335             send_to_client(new_apdu);
2336
2337             return 0;
2338         }
2339         else if (m_backend_type)
2340         {
2341             sr->preferredRecordSyntax =
2342                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2343         }
2344     }
2345     else if (apdu->which == Z_APDU_presentRequest)
2346     {
2347         Z_PresentRequest *pr = apdu->u.presentRequest;
2348         int err = 0;
2349         char *addinfo = 0;
2350         Yaz_ProxyConfig *cfg = check_reconfigure();
2351
2352         if (pr->preferredRecordSyntax)
2353         {
2354             struct oident *ent;
2355             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2356             m_frontend_type = ent->value;
2357         }
2358         else
2359             m_frontend_type = VAL_NONE;
2360
2361         char *stylesheet_name = 0;
2362         if (cfg)
2363             err = cfg->check_syntax(odr_encode(), m_default_target,
2364                                     pr->preferredRecordSyntax,
2365                                     pr->recordComposition,
2366                                     &addinfo, &stylesheet_name, &m_schema,
2367                                     &m_backend_type, &m_backend_charset,
2368                                     &m_usemarcon_ini_stage1,
2369                                     &m_usemarcon_ini_stage2
2370                                     );
2371         if (stylesheet_name)
2372         {
2373             m_parent->low_socket_close();
2374
2375 #if HAVE_XSLT
2376             if (m_stylesheet_xsp)
2377                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2378             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2379                                                        stylesheet_name);
2380 #endif
2381             m_stylesheet_offset = 0;
2382             xfree(stylesheet_name);
2383
2384             m_parent->low_socket_open();
2385         }
2386         if (err == -1)
2387         {
2388             pr->recordComposition = 0;
2389             m_marcxml_mode = marcxml;
2390             if (m_backend_type)
2391             {
2392
2393                 pr->preferredRecordSyntax =
2394                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2395                                         m_backend_type);
2396             }
2397             else
2398                 pr->preferredRecordSyntax =
2399                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2400                                            VAL_USMARC);
2401         }
2402         else if (err)
2403         {
2404             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2405
2406             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2407             new_apdu->u.presentResponse->records =
2408                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2409             *new_apdu->u.presentResponse->presentStatus =
2410                 Z_PresentStatus_failure;
2411
2412             send_to_client(new_apdu);
2413
2414             return 0;
2415         }
2416         else if (m_backend_type)
2417         {
2418             pr->preferredRecordSyntax =
2419                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2420         }
2421     }
2422     return apdu;
2423 }
2424
2425 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2426 {
2427     if (!schema)
2428         return 0;
2429     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2430         odr_malloc(o, sizeof(Z_ElementSetNames));
2431     esn->which = Z_ElementSetNames_generic;
2432     esn->u.generic = odr_strdup(o, schema);
2433     return esn;
2434 }
2435
2436 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2437 {
2438     const char *t = 0;
2439     Yaz_ProxyConfig *cfg = check_reconfigure();
2440     if (cfg)
2441         t = cfg->get_explain_name(db, backend_db);
2442
2443     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2444     {
2445         releaseClient();
2446     }
2447
2448     if (t)
2449     {
2450         xfree(m_default_target);
2451         m_default_target = xstrdup(t);
2452     }
2453 }
2454
2455 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2456 {
2457     struct stat sbuf;
2458     if (strcmp(hreq->method, "GET"))
2459         return 0;
2460     if (hreq->path[0] != '/')
2461     {
2462         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2463         return 0;
2464     }
2465     const char *cp = hreq->path;
2466     while (*cp)
2467     {
2468         if (*cp == '/' && strchr("/.", cp[1]))
2469         {
2470             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2471             return 0;
2472         }
2473         cp++;
2474     }
2475     const char *fname = hreq->path+1;
2476     if (stat(fname, &sbuf))
2477     {
2478         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2479         return 0;
2480     }
2481     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2482     {
2483         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2484         return 0;
2485     }
2486     if (sbuf.st_size > (off_t) 1000000)
2487     {
2488         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2489         return 0;
2490     }
2491
2492     ODR o = odr_encode();
2493     Yaz_ProxyConfig *cfg = check_reconfigure();
2494     const char *ctype = cfg->check_mime_type(fname);
2495     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2496     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2497     if (m_http_version)
2498         hres->version = odr_strdup(o, m_http_version);
2499     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2500     if (m_http_keepalive)
2501         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2502     else
2503         timeout(0);
2504
2505     hres->content_len = sbuf.st_size;
2506     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2507     FILE *f = fopen(fname, "rb");
2508     if (f)
2509     {
2510         fread(hres->content_buf, 1, hres->content_len, f);
2511         fclose(f);
2512     }
2513     else
2514     {
2515         return 0;
2516     }
2517     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2518     {
2519         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2520                  fname);
2521     }
2522     int len;
2523     send_GDU(gdu, &len);
2524     recv_GDU_more(true);
2525     return 1;
2526 }
2527
2528 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2529 {
2530     if (m_s2z_odr_init)
2531     {
2532         odr_destroy(m_s2z_odr_init);
2533         m_s2z_odr_init = 0;
2534     }
2535     if (m_s2z_odr_search)
2536     {
2537         odr_destroy(m_s2z_odr_search);
2538         m_s2z_odr_search = 0;
2539     }
2540
2541     m_http_keepalive = 0;
2542     m_http_version = 0;
2543     if (!strcmp(hreq->version, "1.0"))
2544     {
2545         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2546         if (v && !strcmp(v, "Keep-Alive"))
2547             m_http_keepalive = 1;
2548         else
2549             m_http_keepalive = 0;
2550         m_http_version = "1.0";
2551     }
2552     else
2553     {
2554         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2555         if (v && !strcmp(v, "close"))
2556             m_http_keepalive = 0;
2557         else
2558             m_http_keepalive = 1;
2559         m_http_version = "1.1";
2560     }
2561
2562     const char *a = z_HTTP_header_lookup(hreq->headers, "Authorization");
2563     char authorization_str[255];
2564     *authorization_str = '\0';
2565     if (a && strncasecmp(a, "Basic ", 6) == 0)
2566         base64_decode(a + 6, authorization_str, 254);
2567
2568     Z_SRW_PDU *srw_pdu = 0;
2569     Z_SOAP *soap_package = 0;
2570     char *charset = 0;
2571     Z_SRW_diagnostic *diagnostic = 0;
2572     int num_diagnostic = 0;
2573
2574     if (file_access(hreq))
2575     {
2576         return;
2577     }
2578     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2579                             &charset) == 0
2580              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2581                                &charset, &diagnostic, &num_diagnostic) == 0)
2582     {
2583         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2584         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2585         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2586         m_s2z_init_apdu = 0;
2587         m_s2z_search_apdu = 0;
2588         m_s2z_present_apdu = 0;
2589
2590         m_s2z_stylesheet = 0;
2591
2592         Z_IdAuthentication *auth = NULL;
2593         if (*authorization_str)
2594         {
2595             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
2596             auth->which = Z_IdAuthentication_idPass;
2597             auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
2598             auth->u.idPass->groupId = NULL;
2599             char *p = strchr(authorization_str, ':');
2600             if (p)
2601             {
2602                 *p = '\0';
2603                 p++;
2604                 auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
2605             }
2606             auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
2607         }
2608
2609         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2610         {
2611             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2612
2613             const char *backend_db = srw_req->database;
2614             srw_get_client(srw_req->database, &backend_db);
2615
2616             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2617             // recordXPath unsupported.
2618             if (srw_req->recordXPath)
2619             {
2620                 yaz_add_srw_diagnostic(odr_decode(),
2621                                        &diagnostic, &num_diagnostic,
2622                                        72, 0);
2623             }
2624             // sort unsupported
2625             if (srw_req->sort_type != Z_SRW_sort_type_none)
2626             {
2627                 yaz_add_srw_diagnostic(odr_decode(),
2628                                        &diagnostic, &num_diagnostic,
2629                                        80, 0);
2630             }
2631             // save stylesheet
2632             if (srw_req->stylesheet)
2633                 m_s2z_stylesheet =
2634                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2635
2636             // set packing for response records ..
2637             if (srw_req->recordPacking &&
2638                 !strcmp(srw_req->recordPacking, "xml"))
2639                 m_s2z_packing = Z_SRW_recordPacking_XML;
2640             else
2641                 m_s2z_packing = Z_SRW_recordPacking_string;
2642
2643             if (num_diagnostic)
2644             {
2645                 Z_SRW_PDU *srw_pdu =
2646                     yaz_srw_get(odr_encode(),
2647                                 Z_SRW_searchRetrieve_response);
2648                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2649
2650                 srw_res->diagnostics = diagnostic;
2651                 srw_res->num_diagnostics = num_diagnostic;
2652                 send_srw_response(srw_pdu);
2653                 return;
2654             }
2655
2656             // prepare search PDU
2657             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2658                                           Z_APDU_searchRequest);
2659             Z_SearchRequest *z_searchRequest =
2660                 m_s2z_search_apdu->u.searchRequest;
2661
2662             z_searchRequest->num_databaseNames = 1;
2663             z_searchRequest->databaseNames = (char**)
2664                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2665             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2666                                                            backend_db);
2667
2668             // query transformation
2669             Z_Query *query = (Z_Query *)
2670                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2671             z_searchRequest->query = query;
2672
2673             if (srw_req->query_type == Z_SRW_query_type_cql)
2674             {
2675                 Z_External *ext = (Z_External *)
2676                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2677                 ext->direct_reference =
2678                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2679                 ext->indirect_reference = 0;
2680                 ext->descriptor = 0;
2681                 ext->which = Z_External_CQL;
2682                 ext->u.cql = srw_req->query.cql;
2683
2684                 query->which = Z_Query_type_104;
2685                 query->u.type_104 =  ext;
2686             }
2687             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2688             {
2689                 Z_RPNQuery *RPNquery;
2690                 YAZ_PQF_Parser pqf_parser;
2691
2692                 pqf_parser = yaz_pqf_create ();
2693
2694                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2695                                           srw_req->query.pqf);
2696                 if (!RPNquery)
2697                 {
2698                     const char *pqf_msg;
2699                     size_t off;
2700                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2701                     int ioff = off;
2702                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
2703                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2704
2705                     send_to_srw_client_error(10, 0);
2706                     return;
2707                 }
2708                 query->which = Z_Query_type_1;
2709                 query->u.type_1 =  RPNquery;
2710
2711                 yaz_pqf_destroy (pqf_parser);
2712             }
2713             else
2714             {
2715                 send_to_srw_client_error(7, "query");
2716                 return;
2717             }
2718
2719             // present
2720             m_s2z_present_apdu = 0;
2721             int max = 0;
2722             if (srw_req->maximumRecords)
2723                 max = *srw_req->maximumRecords;
2724             int start = 1;
2725             if (srw_req->startRecord)
2726                 start = *srw_req->startRecord;
2727             if (max > 0)
2728             {
2729                 // Some backend, such as Voyager doesn't honor piggyback
2730                 // So we use present always (0 &&).
2731                 if (0 && start <= 1)  // Z39.50 piggyback
2732                 {
2733                     *z_searchRequest->smallSetUpperBound = max;
2734                     *z_searchRequest->mediumSetPresentNumber = max;
2735                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2736
2737                     z_searchRequest->preferredRecordSyntax =
2738                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2739                                                VAL_TEXT_XML);
2740                     if (srw_req->recordSchema)
2741                     {
2742                         z_searchRequest->smallSetElementSetNames =
2743                             z_searchRequest->mediumSetElementSetNames =
2744                             mk_esn_from_schema(m_s2z_odr_search,
2745                                                srw_req->recordSchema);
2746                     }
2747                 }
2748                 else   // Z39.50 present
2749                 {
2750                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2751                                                    Z_APDU_presentRequest);
2752                     Z_PresentRequest *z_presentRequest =
2753                         m_s2z_present_apdu->u.presentRequest;
2754                     *z_presentRequest->resultSetStartPoint = start;
2755                     *z_presentRequest->numberOfRecordsRequested = max;
2756                     z_presentRequest->preferredRecordSyntax =
2757                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2758                                                VAL_TEXT_XML);
2759                     if (srw_req->recordSchema)
2760                     {
2761                         z_presentRequest->recordComposition =
2762                             (Z_RecordComposition *)
2763                             odr_malloc(m_s2z_odr_search,
2764                                        sizeof(Z_RecordComposition));
2765                         z_presentRequest->recordComposition->which =
2766                             Z_RecordComp_simple;
2767                         z_presentRequest->recordComposition->u.simple =
2768                             mk_esn_from_schema(m_s2z_odr_search,
2769                                                srw_req->recordSchema);
2770                     }
2771                 }
2772             }
2773             if (!m_client)
2774             {
2775                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2776                                             Z_APDU_initRequest);
2777
2778                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2779
2780                 // prevent m_initRequest_apdu memory from being grabbed
2781                 // in Yaz_Proxy::handle_incoming_Z_PDU
2782                 m_initRequest_apdu = m_s2z_init_apdu;
2783                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2784                 return;
2785             }
2786             else
2787             {
2788                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2789                 return;
2790             }
2791         }
2792         else if (srw_pdu->which == Z_SRW_explain_request)
2793         {
2794             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2795
2796             const char *backend_db = srw_req->database;
2797             srw_get_client(srw_req->database, &backend_db);
2798
2799             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2800
2801             // save stylesheet
2802             if (srw_req->stylesheet)
2803                 m_s2z_stylesheet =
2804                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2805
2806             if (srw_req->recordPacking &&
2807                 !strcmp(srw_req->recordPacking, "xml"))
2808                 m_s2z_packing = Z_SRW_recordPacking_XML;
2809             else
2810                 m_s2z_packing = Z_SRW_recordPacking_string;
2811
2812             if (num_diagnostic)
2813             {
2814                 send_srw_explain_response(diagnostic, num_diagnostic);
2815                 return;
2816             }
2817
2818             if (!m_client)
2819             {
2820                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2821                                             Z_APDU_initRequest);
2822
2823                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2824
2825                 // prevent m_initRequest_apdu memory from being grabbed
2826                 // in Yaz_Proxy::handle_incoming_Z_PDU
2827                 m_initRequest_apdu = m_s2z_init_apdu;
2828                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2829             }
2830             else
2831                 send_srw_explain_response(0, 0);
2832             return;
2833         }
2834         else if (srw_pdu->which == Z_SRW_scan_request)
2835         {
2836             m_s2z_database = odr_strdup(m_s2z_odr_init,
2837                                         srw_pdu->u.scan_request->database);
2838
2839             yaz_add_srw_diagnostic(odr_decode(),
2840                                    &diagnostic, &num_diagnostic,
2841                                    4, "scan");
2842             Z_SRW_PDU *srw_pdu =
2843                 yaz_srw_get(odr_encode(),
2844                             Z_SRW_scan_response);
2845             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2846
2847             srw_res->diagnostics = diagnostic;
2848             srw_res->num_diagnostics = num_diagnostic;
2849             send_srw_response(srw_pdu);
2850             return;
2851         }
2852         else
2853         {
2854             m_s2z_database = 0;
2855
2856             send_to_srw_client_error(4, 0);
2857         }
2858     }
2859     send_http_response(400);
2860 }
2861
2862 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2863 {
2864
2865     Z_OtherInformation **oi;
2866     get_otherInfoAPDU(apdu, &oi);
2867
2868     if (apdu->u.initRequest->implementationId)
2869         yaz_log(YLOG_LOG, "%simplementationId: %s",
2870                 m_session_str, apdu->u.initRequest->implementationId);
2871     if (apdu->u.initRequest->implementationName)
2872         yaz_log(YLOG_LOG, "%simplementationName: %s",
2873                 m_session_str, apdu->u.initRequest->implementationName);
2874     if (apdu->u.initRequest->implementationVersion)
2875         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2876                 m_session_str, apdu->u.initRequest->implementationVersion);
2877     if (m_initRequest_apdu == 0)
2878     {
2879         if (m_initRequest_mem)
2880             nmem_destroy(m_initRequest_mem);
2881
2882         m_initRequest_apdu = apdu;
2883         m_initRequest_mem = odr_extract_mem(odr_decode());
2884
2885         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2886             preferredMessageSize;
2887         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2888         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2889             maximumRecordSize;
2890         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2891
2892         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2893             yaz_get_charneg_record(*oi);
2894
2895         // Save proposal charsets and langs.
2896         if (ODR_MASK_GET(apdu->u.initRequest->options,
2897                          Z_Options_negotiationModel)
2898             && charSetandLangRecord)
2899         {
2900
2901             yaz_get_proposal_charneg(m_referenceId_mem,
2902                                      charSetandLangRecord,
2903                                      &m_initRequest_oi_negotiation_charsets,
2904                                      &m_initRequest_oi_negotiation_num_charsets,
2905                                      &m_initRequest_oi_negotiation_langs,
2906                                      &m_initRequest_oi_negotiation_num_langs,
2907                                      &m_initRequest_oi_negotiation_selected);
2908
2909             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2910             {
2911                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2912                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2913                         m_initRequest_oi_negotiation_charsets[i]:"none");
2914             }
2915             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2916             {
2917                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2918                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2919                         m_initRequest_oi_negotiation_langs[i]:"none");
2920             }
2921             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2922                     m_session_str, m_initRequest_oi_negotiation_selected);
2923         }
2924         // save init options for the response..
2925         m_initRequest_options = apdu->u.initRequest->options;
2926
2927         apdu->u.initRequest->options =
2928             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2929                                        sizeof(Odr_bitmask));
2930         ODR_MASK_ZERO(apdu->u.initRequest->options);
2931         int i;
2932         for (i = 0; i<= 24; i++)
2933             ODR_MASK_SET(apdu->u.initRequest->options, i);
2934         // check negotiation option
2935         if (!ODR_MASK_GET(m_initRequest_options,
2936                           Z_Options_negotiationModel))
2937         {
2938             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2939                            Z_Options_negotiationModel);
2940         }
2941         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2942                        Z_Options_concurrentOperations);
2943         // make new version
2944         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2945         apdu->u.initRequest->protocolVersion =
2946             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2947                                        sizeof(Odr_bitmask));
2948         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2949
2950         for (i = 0; i<= 8; i++)
2951             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2952     }
2953     handle_charset_lang_negotiation(apdu);
2954     if (m_client->m_init_flag)
2955     {
2956         if (handle_init_response_for_invalid_session(apdu))
2957             return;
2958         if (m_client->m_initResponse)
2959         {
2960             Z_APDU *apdu2 = m_client->m_initResponse;
2961             apdu2->u.initResponse->otherInfo = 0;
2962             if (m_client->m_cookie && *m_client->m_cookie)
2963                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2964                                            m_client->m_cookie);
2965             apdu2->u.initResponse->referenceId =
2966                 apdu->u.initRequest->referenceId;
2967             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2968             apdu2->u.initResponse->protocolVersion =
2969                 m_client->m_initResponse_version;
2970
2971             handle_charset_lang_negotiation(apdu2);
2972
2973             send_to_client(apdu2);
2974             m_timeout_mode = timeout_normal;
2975             return;
2976         }
2977     }
2978     m_client->m_init_flag = 1;
2979
2980 #if USE_AUTH_MSG
2981     yaz_log(YLOG_LOG, "%suse_auth_msg", m_session_str);
2982     Auth_Msg *m = new Auth_Msg;
2983     m->m_proxy = this;
2984     z_APDU(odr_encode(), &apdu, 0, "encode");
2985     char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
2986     m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
2987     memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
2988     odr_reset(odr_encode());
2989
2990     inc_ref();
2991     m_my_thread->put(m);
2992 #else
2993     int ret = handle_authentication(apdu);
2994     result_authentication(apdu, ret);
2995 #endif
2996 }
2997
2998 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2999 {
3000     Z_ReferenceId **refid = get_referenceIdP(apdu);
3001     nmem_reset(m_referenceId_mem);
3002     if (refid && *refid)
3003     {
3004         m_referenceId = (Z_ReferenceId *)
3005             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
3006         m_referenceId->len = m_referenceId->size = (*refid)->len;
3007         m_referenceId->buf = (unsigned char *)
3008             nmem_malloc(m_referenceId_mem, (*refid)->len);
3009         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
3010     }
3011     else
3012         m_referenceId = 0;
3013
3014     if (!m_client && m_flag_invalid_session)
3015     {
3016         // Got request for a session that is invalid..
3017         m_apdu_invalid_session = apdu; // save package
3018         m_mem_invalid_session = odr_extract_mem(odr_decode());
3019         apdu = m_initRequest_apdu;     // but throw an init to the target
3020     }
3021
3022     // Determine our client.
3023     Z_OtherInformation **oi;
3024     get_otherInfoAPDU(apdu, &oi);
3025     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
3026     if (!m_client)
3027     {
3028         if (m_http_version)
3029         {   // HTTP. Send not found
3030             send_http_response(404);
3031             return;
3032         }
3033         else
3034         {
3035             // Z39.50 just shutdown
3036             delete this;
3037             return;
3038         }
3039     }
3040
3041     m_client->m_server = this;
3042
3043     if (apdu->which == Z_APDU_initRequest)
3044         handle_init(apdu);
3045     else
3046         handle_incoming_Z_PDU_2(apdu);
3047 }
3048
3049 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
3050 {
3051     handle_max_record_retrieve(apdu);
3052
3053     if (apdu)
3054         apdu = handle_syntax_validation(apdu);
3055
3056     if (apdu)
3057         apdu = handle_query_transformation(apdu);
3058
3059     if (apdu)
3060         apdu = handle_target_charset_conversion(apdu);
3061
3062     if (apdu)
3063         apdu = handle_query_validation(apdu);
3064
3065     if (apdu)
3066         apdu = result_set_optimize(apdu);
3067
3068     if (!apdu)
3069     {
3070         m_client->timeout(m_target_idletime);  // mark it active even
3071         recv_GDU_more(true);
3072         // though we didn't use it
3073         return;
3074     }
3075
3076     // delete other info construct completely if 0 elements
3077     Z_OtherInformation **oi;
3078     get_otherInfoAPDU(apdu, &oi);
3079     if (oi && *oi && (*oi)->num_elements == 0)
3080         *oi = 0;
3081
3082     if (apdu->which == Z_APDU_presentRequest &&
3083         m_client->m_resultSetStartPoint == 0)
3084     {
3085         Z_PresentRequest *pr = apdu->u.presentRequest;
3086         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3087         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3088     } else {
3089         m_client->m_resultSetStartPoint = 0;
3090     }
3091     if (m_client->send_to_target(apdu) < 0)
3092     {
3093         m_client->shutdown();
3094     }
3095     else
3096         m_client->m_waiting = 1;
3097 }
3098
3099 void Yaz_Proxy::connectNotify()
3100 {
3101 }
3102
3103 void Yaz_Proxy::releaseClient()
3104 {
3105     xfree(m_proxyTarget);
3106     m_proxyTarget = 0;
3107     m_flag_invalid_session = 0;
3108     // only keep if keep_alive flag is set...
3109     if (m_client &&
3110         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3111         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3112         m_client->m_waiting == 0)
3113     {
3114         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3115                  m_session_str,
3116                  m_client->get_hostname());
3117         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3118                 m_session_str, m_client->m_pdu_recv,
3119                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3120                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3121         assert (m_client->m_waiting != 2);
3122         // Tell client (if any) that no server connection is there..
3123         m_client->m_server = 0;
3124         m_client = 0;
3125     }
3126     else if (m_client)
3127     {
3128         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
3129                  m_session_str,
3130                  m_client->get_hostname());
3131         assert (m_client->m_waiting != 2);
3132         delete m_client;
3133         m_client = 0;
3134     }
3135     else if (!m_parent)
3136     {
3137         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
3138                  m_session_str);
3139         assert (m_parent);
3140     }
3141     else
3142     {
3143         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
3144                  m_session_str);
3145     }
3146     if (m_parent)
3147         m_parent->pre_init();
3148 }
3149
3150 bool Yaz_Proxy::dec_ref(bool main_ptr)
3151 {
3152     yaz_log(YLOG_LOG, "%sdec_ref count=%d", m_session_str, m_ref_count);
3153
3154     assert(m_ref_count > 0);
3155     if (main_ptr)
3156     {
3157         if (m_main_ptr_dec)
3158             return false;
3159         m_main_ptr_dec = true;
3160     }
3161
3162     m_http_keepalive = 0;
3163
3164     --m_ref_count;
3165     if (m_ref_count > 0)
3166         return false;
3167
3168     releaseClient();
3169
3170     delete this;
3171     return true;
3172 }
3173
3174 const char *Yaz_ProxyClient::get_session_str()
3175 {
3176     if (!m_server)
3177         return "0 ";
3178     return m_server->get_session_str();
3179 }
3180
3181 void Yaz_ProxyClient::shutdown()
3182 {
3183     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3184              get_hostname());
3185
3186     if (m_server)
3187     {
3188         m_waiting = 1;      // ensure it's released from Proxy in releaseClient
3189         m_server->dec_ref(true);
3190     }
3191     else
3192         delete this;
3193 }
3194
3195 void Yaz_Proxy::failNotify()
3196 {
3197     inc_request_no();
3198     yaz_log (YLOG_LOG, "%sConnection closed by client",
3199              get_session_str());
3200     dec_ref(true);
3201 }
3202
3203 void Yaz_Proxy::send_response_fail_client(const char *addr)
3204 {
3205     yaz_log(YLOG_LOG, "%ssend_close_response", get_session_str());
3206     if (m_http_version)
3207     {
3208         Z_SRW_diagnostic *diagnostic = 0;
3209         int num_diagnostic = 0;
3210
3211         yaz_add_srw_diagnostic(odr_encode(),
3212                                &diagnostic, &num_diagnostic,
3213                                YAZ_SRW_SYSTEM_TEMPORARILY_UNAVAILABLE, addr);
3214         if (m_s2z_search_apdu)
3215             send_srw_search_response(diagnostic, num_diagnostic);
3216         else
3217             send_srw_explain_response(diagnostic, num_diagnostic);
3218     }
3219 }
3220 void Yaz_ProxyClient::failNotify()
3221 {
3222     if (m_server)
3223         m_server->inc_request_no();
3224     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
3225              get_session_str(), get_hostname());
3226
3227     if (m_server)
3228         m_server->send_response_fail_client(get_hostname());
3229     shutdown();
3230 }
3231
3232 void Yaz_ProxyClient::connectNotify()
3233 {
3234     const char *s = get_session_str();
3235     const char *h = get_hostname();
3236     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3237              m_target_idletime);
3238     timeout(m_target_idletime);
3239     if (!m_server)
3240         pre_init_client();
3241 }
3242
3243 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3244                                               *the_PDU_Observable, int fd)
3245 {
3246     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3247 }
3248
3249 Yaz_ProxyClient::~Yaz_ProxyClient()
3250 {
3251     if (m_prev)
3252         *m_prev = m_next;
3253     if (m_next)
3254         m_next->m_prev = m_prev;
3255     m_waiting = 2;     // for debugging purposes only.
3256     odr_destroy(m_init_odr);
3257     odr_destroy(m_idAuthentication_odr);
3258     delete m_last_query;
3259     xfree (m_last_resultSetId);
3260     xfree (m_cookie);
3261 }
3262
3263 void Yaz_ProxyClient::pre_init_client()
3264 {
3265     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3266     Z_InitRequest *req = apdu->u.initRequest;
3267
3268     int i;
3269     for (i = 0; i<= 24; i++)
3270         ODR_MASK_SET(req->options, i);
3271     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3272                    Z_Options_negotiationModel);
3273     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3274                    Z_Options_concurrentOperations);
3275     for (i = 0; i<= 10; i++)
3276         ODR_MASK_SET(req->protocolVersion, i);
3277
3278     if (send_to_target(apdu) < 0)
3279     {
3280         delete this;
3281     }
3282     else
3283     {
3284         m_waiting = 1;
3285         m_init_flag = 1;
3286     }
3287 }
3288
3289 void Yaz_Proxy::pre_init()
3290 {
3291     int i;
3292     const char *name = 0;
3293     const char *zurl_in_use[MAX_ZURL_PLEX];
3294     int limit_bw, limit_pdu, limit_req, limit_search, limit_connect;
3295     int target_idletime, client_idletime;
3296     int max_clients;
3297     int keepalive_limit_bw, keepalive_limit_pdu;
3298     int pre_init;
3299     const char *cql2rpn = 0;
3300     const char *authentication = 0;
3301     const char *negotiation_charset = 0;
3302     const char *negotiation_lang = 0;
3303
3304     Yaz_ProxyConfig *cfg = check_reconfigure();
3305
3306     zurl_in_use[0] = 0;
3307
3308     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3309         set_APDU_yazlog(1);
3310     else
3311         set_APDU_yazlog(0);
3312
3313     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3314                                           &limit_bw, &limit_pdu, &limit_req,
3315                                           &limit_search, &limit_connect,
3316                                           &target_idletime, &client_idletime,
3317                                           &max_clients,
3318                                           &keepalive_limit_bw,
3319                                           &keepalive_limit_pdu,
3320                                           &pre_init,
3321                                           &cql2rpn,
3322                                           &authentication,
3323                                           &negotiation_charset,
3324                                           &negotiation_lang,
3325                                           0,
3326                                           0) ; i++)
3327     {
3328         if (pre_init)
3329         {
3330             int j;
3331             for (j = 0; zurl_in_use[j]; j++)
3332             {
3333                 Yaz_ProxyClient *c;
3334                 int spare = 0;
3335                 int spare_waiting = 0;
3336                 int in_use = 0;
3337                 int other = 0;
3338                 for (c = m_clientPool; c; c = c->m_next)
3339                 {
3340                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3341                     {
3342                         if (c->m_cookie == 0)
3343                         {
3344                             if (c->m_server == 0)
3345                                 if (c->m_waiting)
3346                                     spare_waiting++;
3347                                 else
3348                                     spare++;
3349                             else
3350                                 in_use++;
3351                         }
3352                         else
3353                             other++;
3354                     }
3355                 }
3356                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3357                         "sparew=%d preinit=%d",m_session_str,
3358                         name, zurl_in_use[j], in_use, other,
3359                         spare, spare_waiting, pre_init);
3360                 if (spare + spare_waiting < pre_init)
3361                 {
3362                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3363                     c->m_next = m_clientPool;
3364                     if (c->m_next)
3365                         c->m_next->m_prev = &c->m_next;
3366                     m_clientPool = c;
3367                     c->m_prev = &m_clientPool;
3368
3369                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3370                         c->set_APDU_yazlog(1);
3371                     else
3372                         c->set_APDU_yazlog(0);
3373
3374                     if (c->client(zurl_in_use[j]))
3375                     {
3376                         timeout(60);
3377                         delete c;
3378                         return;
3379                     }
3380                     c->timeout(30);
3381                     c->m_waiting = 1;
3382                     c->m_target_idletime = target_idletime;
3383                     c->m_seqno = m_seqno++;
3384                 }
3385             }
3386         }
3387     }
3388 }
3389
3390 void Yaz_Proxy::timeoutNotify()
3391 {
3392     if (m_parent)
3393     {
3394         GDU *gdu;
3395         switch(m_timeout_mode)
3396         {
3397         case timeout_normal:
3398         case timeout_busy:
3399             inc_request_no();
3400             m_in_queue.clear();
3401             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3402             dec_ref(true);
3403             break;
3404         case timeout_reduce:
3405             timeout(m_client_idletime);
3406             m_timeout_mode = timeout_busy;
3407             gdu = m_timeout_gdu;
3408             m_timeout_gdu = 0;
3409             recv_GDU_normal(gdu);
3410             break;
3411         case timeout_xsl:
3412             assert(m_stylesheet_nprl);
3413             convert_xsl_delay();
3414             recv_GDU_more(true);
3415         }
3416     }
3417     else
3418     {
3419         timeout(600);
3420         pre_init();
3421     }
3422 }
3423
3424 void Yaz_Proxy::markInvalid()
3425 {
3426     m_client = 0;
3427     m_flag_invalid_session = 1;
3428 }
3429
3430 void Yaz_ProxyClient::timeoutNotify()
3431 {
3432     if (m_server)
3433         m_server->inc_request_no();
3434
3435     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3436              get_hostname());
3437
3438     if (m_server)
3439         m_server->send_response_fail_client(get_hostname());
3440     shutdown();
3441
3442     m_root->pre_init();
3443 }
3444
3445 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
3446                                  Yaz_Proxy *parent) :
3447     Z_Assoc (the_PDU_Observable)
3448 {
3449     m_cookie = 0;
3450     m_next = 0;
3451     m_prev = 0;
3452     m_init_flag = 0;
3453     m_last_query = 0;
3454     m_last_resultSetId = 0;
3455     m_last_resultCount = 0;
3456     m_last_ok = 0;
3457     m_sr_transform = 0;
3458     m_waiting = 0;
3459     m_init_odr = odr_createmem (ODR_DECODE);
3460     m_initResponse = 0;
3461     m_initResponse_options = 0;
3462     m_initResponse_version = 0;
3463     m_initResponse_preferredMessageSize = 0;
3464     m_initResponse_maximumRecordSize = 0;
3465     m_resultSetStartPoint = 0;
3466     m_bytes_sent = m_bytes_recv = 0;
3467     m_pdu_recv = 0;
3468     m_server = 0;
3469     m_seqno = 0;
3470     m_target_idletime = 600;
3471     m_root = parent;
3472     m_idAuthentication_odr = odr_createmem(ODR_ENCODE);
3473     m_idAuthentication_ber_buf = 0;
3474     m_idAuthentication_ber_size = 0;
3475 }
3476
3477 const char *Yaz_Proxy::option(const char *name, const char *value)
3478 {
3479     if (!strcmp (name, "optimize")) {
3480         if (value) {
3481             xfree (m_optimize);
3482             m_optimize = xstrdup (value);
3483         }
3484         return m_optimize;
3485     }
3486     return 0;
3487 }
3488
3489 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
3490 {
3491
3492 }
3493
3494 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
3495 {
3496     if (apdu->which == Z_GDU_Z3950)
3497         recv_Z_PDU(apdu->u.z3950, len);
3498     else if (apdu->which == Z_GDU_HTTP_Response)
3499         recv_HTTP_response(apdu->u.HTTP_Response, len);
3500     else
3501         shutdown();
3502 }
3503
3504 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
3505 {
3506     if (!m_flag_invalid_session)
3507         return 0;
3508     m_flag_invalid_session = 0;
3509     handle_incoming_Z_PDU(m_apdu_invalid_session);
3510     assert (m_mem_invalid_session);
3511     nmem_destroy(m_mem_invalid_session);
3512     m_mem_invalid_session = 0;
3513     return 1;
3514 }
3515
3516 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
3517 {
3518     m_bytes_recv += len;
3519
3520     m_pdu_recv++;
3521     m_waiting = 0;
3522     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
3523         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
3524                  apdu_name(apdu), get_hostname(), len);
3525     if (apdu->which == Z_APDU_initResponse)
3526     {
3527         if (!m_server)  // if this is a pre init session , check for more
3528             m_root->pre_init();
3529         NMEM nmem = odr_extract_mem (odr_decode());
3530         odr_reset (m_init_odr);
3531         nmem_transfer (m_init_odr->mem, nmem);
3532         m_initResponse = apdu;
3533         m_initResponse_options = apdu->u.initResponse->options;
3534         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3535         m_initResponse_preferredMessageSize =
3536             *apdu->u.initResponse->preferredMessageSize;
3537         m_initResponse_maximumRecordSize =
3538             *apdu->u.initResponse->maximumRecordSize;
3539
3540         Z_InitResponse *ir = apdu->u.initResponse;
3541         char *im0 = ir->implementationName;
3542
3543         char *im1 = (char*)
3544             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3545         *im1 = '\0';
3546         if (im0)
3547         {
3548             strcat(im1, im0);
3549             strcat(im1, " ");
3550         }
3551         strcat(im1, "(YAZ Proxy)");
3552         ir->implementationName = im1;
3553
3554         nmem_destroy (nmem);
3555
3556         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3557             return;
3558     }
3559     if (apdu->which == Z_APDU_searchResponse)
3560     {
3561         Z_SearchResponse *sr = apdu->u.searchResponse;
3562         m_last_resultCount = *sr->resultCount;
3563         int status = *sr->searchStatus;
3564         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3565         {
3566             m_last_ok = 1;
3567
3568             if (sr->records && sr->records->which == Z_Records_DBOSD)
3569             {
3570                 m_cache.add(odr_decode(),
3571                             sr->records->u.databaseOrSurDiagnostics, 1,
3572                             *sr->resultCount);
3573             }
3574         }
3575     }
3576     if (apdu->which == Z_APDU_presentResponse)
3577     {
3578         Z_PresentResponse *pr = apdu->u.presentResponse;
3579         if (m_sr_transform)
3580         {
3581             m_sr_transform = 0;
3582             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3583             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3584             sr->referenceId = pr->referenceId;
3585             *sr->resultCount = m_last_resultCount;
3586             sr->records = pr->records;
3587             sr->nextResultSetPosition = pr->nextResultSetPosition;
3588             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3589             apdu = new_apdu;
3590         }
3591         if (pr->records &&
3592             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3593         {
3594             m_cache.add(odr_decode(),
3595                         pr->records->u.databaseOrSurDiagnostics,
3596                         m_resultSetStartPoint, -1);
3597             m_resultSetStartPoint = 0;
3598         }
3599     }
3600     if (m_cookie)
3601         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3602
3603     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
3604
3605     if (server)
3606         server->send_to_client(apdu);
3607     if (apdu->which == Z_APDU_close)
3608         shutdown();
3609     else if (server)
3610         server->recv_GDU_more(true);
3611 }
3612
3613 void Yaz_Proxy::low_socket_close()
3614 {
3615 #if WIN32
3616 #else
3617     int i;
3618     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3619         if  (m_lo_fd[i] >= 0)
3620             ::close(m_lo_fd[i]);
3621 #endif
3622 }
3623
3624 void Yaz_Proxy::low_socket_open()
3625 {
3626 #if WIN32
3627 #else
3628     int i;
3629     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3630         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3631 #endif
3632 }
3633
3634 int Yaz_Proxy::server(const char *addr)
3635 {
3636     int r = Z_Assoc::server(addr);
3637     if (!r)
3638     {
3639         yaz_log(YLOG_LOG, "%sStarted proxy "
3640 #ifdef VERSION
3641             VERSION
3642 #endif
3643             " on %s", m_session_str, addr);
3644         timeout(1);
3645     }
3646     return r;
3647 }
3648
3649 void Yaz_Proxy::base64_decode(const char *base64, char *buf, int buf_len)
3650 {
3651     const char *base64_chars =
3652         "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
3653     int len = strlen(base64);
3654     int buf_pos = 0;
3655     int index = 1;
3656
3657     for (int pos = 0; pos <= len; pos++)
3658     {
3659         if (base64[pos] == '=' || buf_pos + 1 >= buf_len)
3660             break;
3661
3662         const char *ch_ptr = strchr(base64_chars, base64[pos]);
3663         if (!ch_ptr)
3664             break;
3665         char ch = (char) (ch_ptr - base64_chars);
3666         switch (index)
3667         {
3668             case 1:
3669                 buf[buf_pos] = ch << 2;
3670                 break;
3671             case 2:
3672                 buf[buf_pos++] += (ch & 0x30) >> 4;
3673                 buf[buf_pos] = (ch & 0x0f) << 4;
3674                 break;
3675             case 3:
3676                 buf[buf_pos++] += (ch & 0x3c) >> 2;
3677                 buf[buf_pos] = (ch & 0x03) << 6;
3678                 break;
3679             case 4:
3680                 buf[buf_pos++] += ch;
3681         }
3682         if (index < 4)
3683             index++;
3684         else
3685             index = 1;
3686     }
3687     buf[buf_pos] = '\0';
3688 }
3689
3690 /*
3691  * Local variables:
3692  * c-basic-offset: 4
3693  * indent-tabs-mode: nil
3694  * End:
3695  * vim: shiftwidth=4 tabstop=8 expandtab
3696  */
3697