src/yaz-proxy.cpp

   1 /* This file is part of YAZ proxy
   2    Copyright (C) 1998-2011 Index Data
   3
   4 YAZ proxy is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17 */
  18
  19 #ifdef WIN32
  20 #define HAVE_SYS_STAT_H 1
  21 #define HAVE_SYS_TYPES_H 1
  22 #endif
  23
  24 #if HAVE_UNISTD_H
  25 #include <unistd.h>
  26 #endif
  27 #if HAVE_SYS_TIME_H
  28 #include <sys/time.h>
  29 #endif
  30 #if HAVE_SYS_TYPES_H
  31 #include <sys/types.h>
  32 #endif
  33 #if HAVE_SYS_STAT_H
  34 #include <sys/stat.h>
  35 #endif
  36
  37 #include <assert.h>
  38 #include <stdlib.h>
  39 #include <time.h>
  40 #include <fcntl.h>
  41
  42 #include <yaz/srw.h>
  43 #include <yaz/marcdisp.h>
  44 #include <yaz/yaz-iconv.h>
  45 #include <yaz/log.h>
  46 #include <yaz/diagbib1.h>
  47 #include "proxyp.h"
  48 #include <yaz/pquery.h>
  49 #include <yaz/otherinfo.h>
  50 #include <yaz/charneg.h>
  51 #include <yaz/oid_db.h>
  52 #include "msg-thread.h"
  53
  54 #if YAZ_VERSIONL >= 0x03001D
  55 #define YAZ_HAS_MK_SURROGATE 1
  56 #else
  57 #define YAZ_HAS_MK_SURROGATE 0
  58 #endif
  59
  60 using namespace yazpp_1;
  61
  62 #ifdef WIN32
  63 #define strncasecmp _strnicmp
  64 #endif
  65
  66 class YAZ_EXPORT Auth_Msg : public IMsg_Thread {
  67 public:
  68     int m_ret;
  69     IMsg_Thread *handle();
  70     void result();
  71     Yaz_Proxy *m_proxy;
  72     NMEM m_nmem;
  73     char *m_apdu_buf;
  74     int m_apdu_len;
  75     Auth_Msg();
  76     virtual ~Auth_Msg();
  77 };
  78
  79 Auth_Msg::Auth_Msg()
  80 {
  81     m_nmem = nmem_create();
  82 }
  83
  84 Auth_Msg::~Auth_Msg()
  85 {
  86     nmem_destroy(m_nmem);
  87 }
  88
  89 IMsg_Thread *Auth_Msg::handle()
  90 {
  91     ODR decode = odr_createmem(ODR_DECODE);
  92     Z_APDU *apdu;
  93
  94     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
  95     int r = z_APDU(decode, &apdu, 0, 0);
  96     if (!r)
  97     {
  98         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
  99     }
 100     else
 101     {
 102         m_ret = m_proxy->handle_authentication(apdu);
 103     }
 104     odr_destroy(decode);
 105     return this;
 106 }
 107
 108 void Auth_Msg::result()
 109 {
 110     if (m_proxy->dec_ref())
 111     {
 112         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
 113     }
 114     else
 115     {
 116         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
 117         Z_APDU *apdu = 0;
 118         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
 119         if (!r)
 120             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
 121         m_proxy->result_authentication(apdu, m_ret);
 122     }
 123     delete this;
 124 }
 125
 126 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
 127 {
 128     if (apdu == 0 || ret == 0)
 129     {
 130         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
 131         *apdu_reject->u.initResponse->result = 0;
 132         send_to_client(apdu_reject);
 133         dec_ref();
 134     }
 135     else
 136     {
 137         if (apdu->which == Z_APDU_initRequest)
 138         {
 139             Yaz_ProxyConfig *cfg = check_reconfigure();
 140             if (cfg)
 141                 cfg->target_authentication(m_default_target, odr_encode(),
 142                                            apdu->u.initRequest);
 143         }
 144         handle_incoming_Z_PDU_2(apdu);
 145     }
 146 }
 147
 148 static const char *apdu_name(Z_APDU *apdu)
 149 {
 150     switch (apdu->which)
 151     {
 152     case Z_APDU_initRequest:
 153         return "initRequest";
 154     case Z_APDU_initResponse:
 155         return "initResponse";
 156     case Z_APDU_searchRequest:
 157         return "searchRequest";
 158     case Z_APDU_searchResponse:
 159         return "searchResponse";
 160     case Z_APDU_presentRequest:
 161         return "presentRequest";
 162     case Z_APDU_presentResponse:
 163         return "presentResponse";
 164     case Z_APDU_deleteResultSetRequest:
 165         return "deleteResultSetRequest";
 166     case Z_APDU_deleteResultSetResponse:
 167         return "deleteResultSetResponse";
 168     case Z_APDU_scanRequest:
 169         return "scanRequest";
 170     case Z_APDU_scanResponse:
 171         return "scanResponse";
 172     case Z_APDU_sortRequest:
 173         return "sortRequest";
 174     case Z_APDU_sortResponse:
 175         return "sortResponse";
 176     case Z_APDU_extendedServicesRequest:
 177         return "extendedServicesRequest";
 178     case Z_APDU_extendedServicesResponse:
 179         return "extendedServicesResponse";
 180     case Z_APDU_close:
 181         return "close";
 182     }
 183     return "other";
 184 }
 185
 186 static const char *gdu_name(Z_GDU *gdu)
 187 {
 188     switch(gdu->which)
 189     {
 190     case Z_GDU_Z3950:
 191         return apdu_name(gdu->u.z3950);
 192     case Z_GDU_HTTP_Request:
 193         return "HTTP Request";
 194     case Z_GDU_HTTP_Response:
 195         return "HTTP Response";
 196     }
 197     return "Unknown request/response";
 198 }
 199
 200 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
 201                      ISocketObservable *the_socket_observable,
 202                      Yaz_Proxy *parent)
 203     :
 204     Z_Assoc(the_PDU_Observable),
 205     m_bw_stat(60), m_pdu_stat(60), m_search_stat(60)
 206 {
 207     m_PDU_Observable = the_PDU_Observable;
 208     m_socket_observable = the_socket_observable;
 209     m_client = 0;
 210     m_parent = parent;
 211     m_clientPool = 0;
 212     m_seqno = 1;
 213     m_keepalive_limit_bw = 500000;
 214     m_keepalive_limit_pdu = 1000;
 215     m_proxyTarget = 0;
 216     m_default_target = 0;
 217     m_proxy_negotiation_charset = 0;
 218     m_proxy_negotiation_lang = 0;
 219     m_proxy_negotiation_default_charset = 0;
 220     m_charset_converter = new Yaz_CharsetConverter;
 221     m_max_clients = 150;
 222     m_log_mask = 0;
 223     m_seed = time(0);
 224     m_client_idletime = 600;
 225     m_target_idletime = 600;
 226     m_max_sockets = 1024;
 227     m_optimize = xstrdup("1");
 228     strcpy(m_session_str, "0 ");
 229     m_session_no = 0;
 230     m_bytes_sent = 0;
 231     m_bytes_recv = 0;
 232     m_bw_max = 0;
 233     m_pdu_max = 0;
 234     m_search_max = 0;
 235     m_max_connect = 0;
 236     m_max_connect_period = 0;
 237     m_limit_connect = 0;
 238     m_limit_connect_period = 0;
 239     m_timeout_mode = timeout_normal;
 240     m_timeout_gdu = 0;
 241     m_max_record_retrieve = 0;
 242     m_reconfig_flag = 0;
 243     m_config_fname = 0;
 244     m_request_no = 0;
 245     m_flag_invalid_session = 0;
 246     m_referenceId = 0;
 247     m_referenceId_mem = nmem_create();
 248     m_config = 0;
 249     m_marcxml_mode = none;
 250     m_stylesheet_xsp = 0;
 251     m_stylesheet_nprl = 0;
 252     m_stylesheet_apdu = 0;
 253     m_s2z_stylesheet = 0;
 254     m_s2z_database = 0;
 255     m_schema = 0;
 256     m_backend_type = 0;
 257     m_backend_charset = 0;
 258     m_frontend_type[0] = -1;
 259     m_initRequest_apdu = 0;
 260     m_initRequest_mem = 0;
 261     m_initRequest_preferredMessageSize = 0;
 262     m_initRequest_maximumRecordSize = 0;
 263     m_initRequest_options = 0;
 264     m_initRequest_version = 0;
 265     m_initRequest_oi_negotiation_charsets = 0;
 266     m_initRequest_oi_negotiation_num_charsets = 0;
 267     m_initRequest_oi_negotiation_langs = 0;
 268     m_initRequest_oi_negotiation_num_langs = 0;
 269     m_initRequest_oi_negotiation_selected = 0;
 270     m_apdu_invalid_session = 0;
 271     m_mem_invalid_session = 0;
 272     m_s2z_odr_init = 0;
 273     m_s2z_odr_search = 0;
 274     m_s2z_odr_scan = 0;
 275     m_s2z_init_apdu = 0;
 276     m_s2z_search_apdu = 0;
 277     m_s2z_present_apdu = 0;
 278     m_s2z_scan_apdu = 0;
 279     m_http_keepalive = 0;
 280     m_http_version = 0;
 281     m_soap_ns = 0;
 282     m_s2z_packing = Z_SRW_recordPacking_string;
 283 #if HAVE_GETTIMEOFDAY
 284     m_time_tv = xmalloc(sizeof(struct timeval));
 285     struct timeval *tv = (struct timeval *) m_time_tv;
 286     tv->tv_sec = 0;
 287     tv->tv_usec = 0;
 288 #else
 289     m_time_tv = 0;
 290 #endif
 291     m_usemarcon_ini_stage1 = 0;
 292     m_usemarcon_ini_stage2 = 0;
 293     m_usemarcon = new Yaz_usemarcon();
 294     if (!m_parent)
 295         low_socket_open();
 296     m_backend_elementset = 0;
 297     m_my_thread = 0;
 298     m_ref_count = 1;
 299     m_main_ptr_dec = false;
 300     m_peername = 0;
 301     m_num_msg_threads = 0;
 302 }
 303
 304 void Yaz_Proxy::inc_ref()
 305 {
 306     m_ref_count++;
 307 }
 308
 309 Yaz_Proxy::~Yaz_Proxy()
 310 {
 311     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 312             m_bytes_sent, m_bytes_recv);
 313     nmem_destroy(m_initRequest_mem);
 314     nmem_destroy(m_mem_invalid_session);
 315     nmem_destroy(m_referenceId_mem);
 316
 317     xfree(m_proxyTarget);
 318     xfree(m_default_target);
 319     xfree(m_proxy_negotiation_charset);
 320     xfree(m_proxy_negotiation_lang);
 321     xfree(m_proxy_negotiation_default_charset);
 322     delete m_charset_converter;
 323     xfree(m_optimize);
 324
 325 #if YAZ_HAVE_XSLT
 326     if (m_stylesheet_xsp)
 327         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 328 #endif
 329     xfree(m_time_tv);
 330
 331     xfree(m_peername);
 332     xfree(m_schema);
 333     xfree(m_backend_type);
 334     xfree(m_backend_charset);
 335     xfree(m_usemarcon_ini_stage1);
 336     xfree(m_usemarcon_ini_stage2);
 337     xfree(m_backend_elementset);
 338     delete m_usemarcon;
 339     if (m_s2z_odr_init)
 340         odr_destroy(m_s2z_odr_init);
 341     if (m_s2z_odr_search)
 342         odr_destroy(m_s2z_odr_search);
 343     if (!m_parent)
 344         low_socket_close();
 345     if (!m_parent)
 346         delete m_my_thread;
 347     delete m_config;
 348 }
 349
 350 void Yaz_Proxy::set_debug_mode(int mode)
 351 {
 352     m_debug_mode = mode;
 353 }
 354
 355 int Yaz_Proxy::set_config(const char *config)
 356 {
 357     delete m_config;
 358     m_config = new Yaz_ProxyConfig();
 359     xfree(m_config_fname);
 360     m_config_fname = xstrdup(config);
 361     int r = m_config->read_xml(config);
 362     if (!r)
 363     {
 364         int period = 60;
 365         m_config->get_generic_info(&m_log_mask, &m_max_clients,
 366                                    &m_max_connect, &m_limit_connect, &period,
 367                                    &m_num_msg_threads);
 368         m_connect.set_period(period);
 369     }
 370     return r;
 371 }
 372
 373 void Yaz_Proxy::set_default_target(const char *target)
 374 {
 375     xfree(m_default_target);
 376     m_default_target = 0;
 377     if (target)
 378         m_default_target = (char *) xstrdup(target);
 379 }
 380
 381 void Yaz_Proxy::set_proxy_negotiation(const char *charset, const char *lang,
 382                                        const char *default_charset)
 383 {
 384     yaz_log(YLOG_DEBUG, "%sSet the proxy negotiation: charset to '%s', "
 385         "default charset to '%s', language to '%s'", m_session_str,
 386         charset?charset:"none",
 387         default_charset?default_charset:"none",
 388         lang?lang:"none");
 389     xfree(m_proxy_negotiation_charset);
 390     xfree(m_proxy_negotiation_lang);
 391     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 392     if (charset)
 393         m_proxy_negotiation_charset = (char *) xstrdup(charset);
 394     if (lang)
 395         m_proxy_negotiation_lang = (char *) xstrdup(lang);
 396     if (default_charset)
 397         m_proxy_negotiation_default_charset =
 398             (char *) xstrdup(default_charset);
 399 }
 400
 401 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 402 {
 403     if (m_parent)
 404         return m_parent->check_reconfigure();
 405
 406     Yaz_ProxyConfig *cfg = m_config;
 407     if (m_reconfig_flag)
 408     {
 409         yaz_log(YLOG_LOG, "reconfigure");
 410         if (m_config_fname && cfg)
 411         {
 412             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 413             int r = cfg->read_xml(m_config_fname);
 414             if (r)
 415                 yaz_log(YLOG_WARN, "reconfigure failed");
 416             else
 417             {
 418                 m_log_mask = 0;
 419                 int period = 60;
 420                 cfg->get_generic_info(&m_log_mask, &m_max_clients,
 421                                       &m_max_connect, &m_limit_connect,
 422                                       &period, &m_num_msg_threads);
 423                 m_connect.set_period(period);
 424             }
 425         }
 426         else
 427             yaz_log(YLOG_LOG, "reconfigure");
 428         m_reconfig_flag = 0;
 429     }
 430     return cfg;
 431 }
 432
 433 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 434                                         *the_PDU_Observable, int fd)
 435 {
 436     check_reconfigure();
 437
 438     char session_str[200];
 439     const char *peername = the_PDU_Observable->getpeername();
 440     if (!peername)
 441         peername = "nullpeer";
 442
 443     if (m_log_mask & PROXY_LOG_IP_CLIENT)
 444         sprintf(session_str, "%ld:%d %.80s %d ",
 445                 (long) time(0), m_session_no, peername, 0);
 446     else
 447         sprintf(session_str, "%ld:%d %d ",
 448                 (long) time(0), m_session_no, 0);
 449     m_session_no++;
 450
 451     yaz_log(YLOG_LOG, "%sNew session %s", session_str, peername);
 452
 453     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 454                                          m_socket_observable, this);
 455
 456     new_proxy->m_config = 0;
 457     new_proxy->m_config_fname = 0;
 458     new_proxy->timeout(m_client_idletime);
 459     new_proxy->m_target_idletime = m_target_idletime;
 460     new_proxy->set_default_target(m_default_target);
 461     new_proxy->m_max_clients = m_max_clients;
 462     new_proxy->m_log_mask = m_log_mask;
 463     new_proxy->m_session_no = m_session_no;
 464     new_proxy->m_num_msg_threads = m_num_msg_threads;
 465
 466 #if 0
 467     // in case we want to watch a particular client..
 468     if (!strcmp(peername, "tcp:163.121.19.82")) // NIS GROUP
 469         new_proxy->m_log_mask = 255;
 470 #endif
 471
 472     new_proxy->set_APDU_log(get_APDU_log());
 473     if (new_proxy->m_log_mask & PROXY_LOG_APDU_CLIENT)
 474         new_proxy->set_APDU_yazlog(1);
 475     else
 476         new_proxy->set_APDU_yazlog(0);
 477     strcpy(new_proxy->m_session_str, session_str);
 478     new_proxy->m_peername = xstrdup(peername);
 479     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 480         m_proxy_negotiation_lang, m_proxy_negotiation_default_charset);
 481     // create thread object the first time we get an incoming connection
 482     if (!m_my_thread && m_num_msg_threads > 0)
 483     {
 484         yaz_log(YLOG_LOG, "%sStarting message thread management. number=%d",
 485                  session_str, m_num_msg_threads);
 486         m_my_thread = new Msg_Thread(m_socket_observable, m_num_msg_threads);
 487     }
 488     new_proxy->m_my_thread = m_my_thread;
 489     return new_proxy;
 490 }
 491
 492 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 493 {
 494     Z_OtherInformationUnit *oi =
 495         update_otherInformation(otherInfo, 0, yaz_oid_userinfo_cookie, 1, 1);
 496
 497     if (oi && oi->which == Z_OtherInfo_characterInfo)
 498         return oi->information.characterInfo;
 499     return 0;
 500 }
 501
 502 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 503 {
 504     Z_OtherInformationUnit *oi =
 505         update_otherInformation(otherInfo, 0, yaz_oid_userinfo_proxy, 1, 1);
 506
 507     if (oi && oi->which == Z_OtherInfo_characterInfo)
 508         return oi->information.characterInfo;
 509     return 0;
 510 }
 511 const char *Yaz_Proxy::load_balance(const char **url)
 512 {
 513     int zurl_in_use[MAX_ZURL_PLEX];
 514     int zurl_in_spare[MAX_ZURL_PLEX];
 515     Yaz_ProxyClient *c;
 516     int i;
 517
 518     for (i = 0; i<MAX_ZURL_PLEX; i++)
 519     {
 520         zurl_in_use[i] = 0;
 521         zurl_in_spare[i] = 0;
 522     }
 523     for (c = m_parent->m_clientPool; c; c = c->m_next)
 524     {
 525         for (i = 0; url[i]; i++)
 526             if (!strcmp(url[i], c->get_hostname()))
 527             {
 528                 zurl_in_use[i]++;
 529                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 530                     zurl_in_spare[i]++;
 531             }
 532     }
 533     int min_use = 100000;
 534     int spare_for_min = 0;
 535     int max_spare = 0;
 536     const char *ret_min = 0;
 537     const char *ret_spare = 0;
 538     for (i = 0; url[i]; i++)
 539     {
 540         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 541                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 542         if (min_use > zurl_in_use[i])
 543         {
 544             ret_min = url[i];
 545             min_use = zurl_in_use[i];
 546             spare_for_min = zurl_in_spare[i];
 547         }
 548         if (max_spare < zurl_in_spare[i])
 549         {
 550             ret_spare = url[i];
 551             max_spare = zurl_in_spare[i];
 552         }
 553     }
 554     return ret_min;
 555 }
 556
 557 int Yaz_Proxy::get_number_of_connections()
 558 {
 559     int no_connections = 0;
 560     Yaz_ProxyClient *c;
 561
 562     for (c = m_parent->m_clientPool; c; c = c->m_next)
 563     {
 564         assert(c->m_prev);
 565         assert(*c->m_prev == c);
 566         if (!strcmp(m_proxyTarget, c->get_hostname()))
 567         {
 568             no_connections++;
 569         }
 570     }
 571     yaz_log(YLOG_LOG, "%sExisting %s connections: %d", m_session_str, m_proxyTarget,
 572         no_connections);
 573     return no_connections;
 574 }
 575
 576 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 577                                        const char *proxy_host, int *http_code)
 578 {
 579     assert(m_parent);
 580     Yaz_Proxy *parent = m_parent;
 581     Yaz_ProxyClient *c = m_client;
 582
 583     if (!m_proxyTarget)
 584     {
 585         const char *url[MAX_ZURL_PLEX];
 586         Yaz_ProxyConfig *cfg = check_reconfigure();
 587         if (proxy_host)
 588         {
 589             if (parent && parent->m_debug_mode)
 590             {
 591                 // only to be enabled for debugging...
 592                 if (!strcmp(proxy_host, "stop"))
 593                     exit(0);
 594             }
 595             xfree(m_default_target);
 596             m_default_target = xstrdup(proxy_host);
 597         }
 598         proxy_host = m_default_target;
 599         int client_idletime = -1;
 600         const char *cql2rpn_fname = 0;
 601         const char *negotiation_charset = 0;
 602         const char *negotiation_lang = 0;
 603         const char *query_charset = 0;
 604         const char *default_client_query_charset = 0;
 605         url[0] = m_default_target;
 606         url[1] = 0;
 607         if (cfg)
 608         {
 609             int pre_init = 0;
 610             cfg->get_target_info(proxy_host, url, &m_bw_max,
 611                                  &m_pdu_max, &m_max_record_retrieve,
 612                                  &m_search_max,
 613                                  &m_target_idletime, &client_idletime,
 614                                  &m_max_sockets,
 615                                  &parent->m_max_clients,
 616                                  &m_keepalive_limit_bw,
 617                                  &m_keepalive_limit_pdu,
 618                                  &pre_init,
 619                                  &cql2rpn_fname,
 620                                  &negotiation_charset,
 621                                  &negotiation_lang,
 622                                  &query_charset,
 623                                  &default_client_query_charset);
 624         }
 625         if (client_idletime != -1)
 626         {
 627             m_client_idletime = client_idletime;
 628             timeout(m_client_idletime);
 629         }
 630
 631         // get those FILE descriptors available
 632         m_parent->low_socket_close();
 633         if (cql2rpn_fname)
 634             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 635         // reserve them again
 636         m_parent->low_socket_open();
 637
 638         if (negotiation_charset || negotiation_lang || default_client_query_charset)
 639         {
 640             set_proxy_negotiation(negotiation_charset,
 641                 negotiation_lang, default_client_query_charset);
 642         }
 643         m_charset_converter->set_target_query_charset(query_charset);
 644         if (!url[0])
 645         {
 646             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 647             return 0;
 648         }
 649         // we don't handle multiplexing for cookie session, so we just
 650         // pick the first one in this case (anonymous users will be able
 651         // to use any backend)
 652         if (cookie && *cookie)
 653             m_proxyTarget = (char*) xstrdup(url[0]);
 654         else
 655             m_proxyTarget = (char*) xstrdup(load_balance(url));
 656     }
 657     if (cookie && *cookie)
 658     {   // search in sessions with a cookie
 659         for (c = parent->m_clientPool; c; c = c->m_next)
 660         {
 661             assert(c->m_prev);
 662             assert(*c->m_prev == c);
 663             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 664                 !strcmp(m_proxyTarget, c->get_hostname()))
 665             {
 666                 // Found it in cache
 667                 // The following handles "cancel"
 668                 // If connection is busy (waiting for PDU) and
 669                 // we have an initRequest we can safely do re-open
 670                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 671                 {
 672                     yaz_log(YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 673                              c->get_hostname());
 674                     c->close();
 675                     c->m_init_flag = 0;
 676
 677                     c->m_last_ok = 0;
 678                     c->m_cache.clear();
 679                     c->m_last_resultCount = 0;
 680                     c->m_sr_transform = 0;
 681                     c->m_waiting = 0;
 682                     c->m_resultSetStartPoint = 0;
 683                     c->m_target_idletime = m_target_idletime;
 684                     if (c->client(m_proxyTarget))
 685                     {
 686                         delete c;
 687                         return 0;
 688                     }
 689                     c->timeout(30);
 690                 }
 691                 c->m_seqno = parent->m_seqno;
 692                 if (c->m_server && c->m_server != this)
 693                     c->m_server->m_client = 0;
 694                 c->m_server = this;
 695                 (parent->m_seqno)++;
 696                 yaz_log(YLOG_DEBUG, "get_client 1 %p %p", this, c);
 697                 return c;
 698             }
 699         }
 700     }
 701     else if (!c && apdu->which == Z_APDU_initRequest )
 702     {
 703         // anonymous sessions without cookie.
 704         // if authentication is set it is NOT anonymous se we can't share them.
 705         // If charset and lang negotiation is use it is NOT anonymous session too.
 706         for (c = parent->m_clientPool; c; c = c->m_next)
 707         {
 708             assert(c->m_prev);
 709             assert(*c->m_prev == c);
 710             if (c->m_server == 0 && c->m_cookie == 0 && c->m_waiting == 0
 711                 && c->compare_idAuthentication(apdu)
 712                 && c->compare_charset(apdu)
 713                 && !strcmp(m_proxyTarget, c->get_hostname()))
 714             {
 715                 // found it in cache
 716                 yaz_log(YLOG_LOG, "%sREUSE %d %s",
 717                          m_session_str, parent->m_seqno, c->get_hostname());
 718
 719                 c->m_seqno = parent->m_seqno;
 720                 assert(c->m_server == 0);
 721                 c->m_server = this;
 722
 723                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 724                     c->set_APDU_yazlog(1);
 725                 else
 726                     c->set_APDU_yazlog(0);
 727
 728                 (parent->m_seqno)++;
 729
 730                 parent->pre_init();
 731
 732                 return c;
 733             }
 734         }
 735     }
 736     if (!m_client)
 737     {
 738         if (apdu->which != Z_APDU_initRequest)
 739         {
 740             yaz_log(YLOG_LOG, "%sno init request as first PDU", m_session_str);
 741             *http_code = 500;
 742             return 0;
 743         }
 744
 745         int no_in_use = get_number_of_connections();
 746         if (no_in_use >= m_max_sockets)
 747         {
 748             yaz_log(YLOG_LOG, "%smax sockets reached %d", m_session_str,
 749                 m_max_sockets);
 750             *http_code = 500;
 751             return 0;
 752         }
 753         // go through list of clients - and find the lowest/oldest one.
 754         Yaz_ProxyClient *c_min = 0;
 755         int min_seq = -1;
 756         int no_of_clients = 0;
 757         if (parent->m_clientPool)
 758             yaz_log(YLOG_DEBUG, "Existing sessions");
 759         for (c = parent->m_clientPool; c; c = c->m_next)
 760         {
 761             yaz_log(YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 762                                c->m_waiting, c->get_hostname(),
 763                                c->m_cookie ? c->m_cookie : "");
 764             no_of_clients++;
 765             if (min_seq < 0 || c->m_seqno < min_seq)
 766             {
 767                 min_seq = c->m_seqno;
 768                 c_min = c;
 769             }
 770         }
 771         if (no_of_clients >= parent->m_max_clients)
 772         {
 773             c = c_min;
 774             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 775             {
 776                 yaz_log(YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 777                          m_session_str, parent->m_max_clients, c->m_seqno);
 778                 if (c->m_server && c->m_server != this)
 779                     c->m_server->dec_ref();
 780             }
 781             else
 782             {
 783                 yaz_log(YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 784                          m_session_str, parent->m_max_clients,
 785                          c->m_seqno, parent->m_seqno, c->get_hostname());
 786                 xfree(c->m_cookie);
 787                 c->m_cookie = 0;
 788                 if (cookie)
 789                     c->m_cookie = xstrdup(cookie);
 790                 c->m_seqno = parent->m_seqno;
 791                 if (c->m_server && c->m_server != this)
 792                 {
 793                     c->m_server->m_client = 0;
 794                     c->m_server->dec_ref();
 795                 }
 796                 (parent->m_seqno)++;
 797                 c->m_target_idletime = m_target_idletime;
 798                 c->timeout(m_target_idletime);
 799
 800                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 801                     c->set_APDU_yazlog(1);
 802                 else
 803                     c->set_APDU_yazlog(0);
 804
 805                 return c;
 806             }
 807         }
 808         else
 809         {
 810
 811             yaz_log(YLOG_LOG, "%sNEW %d %s",
 812                      m_session_str, parent->m_seqno, m_proxyTarget);
 813             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 814             c->m_next = parent->m_clientPool;
 815             if (c->m_next)
 816                 c->m_next->m_prev = &c->m_next;
 817             parent->m_clientPool = c;
 818             c->m_prev = &parent->m_clientPool;
 819         }
 820
 821         xfree(c->m_cookie);
 822         c->m_cookie = 0;
 823         if (cookie)
 824             c->m_cookie = xstrdup(cookie);
 825
 826         c->m_seqno = parent->m_seqno;
 827         c->m_init_flag = 0;
 828         c->m_last_resultCount = 0;
 829         c->m_last_ok = 0;
 830         c->m_cache.clear();
 831         c->m_sr_transform = 0;
 832         c->m_waiting = 0;
 833         c->m_resultSetStartPoint = 0;
 834         (parent->m_seqno)++;
 835         if (c->client(m_proxyTarget))
 836         {
 837             delete c;
 838             return 0;
 839         }
 840         c->m_target_idletime = m_target_idletime;
 841         c->timeout(30);
 842
 843         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 844             c->set_APDU_yazlog(1);
 845         else
 846             c->set_APDU_yazlog(0);
 847
 848         c->set_idAuthentication(apdu);
 849     }
 850     yaz_log(YLOG_DEBUG, "get_client 3 %p %p", this, c);
 851     return c;
 852 }
 853
 854 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 855 {
 856     int i;
 857     for (i = 0; i<num; i++)
 858     {
 859         Z_DefaultDiagFormat *r;
 860         Z_DiagRec *p = pp[i];
 861         if (p->which != Z_DiagRec_defaultFormat)
 862         {
 863             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 864             return;
 865         }
 866         else
 867             r = p->u.defaultFormat;
 868         switch (r->which)
 869         {
 870         case Z_DefaultDiagFormat_v2Addinfo:
 871             yaz_log(YLOG_LOG, "%sError " ODR_INT_PRINTF " %s:%s",
 872                     m_session_str,
 873                     *r->condition, diagbib1_str(*r->condition),
 874                     r->u.v2Addinfo);
 875             break;
 876         case Z_DefaultDiagFormat_v3Addinfo:
 877             yaz_log(YLOG_LOG, "%sError " ODR_INT_PRINTF " %s:%s",
 878                     m_session_str,
 879                     *r->condition, diagbib1_str(*r->condition),
 880                     r->u.v3Addinfo);
 881             break;
 882         }
 883     }
 884 }
 885
 886 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 887 {
 888     if (!m_stylesheet_xsp || p->num_records <= 0)
 889     {
 890         return 0;  /* no XSLT to be done ... */
 891     }
 892
 893     m_stylesheet_offset = 0;
 894     m_stylesheet_nprl = p;
 895     m_stylesheet_apdu = apdu;
 896     m_timeout_mode = timeout_xsl;
 897
 898     timeout(0);
 899     return 1;
 900 }
 901
 902 void Yaz_Proxy::convert_xsl_delay()
 903 {
 904 #if YAZ_HAVE_XSLT
 905     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 906     if (npr->which == Z_NamePlusRecord_databaseRecord)
 907     {
 908         Z_External *r = npr->u.databaseRecord;
 909         if (r->which == Z_External_octet)
 910         {
 911 #if 0
 912             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 913 #endif
 914             xmlDocPtr res, doc = xmlParseMemory(
 915                 (char*) r->u.octet_aligned->buf,
 916                 r->u.octet_aligned->len);
 917
 918
 919             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 920                     m_session_str, m_stylesheet_offset);
 921             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 922                                       doc, 0);
 923
 924             if (res)
 925             {
 926                 xmlChar *out_buf;
 927                 int out_len;
 928                 xmlDocDumpFormatMemory(res, &out_buf, &out_len, 1);
 929                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 930                     u.databaseRecord =
 931                     z_ext_record_oid(odr_encode(), yaz_oid_recsyn_xml,
 932                                      (char*) out_buf, out_len);
 933                 xmlFree(out_buf);
 934                 xmlFreeDoc(res);
 935             }
 936
 937             xmlFreeDoc(doc);
 938         }
 939     }
 940 #endif
 941     m_stylesheet_offset++;
 942     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 943     {
 944         m_timeout_mode = timeout_normal;
 945         m_stylesheet_nprl = 0;
 946 #if YAZ_HAVE_XSLT
 947         if (m_stylesheet_xsp)
 948             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 949 #endif
 950         m_stylesheet_xsp = 0;
 951         timeout(m_client_idletime);
 952         send_PDU_convert(m_stylesheet_apdu);
 953     }
 954     else
 955         timeout(0);
 956 }
 957
 958 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 959 {
 960     if (m_frontend_type[0] != -1)
 961     {
 962         int i;
 963         for (i = 0; i < p->num_records; i++)
 964         {
 965             Z_NamePlusRecord *npr = p->records[i];
 966             if (npr->which == Z_NamePlusRecord_databaseRecord)
 967             {
 968                 Z_External *r = npr->u.databaseRecord;
 969                 if (r->which == Z_External_octet)
 970                 {
 971 #if !HAVE_USEMARCON
 972                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 973                         yaz_log(YLOG_LOG, "%sError: USEMARCON requested but not available",
 974                             m_session_str);
 975 #endif
 976 #if HAVE_USEMARCON
 977                     yaz_log(YLOG_DEBUG, "%sUSEMARCON stage1=%s stage2=%s",
 978                         m_session_str,
 979                         m_usemarcon_ini_stage1 ? m_usemarcon_ini_stage1 : "(none)",
 980                         m_usemarcon_ini_stage2 ? m_usemarcon_ini_stage2 : "(none)");
 981                     char *converted;
 982                     int convlen;
 983                     if (m_usemarcon->convert(m_usemarcon_ini_stage1, m_usemarcon_ini_stage2,
 984                         (char*) r->u.octet_aligned->buf, r->u.octet_aligned->len,
 985                         &converted, &convlen))
 986                     {
 987                         npr->u.databaseRecord =
 988                             z_ext_record_oid(odr_encode(),
 989                                              m_frontend_type,
 990                                              converted,
 991                                              strlen(converted));
 992                         free(converted);
 993                     }
 994                     else
 995 #endif
 996 /* HAVE_USEMARCON */
 997                     npr->u.databaseRecord =
 998                         z_ext_record_oid(odr_encode(),
 999                                          m_frontend_type,
1000                                          (char*) r->u.octet_aligned->buf,
1001                                          r->u.octet_aligned->len);
1002                 }
1003             }
1004         }
1005     }
1006 }
1007
1008 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
1009                                         const char *backend_charset)
1010 {
1011     int sel =   m_charset_converter->get_client_charset_selected();
1012     const char *client_record_charset =
1013         m_charset_converter->get_client_query_charset();
1014     if (sel && backend_charset && client_record_charset &&
1015         strcmp(backend_charset, client_record_charset))
1016     {
1017         int i;
1018         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
1019                                         backend_charset);
1020         yaz_marc_t mt = yaz_marc_create();
1021         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
1022         yaz_marc_iconv(mt, cd);
1023         for (i = 0; i < p->num_records; i++)
1024         {
1025             Z_NamePlusRecord *npr = p->records[i];
1026             if (npr->which == Z_NamePlusRecord_databaseRecord)
1027             {
1028                 Z_External *r = npr->u.databaseRecord;
1029                 const Odr_oid *oid = r->direct_reference;
1030                 if (!oid)
1031                     continue;
1032
1033                 if (!oid_oidcmp(oid, yaz_oid_recsyn_sutrs))
1034                 {
1035                     WRBUF w = wrbuf_alloc();
1036
1037                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
1038                                       r->u.octet_aligned->len);
1039                     npr->u.databaseRecord =
1040                         z_ext_record_oid(odr_encode(), oid, wrbuf_buf(w),
1041                                          wrbuf_len(w));
1042                     wrbuf_destroy(w);
1043                 }
1044                 else if (!oid_oidcmp(oid, yaz_oid_recsyn_xml))
1045                 {
1046                     ;
1047                 }
1048                 else if (r->which == Z_External_octet)
1049                 {
1050                     size_t rlen;
1051                     const char *result;
1052                     if (yaz_marc_decode_buf(mt,
1053                                             (char*) r->u.octet_aligned->buf,
1054                                             r->u.octet_aligned->len,
1055                                             &result, &rlen))
1056                     {
1057                         npr->u.databaseRecord =
1058                             z_ext_record_oid(odr_encode(), oid, result, rlen);
1059                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1060                                 m_session_str);
1061                     }
1062                 }
1063             }
1064         }
1065         if (cd)
1066             yaz_iconv_close(cd);
1067         yaz_marc_destroy(mt);
1068     }
1069 }
1070
1071 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1072                                    const char *backend_charset)
1073 {
1074     int i;
1075     if (!backend_charset)
1076         backend_charset = "MARC-8";
1077     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1078     yaz_marc_t mt = yaz_marc_create();
1079     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1080     yaz_marc_iconv(mt, cd);
1081     for (i = 0; i < p->num_records; i++)
1082     {
1083         Z_NamePlusRecord *npr = p->records[i];
1084         if (npr->which == Z_NamePlusRecord_databaseRecord)
1085         {
1086             Z_External *r = npr->u.databaseRecord;
1087             if (r->which == Z_External_OPAC)
1088             {
1089                 WRBUF w = wrbuf_alloc();
1090
1091                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
1092                 npr->u.databaseRecord = z_ext_record_oid(
1093                     odr_encode(), yaz_oid_recsyn_xml,
1094                     wrbuf_buf(w), wrbuf_len(w));
1095                 wrbuf_destroy(w);
1096             }
1097             else if (r->which == Z_External_octet)
1098             {
1099                 size_t rlen;
1100                 const char *result;
1101                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1102                                         r->u.octet_aligned->len,
1103                                         &result, &rlen))
1104                 {
1105                     npr->u.databaseRecord =
1106                         z_ext_record_oid(odr_encode(), yaz_oid_recsyn_xml,
1107                                          result, rlen);
1108                 }
1109             }
1110         }
1111     }
1112     if (cd)
1113         yaz_iconv_close(cd);
1114     yaz_marc_destroy(mt);
1115 }
1116
1117 void Yaz_Proxy::logtime()
1118 {
1119 #if HAVE_GETTIMEOFDAY
1120     struct timeval *tv = (struct timeval*) m_time_tv;
1121     if (tv->tv_sec)
1122     {
1123         struct timeval tv1;
1124         gettimeofday(&tv1, 0);
1125         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1126             (tv1.tv_usec - tv->tv_usec);
1127         if (diff >= 0)
1128             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1129                     diff/1000000, (diff/1000)%1000);
1130     }
1131     tv->tv_sec = 0;
1132     tv->tv_usec = 0;
1133 #endif
1134 }
1135
1136 int Yaz_Proxy::send_http_response(int code)
1137 {
1138     ODR o = odr_encode();
1139     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1140     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1141     if (m_http_version)
1142         hres->version = odr_strdup(o, m_http_version);
1143     if (m_http_keepalive)
1144         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1145     else
1146         timeout(0);
1147     if (code == 401)
1148         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate",
1149                           "Basic realm=\"YAZ Proxy\"");
1150
1151
1152     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1153     {
1154         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1155                  gdu_name(gdu));
1156     }
1157     int len;
1158     int r = send_GDU(gdu, &len);
1159     m_bytes_sent += len;
1160     m_bw_stat.add_bytes(len);
1161     logtime();
1162
1163     recv_GDU_more(true);
1164
1165     return r;
1166 }
1167
1168 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */)
1169 {
1170     ODR o = odr_encode();
1171     const char *ctype = "text/xml";
1172     Z_GDU *gdu = z_get_HTTP_Response(o, http_code);
1173     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1174     if (m_http_version)
1175         hres->version = odr_strdup(o, m_http_version);
1176     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1177     if (m_http_keepalive)
1178         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1179     else
1180         timeout(0);
1181     if (http_code == 401)
1182         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\"");
1183
1184     static Z_SOAP_Handler soap_handlers[2] = {
1185 #if YAZ_HAVE_XSLT
1186         { (char*) "http://www.loc.gov/zing/srw/", 0,
1187          (Z_SOAP_fun) yaz_srw_codec},
1188 #endif
1189         {0, 0, 0}
1190     };
1191
1192     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1193     soap_package->which = Z_SOAP_generic;
1194     soap_package->u.generic =
1195         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1196     soap_package->u.generic->no = 0;
1197     soap_package->u.generic->ns = soap_handlers[0].ns;
1198     soap_package->u.generic->p = (void *) srw_pdu;
1199     soap_package->ns = m_soap_ns;
1200     z_soap_codec_enc_xsl(o, &soap_package,
1201                          &hres->content_buf, &hres->content_len,
1202                          soap_handlers, 0, m_s2z_stylesheet);
1203     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1204     {
1205         yaz_log(YLOG_LOG, "%sSending %s to client", m_session_str,
1206                 gdu_name(gdu));
1207     }
1208     int len;
1209     int r = send_GDU(gdu, &len);
1210     m_bytes_sent += len;
1211     m_bw_stat.add_bytes(len);
1212     logtime();
1213
1214     recv_GDU_more(true);
1215
1216     return r;
1217 }
1218
1219 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1220 {
1221     ODR o = odr_encode();
1222     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1223         odr_malloc(o, sizeof(*diagnostic));
1224     int num_diagnostic = 1;
1225     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1226     return send_srw_search_response(diagnostic, num_diagnostic, srw_error == 3 ? 401 : 200);
1227 }
1228
1229 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1230                              Z_DefaultDiagFormat *ddf)
1231 {
1232     int bib1_code = *ddf->condition;
1233     if (bib1_code == 109)
1234         return 404;
1235     srw_res->num_diagnostics = 1;
1236     srw_res->diagnostics = (Z_SRW_diagnostic *)
1237         odr_malloc(o, sizeof(*srw_res->diagnostics));
1238     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1239                           yaz_diag_bib1_to_srw(*ddf->condition),
1240                           ddf->u.v2Addinfo);
1241     return 0;
1242 }
1243
1244 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_scanResponse *srw_res,
1245                              Z_DiagRec *dr)
1246 {
1247     if (dr->which == Z_DiagRec_defaultFormat)
1248     {
1249         int bib1_code = *dr->u.defaultFormat->condition;
1250         if (bib1_code == 109)
1251             return 404;
1252         srw_res->num_diagnostics = 1;
1253         srw_res->diagnostics = (Z_SRW_diagnostic *)
1254             odr_malloc(o, sizeof(*srw_res->diagnostics));
1255         yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1256                           yaz_diag_bib1_to_srw(*dr->u.defaultFormat->condition),
1257                           dr->u.defaultFormat->u.v2Addinfo);
1258     }
1259     return 0;
1260 }
1261
1262 #if YAZ_HAS_MK_SURROGATE
1263 #else
1264 static void yazproxy_mk_sru_surrogate(ODR o, Z_SRW_record *record, int pos,
1265                           int code, const char *details)
1266 {
1267     const char *message = yaz_diag_srw_str(code);
1268     int len = 200;
1269     if (message)
1270         len += strlen(message);
1271     if (details)
1272         len += strlen(details);
1273
1274     record->recordData_buf = (char *) odr_malloc(o, len);
1275
1276     sprintf(record->recordData_buf, "<diagnostic "
1277             "xmlns=\"http://www.loc.gov/zing/srw/diagnostic/\">\n"
1278             " <uri>info:srw/diagnostic/1/%d</uri>\n", code);
1279     if (details)
1280         sprintf(record->recordData_buf + strlen(record->recordData_buf),
1281                 " <details>%s</details>\n", details);
1282     if (message)
1283         sprintf(record->recordData_buf + strlen(record->recordData_buf),
1284                 " <message>%s</message>\n", message);
1285     sprintf(record->recordData_buf + strlen(record->recordData_buf),
1286             "</diagnostic>\n");
1287     record->recordData_len = strlen(record->recordData_buf);
1288     record->recordPosition = odr_intdup(o, pos);
1289     record->recordSchema = odr_strdup(o, "info:srw/schema/1/diagnostics-v1.1");
1290 }
1291 #endif
1292
1293 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1294 {
1295     ODR o = odr_encode();
1296     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1297     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1298
1299     srw_res->numberOfRecords = odr_intdup (o, hits);
1300     if (records && records->which == Z_Records_DBOSD)
1301     {
1302         srw_res->num_records =
1303             records->u.databaseOrSurDiagnostics->num_records;
1304         int i;
1305         srw_res->records = (Z_SRW_record *)
1306             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1307         for (i = 0; i < srw_res->num_records; i++)
1308         {
1309             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1310             if (npr->which != Z_NamePlusRecord_databaseRecord)
1311             {
1312 #if YAZ_HAS_MK_SURROGATE
1313                 yaz_mk_sru_surrogate(
1314                     o, srw_res->records + i, i+start,
1315                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1316 #else
1317                 yazproxy_mk_sru_surrogate(
1318                     o, srw_res->records + i, i+start,
1319                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1320 #endif
1321                 continue;
1322             }
1323             Z_External *r = npr->u.databaseRecord;
1324
1325             if (r->which == Z_External_octet
1326                 && !oid_oidcmp(r->direct_reference, yaz_oid_recsyn_xml))
1327             {
1328                 srw_res->records[i].recordSchema = m_schema;
1329                 srw_res->records[i].recordPacking = m_s2z_packing;
1330                 srw_res->records[i].recordData_buf = (char*)
1331                     r->u.octet_aligned->buf;
1332                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1333                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1334             }
1335             else
1336             {
1337 #if YAZ_HAS_MK_SURROGATE
1338                 yaz_mk_sru_surrogate(
1339                     o, srw_res->records + i, i+start,
1340                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1341 #else
1342                 yazproxy_mk_sru_surrogate(
1343                     o, srw_res->records + i, i+start,
1344                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1345 #endif
1346             }
1347         }
1348     }
1349     if (records && records->which == Z_Records_NSD)
1350     {
1351         int http_code;
1352         http_code = z_to_srw_diag(odr_encode(), srw_res,
1353                                    records->u.nonSurrogateDiagnostic);
1354         if (http_code)
1355             return send_http_response(http_code);
1356     }
1357     return send_srw_response(srw_pdu);
1358
1359 }
1360
1361 int Yaz_Proxy::send_to_srw_client_ok(Z_ListEntries *entries)
1362 {
1363     ODR o = odr_encode();
1364     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_scan_response);
1365     Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1366
1367     if (entries && entries->num_entries > 0)
1368     {
1369         srw_res->num_terms = entries->num_entries;
1370         int i;
1371         srw_res->terms = (Z_SRW_scanTerm *)
1372             odr_malloc(o, srw_res->num_terms * sizeof(Z_SRW_scanTerm));
1373         for (i = 0; i < srw_res->num_terms; i++)
1374         {
1375             if (entries->entries[i]->which == Z_Entry_termInfo)
1376             {
1377                 switch(entries->entries[i]->u.termInfo->term->which)
1378                 {
1379                 case Z_Term_general:
1380                     srw_res->terms[i].value = odr_strdupn(o,
1381                         (char *) entries->entries[i]->u.termInfo->term->u.general->buf,
1382                         entries->entries[i]->u.termInfo->term->u.general->len);
1383                     break;
1384                 default:
1385                     srw_res->terms[i].value = NULL;
1386                 }
1387                 if (entries->entries[i]->u.termInfo->globalOccurrences != NULL)
1388                     srw_res->terms[i].numberOfRecords = odr_intdup(o,
1389                         *entries->entries[i]->u.termInfo->globalOccurrences);
1390                 else
1391                     srw_res->terms[i].numberOfRecords = NULL;
1392                 if (entries->entries[i]->u.termInfo->displayTerm != NULL)
1393                     srw_res->terms[i].displayTerm = odr_strdup(o,
1394                         entries->entries[i]->u.termInfo->displayTerm);
1395                 else
1396                     srw_res->terms[i].displayTerm = NULL;
1397                 srw_res->terms[i].whereInList = NULL;
1398             }
1399         }
1400     }
1401     if (entries && entries->num_nonsurrogateDiagnostics > 0)
1402     {
1403         int http_code;
1404         http_code = z_to_srw_diag(odr_encode(), srw_res,
1405                                    entries->nonsurrogateDiagnostics[0]);
1406         if (http_code)
1407             return send_http_response(http_code);
1408     }
1409     return send_srw_response(srw_pdu);
1410
1411 }
1412
1413 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1414                                         int num_diagnostics, int http_code /* = 200 */)
1415 {
1416     ODR o = odr_encode();
1417     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1418     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1419
1420     srw_res->num_diagnostics = num_diagnostics;
1421     srw_res->diagnostics = diagnostics;
1422     return send_srw_response(srw_pdu, http_code);
1423 }
1424
1425 int Yaz_Proxy::send_srw_scan_response(Z_SRW_diagnostic *diagnostics,
1426                                         int num_diagnostics, int http_code /* = 200 */)
1427 {
1428     ODR o = odr_encode();
1429     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_scan_response);
1430     Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1431
1432     srw_res->num_diagnostics = num_diagnostics;
1433     srw_res->diagnostics = diagnostics;
1434     return send_srw_response(srw_pdu, http_code);
1435 }
1436
1437 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1438                                         int num_diagnostics)
1439 {
1440     int http_status = 404;
1441     Yaz_ProxyConfig *cfg = check_reconfigure();
1442     if (cfg)
1443     {
1444         int len;
1445         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1446                                        m_s2z_database, &len, &http_status);
1447         if (b)
1448         {
1449             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1450             Z_SRW_explainResponse *er = res->u.explain_response;
1451
1452             er->record.recordData_buf = b;
1453             er->record.recordData_len = len;
1454             er->record.recordPacking = m_s2z_packing;
1455             er->record.recordSchema = odr_strdup(odr_encode(),
1456                                                  "http://explain.z3950.org/dtd/2.0/");
1457
1458             er->diagnostics = diagnostics;
1459             er->num_diagnostics = num_diagnostics;
1460             return send_srw_response(res);
1461         }
1462     }
1463     return send_http_response(http_status);
1464 }
1465
1466 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1467 {
1468     if (m_http_version)
1469     {
1470         if (apdu->which == Z_APDU_initResponse)
1471         {
1472             Z_InitResponse *res = apdu->u.initResponse;
1473             if (*res->result == 0)
1474             {
1475                 send_to_srw_client_error(3, 0);
1476             }
1477             else if (m_s2z_search_apdu)
1478             {
1479                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1480             }
1481             else if (m_s2z_scan_apdu)
1482             {
1483                 handle_incoming_Z_PDU(m_s2z_scan_apdu);
1484             }
1485             else
1486             {
1487                 send_srw_explain_response(0, 0);
1488             }
1489         }
1490         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1491         {
1492             m_s2z_search_apdu = 0;
1493             Z_SearchResponse *res = apdu->u.searchResponse;
1494             m_s2z_hit_count = *res->resultCount;
1495             if (res->records && res->records->which == Z_Records_NSD)
1496             {
1497                 send_to_srw_client_ok(0, res->records, 1);
1498             }
1499             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1500             {
1501                 // adjust
1502                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1503
1504                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1505                 {
1506                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1507                         > m_s2z_hit_count)
1508                         *pr->numberOfRecordsRequested =
1509                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1510                 }
1511                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1512             }
1513             else
1514             {
1515                 m_s2z_present_apdu = 0;
1516                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1517             }
1518         }
1519         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1520         {
1521             int start =
1522                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1523
1524             m_s2z_present_apdu = 0;
1525             Z_PresentResponse *res = apdu->u.presentResponse;
1526             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1527         }
1528         else if (m_s2z_scan_apdu && apdu->which == Z_APDU_scanResponse)
1529         {
1530             Z_ScanResponse *res = apdu->u.scanResponse;
1531             send_to_srw_client_ok(res->entries);
1532         }
1533     }
1534     else
1535     {
1536         int len = 0;
1537         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1538             yaz_log(YLOG_LOG, "%sSending %s to client", m_session_str,
1539                      apdu_name(apdu));
1540         int r = send_Z_PDU(apdu, &len);
1541         m_bytes_sent += len;
1542         m_bw_stat.add_bytes(len);
1543         logtime();
1544         return r;
1545     }
1546     return 0;
1547 }
1548
1549 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1550 {
1551     int kill_session = 0;
1552     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1553
1554     if (new_id)
1555         *new_id = m_referenceId;
1556
1557     if (apdu->which == Z_APDU_searchResponse)
1558     {
1559         Z_SearchResponse *sr = apdu->u.searchResponse;
1560         Z_Records *p = sr->records;
1561         if (p && p->which == Z_Records_NSD)
1562         {
1563             Z_DiagRec dr, *dr_p = &dr;
1564             dr.which = Z_DiagRec_defaultFormat;
1565             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1566
1567             *sr->searchStatus = 0;
1568             display_diagrecs(&dr_p, 1);
1569         }
1570         else
1571         {
1572             if (p && p->which == Z_Records_DBOSD)
1573             {
1574                 if (m_backend_type
1575 #if HAVE_USEMARCON
1576                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1577 #endif
1578                     )
1579                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1580                 if (m_marcxml_mode == marcxml)
1581                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1582                                        m_backend_charset);
1583                 else
1584                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1585                                             m_backend_charset);
1586                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1587                     return 0;
1588
1589             }
1590             if (sr->resultCount)
1591             {
1592                 yaz_log(YLOG_LOG, "%s" ODR_INT_PRINTF " hits", m_session_str,
1593                         *sr->resultCount);
1594                 if (*sr->resultCount < 0)
1595                 {
1596                     m_flag_invalid_session = 1;
1597                     kill_session = 1;
1598
1599                     *sr->searchStatus = 0;
1600                     sr->records =
1601                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1602                     *sr->resultCount = 0;
1603                 }
1604             }
1605         }
1606     }
1607     else if (apdu->which == Z_APDU_presentResponse)
1608     {
1609         Z_PresentResponse *sr = apdu->u.presentResponse;
1610         Z_Records *p = sr->records;
1611         if (p && p->which == Z_Records_NSD)
1612         {
1613             Z_DiagRec dr, *dr_p = &dr;
1614             dr.which = Z_DiagRec_defaultFormat;
1615             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1616             if (*sr->presentStatus == Z_PresentStatus_success)
1617                 *sr->presentStatus = Z_PresentStatus_failure;
1618             display_diagrecs(&dr_p, 1);
1619         }
1620         if (p && p->which == Z_Records_DBOSD)
1621         {
1622             if (m_backend_type
1623 #if HAVE_USEMARCON
1624                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1625 #endif
1626                 )
1627                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1628             if (m_marcxml_mode == marcxml)
1629                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1630                                    m_backend_charset);
1631             else
1632                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1633                                         m_backend_charset);
1634             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1635                 return 0;
1636         }
1637     }
1638     else if (apdu->which == Z_APDU_initResponse)
1639     {
1640         //Get and check negotiation record
1641         //from init response.
1642         handle_charset_lang_negotiation(apdu);
1643
1644         if (m_initRequest_options)
1645         {
1646             Z_Options *nopt =
1647                 (Odr_bitmask *)odr_malloc(odr_encode(),
1648                                           sizeof(Odr_bitmask));
1649             ODR_MASK_ZERO(nopt);
1650
1651             int i;
1652             for (i = 0; i<24; i++)
1653                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1654                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1655                     ODR_MASK_SET(nopt, i);
1656             apdu->u.initResponse->options = nopt;
1657         }
1658         if (m_initRequest_version)
1659         {
1660             Z_ProtocolVersion *nopt =
1661                 (Odr_bitmask *)odr_malloc(odr_encode(),
1662                                           sizeof(Odr_bitmask));
1663             ODR_MASK_ZERO(nopt);
1664
1665             int i;
1666             for (i = 0; i<8; i++)
1667                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1668                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1669                     ODR_MASK_SET(nopt, i);
1670             apdu->u.initResponse->protocolVersion = nopt;
1671         }
1672         apdu->u.initResponse->preferredMessageSize =
1673             odr_intdup(odr_encode(),
1674                        m_client->m_initResponse_preferredMessageSize >
1675                        m_initRequest_preferredMessageSize ?
1676                        m_initRequest_preferredMessageSize :
1677                        m_client->m_initResponse_preferredMessageSize);
1678         apdu->u.initResponse->maximumRecordSize =
1679             odr_intdup(odr_encode(),
1680                        m_client->m_initResponse_maximumRecordSize >
1681                        m_initRequest_maximumRecordSize ?
1682                        m_initRequest_maximumRecordSize :
1683                        m_client->m_initResponse_maximumRecordSize);
1684     }
1685
1686     int r = send_PDU_convert(apdu);
1687     if (r)
1688         return r;
1689     if (kill_session)
1690     {
1691         delete m_client;
1692         m_client = 0;
1693         m_parent->pre_init();
1694     }
1695     return r;
1696 }
1697
1698 void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu)
1699 {
1700     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1701
1702     odr_reset(m_idAuthentication_odr);
1703     z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0);
1704     m_idAuthentication_ber_buf =
1705         odr_getbuf(m_idAuthentication_odr,
1706                    &m_idAuthentication_ber_size, 0);
1707 }
1708
1709 bool Yaz_ProxyClient::compare_charset(Z_APDU *apdu)
1710 {
1711     return true;
1712 }
1713
1714 bool Yaz_ProxyClient::compare_idAuthentication(Z_APDU *apdu)
1715 {
1716     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1717     ODR odr = odr_createmem(ODR_ENCODE);
1718
1719     z_IdAuthentication(odr, &t, 1, 0);
1720     int sz;
1721     char *buf = odr_getbuf(odr, &sz, 0);
1722     if (buf && m_idAuthentication_ber_buf
1723         && sz == m_idAuthentication_ber_size
1724         && !memcmp(m_idAuthentication_ber_buf, buf, sz))
1725     {
1726         odr_destroy(odr);
1727         return true;
1728     }
1729     odr_destroy(odr);
1730     if (!buf && !m_idAuthentication_ber_buf)
1731         return true;
1732     return false;
1733 }
1734
1735 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1736 {
1737     int len = 0;
1738     const char *apdu_name_tmp = apdu_name(apdu);
1739     int r = send_Z_PDU(apdu, &len);
1740     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1741         yaz_log(YLOG_LOG, "%sSending %s to %s %d bytes",
1742                  get_session_str(),
1743                  apdu_name_tmp, get_hostname(), len);
1744     m_bytes_sent += len;
1745     return r;
1746 }
1747
1748 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1749 {
1750     if (apdu->which == Z_APDU_presentRequest)
1751     {
1752         Z_PresentRequest *pr = apdu->u.presentRequest;
1753         int toget = *pr->numberOfRecordsRequested;
1754         int start = *pr->resultSetStartPoint;
1755
1756         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1757                 pr->resultSetId, start, toget);
1758
1759         if (*m_parent->m_optimize == '0')
1760             return apdu;
1761
1762         if (!m_client->m_last_resultSetId)
1763         {
1764             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1765             new_apdu->u.presentResponse->records =
1766                 create_nonSurrogateDiagnostics(
1767                     odr_encode(),
1768                     YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
1769                     pr->resultSetId);
1770             send_to_client(new_apdu);
1771             return 0;
1772         }
1773         if (start < 1 || toget < 0)
1774         {
1775             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1776             new_apdu->u.presentResponse->records =
1777                 create_nonSurrogateDiagnostics(
1778                     odr_encode(),
1779                     YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
1780                     0);
1781             send_to_client(new_apdu);
1782             return 0;
1783         }
1784         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1785         {
1786             if (start+toget-1 > m_client->m_last_resultCount)
1787             {
1788                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1789                 new_apdu->u.presentResponse->records =
1790                     create_nonSurrogateDiagnostics(
1791                         odr_encode(),
1792                         YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
1793                         0);
1794                 send_to_client(new_apdu);
1795                 return 0;
1796             }
1797             Z_NamePlusRecordList *npr;
1798 #if 0
1799             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1800                     m_session_str, start, toget, yaz_z3950oid_to_str(
1801                         pr->preferredRecordSyntax, &oclass));
1802 #endif
1803             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1804                                           pr->preferredRecordSyntax,
1805                                           pr->recordComposition))
1806             {
1807                 yaz_log(YLOG_LOG, "%sReturned cached records for present request",
1808                          m_session_str);
1809                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1810                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1811
1812                 new_apdu->u.presentResponse->numberOfRecordsReturned
1813                     = odr_intdup(odr_encode(), toget);
1814
1815                 new_apdu->u.presentResponse->records = (Z_Records*)
1816                     odr_malloc(odr_encode(), sizeof(Z_Records));
1817                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1818                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1819                 new_apdu->u.presentResponse->nextResultSetPosition =
1820                     odr_intdup(odr_encode(), start+toget);
1821
1822                 send_to_client(new_apdu);
1823                 return 0;
1824             }
1825         }
1826     }
1827
1828     if (apdu->which != Z_APDU_searchRequest)
1829         return apdu;
1830     Z_SearchRequest *sr = apdu->u.searchRequest;
1831     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1832     Yaz_Z_Databases this_databases;
1833
1834     this_databases.set(sr->num_databaseNames, (const char **)
1835                        sr->databaseNames);
1836
1837     this_query->set_Z_Query(sr->query);
1838
1839     // Check for non-negative piggyback params.
1840     if (*sr->smallSetUpperBound < 0
1841         || *sr->largeSetLowerBound < 0
1842         || *sr->mediumSetPresentNumber < 0)
1843     {
1844         Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1845         // Not a present request.. But can't find better diagnostic
1846         new_apdu->u.searchResponse->records =
1847             create_nonSurrogateDiagnostics(
1848                 odr_encode(),
1849                 YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, 0);
1850         send_to_client(new_apdu);
1851         return 0;
1852     }
1853
1854     char query_str[4096];
1855     this_query->print(query_str, sizeof(query_str)-1);
1856     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1857
1858     if (*m_parent->m_optimize != '0' &&
1859         m_client->m_last_ok && m_client->m_last_query &&
1860         m_client->m_last_query->match(this_query) &&
1861         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1862         m_client->m_last_databases.match(this_databases))
1863     {
1864         delete this_query;
1865         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1866             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1867         {
1868             Z_NamePlusRecordList *npr;
1869             int toget = *sr->mediumSetPresentNumber;
1870             Z_RecordComposition *comp = 0;
1871
1872             if (toget > m_client->m_last_resultCount)
1873                 toget = m_client->m_last_resultCount;
1874
1875             if (sr->mediumSetElementSetNames)
1876             {
1877                 comp = (Z_RecordComposition *)
1878                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1879                 comp->which = Z_RecordComp_simple;
1880                 comp->u.simple = sr->mediumSetElementSetNames;
1881             }
1882
1883             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1884                                           sr->preferredRecordSyntax, comp))
1885             {
1886                 yaz_log(YLOG_LOG, "%sReturned cached records for medium set",
1887                          m_session_str);
1888                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1889                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1890                 new_apdu->u.searchResponse->resultCount =
1891                     &m_client->m_last_resultCount;
1892
1893                 new_apdu->u.searchResponse->numberOfRecordsReturned
1894                     = odr_intdup(odr_encode(), toget);
1895
1896                 new_apdu->u.searchResponse->presentStatus =
1897                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1898                 new_apdu->u.searchResponse->records = (Z_Records*)
1899                     odr_malloc(odr_encode(), sizeof(Z_Records));
1900                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1901                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1902                 new_apdu->u.searchResponse->nextResultSetPosition =
1903                     odr_intdup(odr_encode(), toget+1);
1904                 send_to_client(new_apdu);
1905                 return 0;
1906             }
1907             else
1908             {
1909                 // medium Set
1910                 // send present request (medium size)
1911                 yaz_log(YLOG_LOG, "%sOptimizing search for medium set",
1912                          m_session_str);
1913
1914                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1915                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1916                 pr->referenceId = sr->referenceId;
1917                 pr->resultSetId = sr->resultSetName;
1918                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1919                 *pr->numberOfRecordsRequested = toget;
1920                 pr->recordComposition = comp;
1921                 m_client->m_sr_transform = 1;
1922                 return new_apdu;
1923             }
1924         }
1925         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1926             m_client->m_last_resultCount <= 0)
1927         {
1928             // large set. Return pseudo-search response immediately
1929             yaz_log(YLOG_LOG, "%sOptimizing search for large set",
1930                      m_session_str);
1931             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1932             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1933             new_apdu->u.searchResponse->resultCount =
1934                 &m_client->m_last_resultCount;
1935             send_to_client(new_apdu);
1936             return 0;
1937         }
1938         else
1939         {
1940             Z_NamePlusRecordList *npr;
1941             int toget = m_client->m_last_resultCount;
1942             Z_RecordComposition *comp = 0;
1943             // small set
1944             // send a present request (small set)
1945
1946             if (sr->smallSetElementSetNames)
1947             {
1948                 comp = (Z_RecordComposition *)
1949                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1950                 comp->which = Z_RecordComp_simple;
1951                 comp->u.simple = sr->smallSetElementSetNames;
1952             }
1953
1954             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1955                                           sr->preferredRecordSyntax, comp))
1956             {
1957                 yaz_log(YLOG_LOG, "%sReturned cached records for small set",
1958                          m_session_str);
1959                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1960                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1961                 new_apdu->u.searchResponse->resultCount =
1962                     &m_client->m_last_resultCount;
1963
1964                 new_apdu->u.searchResponse->numberOfRecordsReturned
1965                     = odr_intdup(odr_encode(), toget);
1966
1967                 new_apdu->u.searchResponse->presentStatus =
1968                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1969                 new_apdu->u.searchResponse->records = (Z_Records*)
1970                     odr_malloc(odr_encode(), sizeof(Z_Records));
1971                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1972                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1973                 new_apdu->u.searchResponse->nextResultSetPosition =
1974                     odr_intdup(odr_encode(), toget+1);
1975                 send_to_client(new_apdu);
1976                 return 0;
1977             }
1978             else
1979             {
1980                 yaz_log(YLOG_LOG, "%sOptimizing search for small set",
1981                          m_session_str);
1982                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1983                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1984                 pr->referenceId = sr->referenceId;
1985                 pr->resultSetId = sr->resultSetName;
1986                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1987                 *pr->numberOfRecordsRequested = toget;
1988                 pr->recordComposition = comp;
1989                 m_client->m_sr_transform = 1;
1990                 return new_apdu;
1991             }
1992         }
1993     }
1994     else  // query doesn't match
1995     {
1996         delete m_client->m_last_query;
1997         m_client->m_last_query = this_query;
1998         m_client->m_last_ok = 0;
1999         m_client->m_cache.clear();
2000         m_client->m_resultSetStartPoint = 0;
2001
2002         xfree(m_client->m_last_resultSetId);
2003         m_client->m_last_resultSetId = xstrdup(sr->resultSetName);
2004
2005         m_client->m_last_databases.set(sr->num_databaseNames,
2006                                        (const char **) sr->databaseNames);
2007     }
2008     return apdu;
2009 }
2010
2011
2012 void Yaz_Proxy::inc_request_no()
2013 {
2014     m_request_no++;
2015     char *cp = m_session_str + strlen(m_session_str)-1;
2016     if (*cp == ' ')
2017         cp--;
2018     while (*cp && *cp != ' ')
2019         cp--;
2020     if (*cp)
2021         sprintf(cp+1, "%d ", m_request_no);
2022 }
2023
2024 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
2025 {
2026     inc_request_no();
2027
2028     m_bytes_recv += len;
2029
2030     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2031         yaz_log(YLOG_LOG, "%sReceiving %s from client %d bytes",
2032                  m_session_str, gdu_name(apdu), len);
2033
2034 #if 0
2035     // try to make a _bad_ attribute set ID .. Don't enable this in prod.
2036     if (apdu->which == Z_GDU_Z3950
2037         && apdu->u.z3950->which == Z_APDU_searchRequest)
2038     {
2039         Z_SearchRequest *req = apdu->u.z3950->u.searchRequest;
2040         if (req->query && req->query->which == Z_Query_type_1)
2041         {
2042             Z_RPNQuery *rpnquery = req->query->u.type_1;
2043             if (rpnquery->attributeSetId)
2044             {
2045                 rpnquery->attributeSetId[0] = -2;
2046                 rpnquery->attributeSetId[1] = -1;
2047                 yaz_log(YLOG_WARN, "%sBAD FIXUP TEST", m_session_str);
2048             }
2049         }
2050     }
2051 #endif
2052
2053 #if HAVE_GETTIMEOFDAY
2054     gettimeofday((struct timeval *) m_time_tv, 0);
2055 #endif
2056     m_bw_stat.add_bytes(len);
2057     m_pdu_stat.add_bytes(1);
2058
2059     GDU *gdu = new GDU(apdu);
2060
2061     if (gdu->get() == 0)
2062     {
2063         delete gdu;
2064         yaz_log(YLOG_LOG, "%sUnable to encode package", m_session_str);
2065         m_in_queue.clear();
2066         dec_ref();
2067         return;
2068     }
2069     m_in_queue.enqueue(gdu);
2070     recv_GDU_more(false);
2071 }
2072
2073 void Yaz_Proxy::HTTP_Forwarded(Z_GDU *z_gdu)
2074 {
2075     if (z_gdu->which == Z_GDU_HTTP_Request)
2076     {
2077         Z_HTTP_Request *hreq = z_gdu->u.HTTP_Request;
2078         const char *x_forwarded_for =
2079             z_HTTP_header_lookup(hreq->headers, "X-Forwarded-For");
2080         if (x_forwarded_for)
2081         {
2082             xfree(m_peername);
2083             m_peername = (char*) xmalloc(strlen(x_forwarded_for)+5);
2084             sprintf(m_peername, "tcp:%s", x_forwarded_for);
2085
2086             yaz_log(YLOG_LOG, "%sHTTP Forwarded from %s", m_session_str,
2087                     m_peername);
2088             if (m_log_mask & PROXY_LOG_IP_CLIENT)
2089                 sprintf(m_session_str, "%ld:%d %.80s %d ",
2090                         (long) time(0), m_session_no, m_peername, m_request_no);
2091             else
2092                 sprintf(m_session_str, "%ld:%d %d ",
2093                         (long) time(0), m_session_no, m_request_no);
2094         }
2095     }
2096 }
2097
2098 void Yaz_Proxy::connect_stat(bool &block, int &reduce)
2099 {
2100
2101     m_parent->m_connect.cleanup(false);
2102     m_parent->m_connect.add_connect(m_peername);
2103
2104     int connect_total = m_parent->m_connect.get_total(m_peername);
2105     int max_connect = m_parent->m_max_connect;
2106
2107     if (max_connect && connect_total > max_connect)
2108     {
2109         yaz_log(YLOG_LOG, "%sconnect not accepted total=%d max=%d",
2110                 m_session_str, connect_total, max_connect);
2111         block = true;
2112     }
2113     else
2114         block = false;
2115     yaz_log(YLOG_LOG, "%sconnect accepted total=%d", m_session_str,
2116             connect_total);
2117
2118     int limit_connect = m_parent->m_limit_connect;
2119     if (limit_connect)
2120         reduce = connect_total / limit_connect;
2121     else
2122         reduce = 0;
2123 }
2124
2125 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
2126 {
2127     HTTP_Forwarded(gdu->get());
2128
2129     int reduce = 0;
2130
2131     if (m_request_no == 1)
2132     {
2133         bool block = false;
2134
2135         connect_stat(block, reduce);
2136
2137         if (block)
2138         {
2139             m_timeout_mode = timeout_busy;
2140             timeout(0);
2141             return;
2142         }
2143     }
2144
2145     int bw_total = m_bw_stat.get_total();
2146     int pdu_total = m_pdu_stat.get_total();
2147     int search_total = m_search_stat.get_total();
2148
2149     assert(m_timeout_mode == timeout_busy);
2150     assert(m_timeout_gdu == 0);
2151
2152     if (m_search_max)
2153         reduce += search_total / m_search_max;
2154     if (m_bw_max)
2155         reduce += (bw_total/m_bw_max);
2156     if (m_pdu_max)
2157     {
2158         if (pdu_total > m_pdu_max)
2159         {
2160             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
2161             reduce = (reduce > nreduce) ? reduce : nreduce;
2162         }
2163     }
2164     m_http_version = 0;
2165
2166 #if 0
2167     /* uncomment to force a big reduce */
2168     m_timeout_mode = timeout_reduce;
2169     m_timeout_gdu = gdu;
2170     timeout(3);       // call us reduce seconds later
2171     return;
2172 #endif
2173     if (reduce)
2174     {
2175         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d search=%d limit-bw=%d limit-pdu=%d limit-search=%d",
2176                 m_session_str, reduce, bw_total, pdu_total, search_total,
2177                 m_bw_max, m_pdu_max, m_search_max);
2178
2179         m_timeout_mode = timeout_reduce;
2180         m_timeout_gdu = gdu;
2181         timeout(reduce);       // call us reduce seconds later
2182     }
2183     else
2184         recv_GDU_normal(gdu);
2185 }
2186
2187 void Yaz_Proxy::recv_GDU_more(bool normal)
2188 {
2189     GDU *g;
2190     if (normal && m_timeout_mode == timeout_busy)
2191         m_timeout_mode = timeout_normal;
2192     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
2193     {
2194         m_timeout_mode = timeout_busy;
2195         inc_ref();
2196         recv_GDU_reduce(g);
2197         if (dec_ref())
2198             break;
2199     }
2200 }
2201
2202 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
2203 {
2204     Z_GDU *apdu = 0;
2205     gdu->move_away_gdu(odr_decode(), &apdu);
2206     delete gdu;
2207
2208     if (apdu->which == Z_GDU_Z3950)
2209         handle_incoming_Z_PDU(apdu->u.z3950);
2210     else if (apdu->which == Z_GDU_HTTP_Request)
2211         handle_incoming_HTTP(apdu->u.HTTP_Request);
2212 }
2213
2214 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
2215 {
2216     if (m_max_record_retrieve)
2217     {
2218         if (apdu->which == Z_APDU_presentRequest)
2219         {
2220             Z_PresentRequest *pr = apdu->u.presentRequest;
2221             if (pr->numberOfRecordsRequested &&
2222                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
2223                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
2224         }
2225     }
2226 }
2227
2228 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
2229 {
2230     if (apdu->which == Z_APDU_initRequest)
2231     {
2232         if (m_initRequest_options &&
2233             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
2234             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
2235         {
2236             // There is no negotiation proposal from
2237             // client's side. OK. The proxy negotiation
2238             // in use, only.
2239             Z_InitRequest *initRequest = apdu->u.initRequest;
2240             Z_OtherInformation **otherInfo;
2241             Z_OtherInformationUnit *oi;
2242             get_otherInfoAPDU(apdu, &otherInfo);
2243             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2244             if (oi)
2245             {
2246                 ODR_MASK_SET(initRequest->options,
2247                     Z_Options_negotiationModel);
2248                 oi->which = Z_OtherInfo_externallyDefinedInfo;
2249                 oi->information.externallyDefinedInfo =
2250                 yaz_set_proposal_charneg(odr_encode(),
2251                     (const char**)&m_proxy_negotiation_charset,
2252                     m_proxy_negotiation_charset ? 1:0,
2253                     (const char**)&m_proxy_negotiation_lang,
2254                     m_proxy_negotiation_lang ? 1:0,
2255                     1);
2256             }
2257         }
2258         else if (m_initRequest_options &&
2259                  ODR_MASK_GET(m_initRequest_options,
2260                               Z_Options_negotiationModel) &&
2261                  m_charset_converter->get_target_query_charset())
2262         {
2263             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
2264                     m_session_str,
2265                     m_charset_converter->get_target_query_charset());
2266             Z_InitRequest *initRequest = apdu->u.initRequest;
2267             Z_CharSetandLanguageNegotiation *negotiation =
2268                 yaz_get_charneg_record (initRequest->otherInfo);
2269             if (negotiation &&
2270                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
2271             {
2272                 NMEM nmem = nmem_create();
2273                 char **charsets = 0;
2274                 int num_charsets = 0;
2275                 char **langs = 0;
2276                 int num_langs = 0;
2277                 int selected = 0;
2278                 yaz_get_proposal_charneg (nmem, negotiation,
2279                                           &charsets, &num_charsets,
2280                                           &langs, &num_langs, &selected);
2281                 int i;
2282                 for (i = 0; i<num_charsets; i++)
2283                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
2284                             charsets[i]);
2285                 for (i = 0; i<num_langs; i++)
2286                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
2287                             langs[i]);
2288
2289                 const char *t_charset =
2290                     m_charset_converter->get_target_query_charset();
2291                 // sweep through charsets and pick the first supported
2292                 // conversion
2293                 for (i = 0; i<num_charsets; i++)
2294                 {
2295                     const char *c_charset = charsets[i];
2296                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
2297                         break;
2298                 }
2299                 if (i != num_charsets)
2300                 {
2301                     // got one .. set up ODR for reverse direction
2302                     const char *c_charset = charsets[i];
2303                     odr_set_charset(odr_encode(), c_charset, t_charset);
2304                     m_charset_converter->set_client_query_charset(c_charset);
2305                     m_charset_converter->set_client_charset_selected(selected);
2306                 }
2307                 nmem_destroy(nmem);
2308                 ODR_MASK_CLEAR(m_initRequest_options,
2309                                Z_Options_negotiationModel);
2310                 yaz_del_charneg_record(&initRequest->otherInfo);
2311             }
2312             else
2313             {
2314                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
2315                         m_session_str);
2316             }
2317         }
2318         else if (m_charset_converter->get_target_query_charset() &&
2319             m_proxy_negotiation_default_charset)
2320         {
2321             m_charset_converter->
2322                 set_client_query_charset(m_proxy_negotiation_default_charset);
2323         }
2324     }
2325     else if (apdu->which == Z_APDU_initResponse)
2326     {
2327         Z_InitResponse *initResponse = apdu->u.initResponse;
2328         Z_OtherInformation **otherInfo;
2329         get_otherInfoAPDU(apdu, &otherInfo);
2330
2331         Z_CharSetandLanguageNegotiation *charneg = 0;
2332
2333         if (otherInfo && *otherInfo &&
2334             ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel)
2335             && (charneg = yaz_get_charneg_record(*otherInfo)))
2336         {
2337             char *charset = 0;
2338             char *lang = 0;
2339             int selected = 0;
2340
2341             yaz_get_response_charneg(m_referenceId_mem, charneg,
2342                 &charset, &lang, &selected);
2343
2344             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
2345                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
2346
2347             if (m_initRequest_options &&
2348                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
2349             {
2350                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
2351                     m_session_str);
2352             }
2353             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2354             {
2355                 // negotiation-charset, negotiation-lang
2356                 // elements of config file in use.
2357
2358                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2359                     m_session_str);
2360
2361                 // clear negotiation option.
2362                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2363
2364                 // Delete negotiation (charneg-3) entry.
2365                 yaz_del_charneg_record(otherInfo);
2366             }
2367         }
2368         else
2369         {
2370             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2371             {
2372                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2373                         m_session_str);
2374             }
2375             else if (m_charset_converter->get_client_query_charset())
2376             {
2377                 Z_OtherInformation **otherInfo;
2378                 Z_OtherInformationUnit *oi;
2379                 get_otherInfoAPDU(apdu, &otherInfo);
2380                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2381                 if (oi)
2382                 {
2383                     ODR_MASK_SET(initResponse->options,
2384                                  Z_Options_negotiationModel);
2385                     if (m_initRequest_options)
2386                         ODR_MASK_SET(m_initRequest_options,
2387                                      Z_Options_negotiationModel);
2388
2389                     oi->which = Z_OtherInfo_externallyDefinedInfo;
2390                     oi->information.externallyDefinedInfo =
2391                         yaz_set_response_charneg(
2392                             odr_encode(),
2393                             m_charset_converter->get_client_query_charset(),
2394                             0 /* no lang */,
2395                             m_charset_converter->get_client_charset_selected());
2396                 }
2397             }
2398         }
2399     }
2400 }
2401
2402 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2403                                                      int error,
2404                                                      const char *addinfo)
2405 {
2406     Z_Records *rec = (Z_Records *)
2407         odr_malloc (odr, sizeof(*rec));
2408     Odr_int *err = (Odr_int *)
2409         odr_malloc (odr, sizeof(*err));
2410     Z_DiagRec *drec = (Z_DiagRec *)
2411         odr_malloc (odr, sizeof(*drec));
2412     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2413         odr_malloc (odr, sizeof(*dr));
2414     *err = error;
2415     rec->which = Z_Records_NSD;
2416     rec->u.nonSurrogateDiagnostic = dr;
2417     dr->diagnosticSetId = odr_oiddup(odr, yaz_oid_diagset_bib_1);
2418     dr->condition = err;
2419     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2420     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2421     return rec;
2422 }
2423
2424 Z_ListEntries *Yaz_Proxy::create_nonSurrogateDiagnostics2(ODR odr,
2425                                                      int error,
2426                                                      const char *addinfo)
2427 {
2428     Z_ListEntries *rec = (Z_ListEntries *)
2429         odr_malloc (odr, sizeof(*rec));
2430     Odr_int *err = (Odr_int *)
2431         odr_malloc (odr, sizeof(*err));
2432     Z_DiagRec *drec = (Z_DiagRec *)
2433         odr_malloc (odr, sizeof(*drec));
2434     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2435         odr_malloc (odr, sizeof(*dr));
2436     *err = error;
2437     drec->which = Z_DiagRec_defaultFormat;
2438     drec->u.defaultFormat = dr;
2439     rec->num_entries = 0;
2440     rec->entries = NULL;
2441     rec->num_nonsurrogateDiagnostics = 1;
2442     rec->nonsurrogateDiagnostics =
2443       (Z_DiagRec **)odr_malloc(odr, sizeof(Z_DiagRec *));
2444     rec->nonsurrogateDiagnostics[0] = drec;
2445     dr->diagnosticSetId = odr_oiddup(odr, yaz_oid_diagset_bib_1);
2446     dr->condition = err;
2447     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2448     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2449     return rec;
2450 }
2451
2452 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2453 {
2454     if (apdu->which == Z_APDU_searchRequest &&
2455         apdu->u.searchRequest->query &&
2456         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2457         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2458     {
2459         Z_RPNQuery *rpnquery = 0;
2460         Z_SearchRequest *sr = apdu->u.searchRequest;
2461         char *addinfo = 0;
2462
2463         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2464                 sr->query->u.type_104->u.cql);
2465
2466         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2467                                           &rpnquery, odr_encode(),
2468                                           &addinfo);
2469         if (r == -3)
2470             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2471         else if (r)
2472         {
2473             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2474             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2475
2476             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2477             new_apdu->u.searchResponse->records =
2478                 create_nonSurrogateDiagnostics(odr_encode(),
2479                                                yaz_diag_srw_to_bib1(r),
2480                                                addinfo);
2481             *new_apdu->u.searchResponse->searchStatus = 0;
2482
2483             send_to_client(new_apdu);
2484
2485             return 0;
2486         }
2487         else
2488         {
2489             sr->query->which = Z_Query_type_1;
2490             sr->query->u.type_1 = rpnquery;
2491         }
2492         return apdu;
2493     }
2494     else if (apdu->which == Z_APDU_scanRequest)
2495     {
2496         Z_RPNQuery *rpnquery = 0;
2497         Z_ScanRequest *sr = apdu->u.scanRequest;
2498         char *addinfo = 0;
2499
2500         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2501                 sr->termListAndStartPoint->term->u.characterString);
2502
2503         int r = m_cql2rpn.query_transform(sr->termListAndStartPoint->term->u.characterString,
2504                                           &rpnquery, odr_encode(),
2505                                           &addinfo);
2506         if (r == -3)
2507             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2508         else if (r)
2509         {
2510             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2511             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_scanResponse);
2512
2513             new_apdu->u.scanResponse->referenceId = sr->referenceId;
2514             new_apdu->u.scanResponse->entries =
2515                 create_nonSurrogateDiagnostics2(odr_encode(),
2516                                                yaz_diag_srw_to_bib1(r),
2517                                                addinfo);
2518             *new_apdu->u.scanResponse->scanStatus = Z_Scan_failure;
2519
2520             send_to_client(new_apdu);
2521
2522             return 0;
2523         }
2524         else
2525         {
2526             sr->attributeSet = rpnquery->attributeSetId;
2527             if (rpnquery->RPNStructure->which == Z_RPNStructure_simple)
2528                 sr->termListAndStartPoint = rpnquery->RPNStructure->u.simple->u.attributesPlusTerm;
2529         }
2530         return apdu;
2531     }
2532     return apdu;
2533 }
2534
2535 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2536 {
2537     if (apdu->which == Z_APDU_searchRequest &&
2538         apdu->u.searchRequest->query)
2539     {
2540         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2541             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2542         {
2543             if (m_http_version)
2544                 m_charset_converter->set_client_query_charset("UTF-8");
2545             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2546             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2547         }
2548     }
2549     else if (apdu->which == Z_APDU_scanRequest &&
2550         apdu->u.scanRequest->termListAndStartPoint)
2551     {
2552         if (apdu->u.scanRequest->termListAndStartPoint->term)
2553             if (m_http_version)
2554                 m_charset_converter->set_client_query_charset("UTF-8");
2555             Z_Term *term = apdu->u.scanRequest->termListAndStartPoint->term;
2556             m_charset_converter->convert_term(term, odr_encode());
2557     }
2558     return apdu;
2559 }
2560
2561
2562 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2563 {
2564     if (apdu->which == Z_APDU_searchRequest)
2565     {
2566         Z_SearchRequest *sr = apdu->u.searchRequest;
2567         int err = 0;
2568         char *addinfo = 0;
2569
2570         Yaz_ProxyConfig *cfg = check_reconfigure();
2571         if (cfg)
2572             err = cfg->check_query(odr_encode(), m_default_target,
2573                                    sr->query, &addinfo);
2574         if (err)
2575         {
2576             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2577
2578             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2579             new_apdu->u.searchResponse->records =
2580                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2581             *new_apdu->u.searchResponse->searchStatus = 0;
2582
2583             send_to_client(new_apdu);
2584
2585             return 0;
2586         }
2587     }
2588     else if (apdu->which == Z_APDU_scanRequest)
2589     {
2590         Z_ScanRequest *sr = apdu->u.scanRequest;
2591         int err = 0;
2592         char *addinfo = 0;
2593
2594         Yaz_ProxyConfig *cfg = check_reconfigure();
2595 // Something like this needs to be implemented later:
2596 /*
2597         if (cfg)
2598             err = cfg->check_type_1_attributes(odr_encode(), m_default_target,
2599                                    sr->termListAndStartPoint->attributes, &addinfo);
2600 */
2601         if (err)
2602         {
2603             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_scanResponse);
2604
2605             new_apdu->u.scanResponse->referenceId = sr->referenceId;
2606             new_apdu->u.scanResponse->entries =
2607                 create_nonSurrogateDiagnostics2(odr_encode(), err, addinfo);
2608             *new_apdu->u.scanResponse->scanStatus = Z_Scan_failure;
2609
2610             send_to_client(new_apdu);
2611
2612             return 0;
2613         }
2614     }
2615
2616     return apdu;
2617 }
2618
2619 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2620 {
2621     if (apdu->which != Z_APDU_initRequest)
2622         return 1;  // pass if no init request
2623     Z_InitRequest *req = apdu->u.initRequest;
2624
2625     Yaz_ProxyConfig *cfg = check_reconfigure();
2626     if (!cfg)
2627         return 1;  // pass if no config
2628
2629     int ret;
2630     if (req->idAuthentication == 0)
2631     {
2632         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2633                                          m_peername);
2634     }
2635     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2636     {
2637         ret = cfg->client_authentication(
2638             m_default_target,
2639             req->idAuthentication->u.idPass->userId,
2640             req->idAuthentication->u.idPass->groupId,
2641             req->idAuthentication->u.idPass->password,
2642             m_peername);
2643     }
2644     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2645     {
2646         char user[64], pass[64];
2647         *user = '\0';
2648         *pass = '\0';
2649         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2650         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2651                                          m_peername);
2652     }
2653     else
2654         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2655                                          m_peername);
2656     return ret;
2657 }
2658
2659 int Yaz_Proxy::handle_global_authentication(Z_APDU *apdu)
2660 {
2661     if (apdu->which != Z_APDU_initRequest)
2662         return 1;  // pass if no init request
2663     Z_InitRequest *req = apdu->u.initRequest;
2664
2665     Yaz_ProxyConfig *cfg = check_reconfigure();
2666     if (!cfg)
2667         return 1;  // pass if no config
2668
2669     int ret;
2670     if (req->idAuthentication == 0)
2671     {
2672         ret = cfg->global_client_authentication(0, 0, 0,
2673                                                 m_peername);
2674     }
2675     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2676     {
2677         ret = cfg->global_client_authentication(
2678             req->idAuthentication->u.idPass->userId,
2679             req->idAuthentication->u.idPass->groupId,
2680             req->idAuthentication->u.idPass->password,
2681             m_peername);
2682     }
2683     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2684     {
2685         char user[64], pass[64];
2686         *user = '\0';
2687         *pass = '\0';
2688         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2689         ret = cfg->global_client_authentication(user, 0, pass,
2690                                                 m_peername);
2691     }
2692     else
2693         ret = cfg->global_client_authentication(0, 0, 0, m_peername);
2694     return ret;
2695 }
2696
2697 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2698 {
2699     m_marcxml_mode = none;
2700     if (apdu->which == Z_APDU_searchRequest)
2701     {
2702         Z_SearchRequest *sr = apdu->u.searchRequest;
2703         int err = 0;
2704         char *addinfo = 0;
2705         Yaz_ProxyConfig *cfg = check_reconfigure();
2706
2707         Z_RecordComposition rc_temp, *rc = 0;
2708         if (sr->smallSetElementSetNames)
2709         {
2710             rc_temp.which = Z_RecordComp_simple;
2711             rc_temp.u.simple = sr->smallSetElementSetNames;
2712             rc = &rc_temp;
2713         }
2714
2715         if (sr->preferredRecordSyntax)
2716             oid_oidcpy(m_frontend_type, sr->preferredRecordSyntax);
2717         else
2718             m_frontend_type[0] = -1;
2719
2720         char *stylesheet_name = 0;
2721         if (cfg)
2722             err = cfg->check_syntax(odr_encode(),
2723                                     m_default_target,
2724                                     sr->preferredRecordSyntax, rc,
2725                                     &addinfo, &stylesheet_name, &m_schema,
2726                                     &m_backend_type, &m_backend_charset,
2727                                     &m_usemarcon_ini_stage1,
2728                                     &m_usemarcon_ini_stage2,
2729                                     &m_backend_elementset);
2730         if (stylesheet_name)
2731         {
2732             m_parent->low_socket_close();
2733
2734 #if YAZ_HAVE_XSLT
2735             if (m_stylesheet_xsp)
2736                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2737             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2738                                                        stylesheet_name);
2739 #endif
2740             m_stylesheet_offset = 0;
2741             xfree(stylesheet_name);
2742
2743             m_parent->low_socket_open();
2744         }
2745         if (err == -1)
2746         {
2747             sr->smallSetElementSetNames = 0;
2748             sr->mediumSetElementSetNames = 0;
2749             m_marcxml_mode = marcxml;
2750             sr->preferredRecordSyntax =
2751                 yaz_string_to_oid_odr(
2752                     yaz_oid_std(), CLASS_RECSYN,
2753                     m_backend_type ? m_backend_type : "usmarc",
2754                     odr_encode());
2755         }
2756         else if (err)
2757         {
2758             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2759
2760             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2761             new_apdu->u.searchResponse->records =
2762                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2763             *new_apdu->u.searchResponse->searchStatus = 0;
2764
2765             send_to_client(new_apdu);
2766
2767             return 0;
2768         }
2769         else if (m_backend_type)
2770         {
2771             sr->preferredRecordSyntax =
2772                 yaz_string_to_oid_odr(yaz_oid_std(), CLASS_RECSYN,
2773                                       m_backend_type, odr_encode());
2774         }
2775         if (m_backend_elementset)
2776         {
2777             Z_ElementSetNames *esn = mk_esn_from_schema(odr_encode(),
2778                                                         m_backend_elementset);
2779             sr->smallSetElementSetNames = esn;
2780             sr->mediumSetElementSetNames = esn;
2781         }
2782     }
2783     else if (apdu->which == Z_APDU_presentRequest)
2784     {
2785         Z_PresentRequest *pr = apdu->u.presentRequest;
2786         int err = 0;
2787         char *addinfo = 0;
2788         Yaz_ProxyConfig *cfg = check_reconfigure();
2789
2790         if (pr->preferredRecordSyntax)
2791             oid_oidcpy(m_frontend_type, pr->preferredRecordSyntax);
2792         else
2793             m_frontend_type[0] = -1;
2794
2795         char *stylesheet_name = 0;
2796         if (cfg)
2797             err = cfg->check_syntax(odr_encode(), m_default_target,
2798                                     pr->preferredRecordSyntax,
2799                                     pr->recordComposition,
2800                                     &addinfo, &stylesheet_name, &m_schema,
2801                                     &m_backend_type, &m_backend_charset,
2802                                     &m_usemarcon_ini_stage1,
2803                                     &m_usemarcon_ini_stage2,
2804                                     &m_backend_elementset);
2805         if (stylesheet_name)
2806         {
2807             m_parent->low_socket_close();
2808
2809 #if YAZ_HAVE_XSLT
2810             if (m_stylesheet_xsp)
2811                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2812             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2813                                                        stylesheet_name);
2814 #endif
2815             m_stylesheet_offset = 0;
2816             xfree(stylesheet_name);
2817
2818             m_parent->low_socket_open();
2819         }
2820         if (err == -1)
2821         {
2822             pr->recordComposition = 0;
2823             m_marcxml_mode = marcxml;
2824
2825             pr->preferredRecordSyntax =
2826                 yaz_string_to_oid_odr(
2827                     yaz_oid_std(), CLASS_RECSYN,
2828                     m_backend_type ? m_backend_type : "usmarc",
2829                     odr_encode());
2830         }
2831         else if (err)
2832         {
2833             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2834
2835             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2836             new_apdu->u.presentResponse->records =
2837                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2838             *new_apdu->u.presentResponse->presentStatus =
2839                 Z_PresentStatus_failure;
2840
2841             send_to_client(new_apdu);
2842
2843             return 0;
2844         }
2845         else if (m_backend_type)
2846         {
2847             pr->preferredRecordSyntax =
2848                 yaz_string_to_oid_odr(yaz_oid_std(),
2849                                       CLASS_RECSYN, m_backend_type,
2850                                       odr_encode());
2851         }
2852         if (m_backend_elementset)
2853         {
2854             Z_ElementSetNames *esn = mk_esn_from_schema(odr_encode(),
2855                                                         m_backend_elementset);
2856             Z_RecordComposition *comp = (Z_RecordComposition *)
2857                 odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
2858             comp->which = Z_RecordComp_simple;
2859             comp->u.simple = esn;
2860             pr->recordComposition = comp;
2861         }
2862     }
2863     return apdu;
2864 }
2865
2866 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2867 {
2868     if (!schema)
2869         return 0;
2870     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2871         odr_malloc(o, sizeof(Z_ElementSetNames));
2872     esn->which = Z_ElementSetNames_generic;
2873     esn->u.generic = odr_strdup(o, schema);
2874     return esn;
2875 }
2876
2877 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2878 {
2879     const char *t = 0;
2880     Yaz_ProxyConfig *cfg = check_reconfigure();
2881     if (cfg)
2882         t = cfg->get_explain_name(db, backend_db);
2883
2884     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2885     {
2886         releaseClient();
2887     }
2888
2889     if (t)
2890     {
2891         xfree(m_default_target);
2892         m_default_target = xstrdup(t);
2893     }
2894 }
2895
2896 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2897 {
2898     struct stat sbuf;
2899     if (strcmp(hreq->method, "GET"))
2900         return 0;
2901     if (hreq->path[0] != '/')
2902         return 0;
2903     const char *cp = hreq->path;
2904     while (*cp)
2905     {
2906         if (*cp == '/' && strchr("/.", cp[1]))
2907             return 0;
2908         cp++;
2909     }
2910
2911     Yaz_ProxyConfig *cfg = check_reconfigure();
2912
2913     if (!cfg->get_file_access_info(hreq->path+1))
2914         return 0;
2915
2916     const char *fname = hreq->path+1;
2917     if (stat(fname, &sbuf))
2918     {
2919         yaz_log(YLOG_LOG|YLOG_ERRNO, "%sstat failed for %s", m_session_str,
2920                 fname);
2921         return 0;
2922     }
2923     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2924     {
2925         yaz_log(YLOG_LOG, "%sNot a regular file %s", m_session_str, fname);
2926         return 0;
2927     }
2928     if (sbuf.st_size > (off_t) 1000000)
2929     {
2930         yaz_log(YLOG_WARN, "%sFile %s too large for transfer", m_session_str,
2931                 fname);
2932         return 0;
2933     }
2934
2935     ODR o = odr_encode();
2936
2937     const char *ctype = cfg->check_mime_type(fname);
2938     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2939     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2940     if (m_http_version)
2941         hres->version = odr_strdup(o, m_http_version);
2942     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2943     if (m_http_keepalive)
2944         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2945     else
2946         timeout(0);
2947
2948     hres->content_len = sbuf.st_size;
2949     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2950     FILE *f = fopen(fname, "rb");
2951     if (f)
2952     {
2953         fread(hres->content_buf, 1, hres->content_len, f);
2954         fclose(f);
2955     }
2956     else
2957     {
2958         return 0;
2959     }
2960     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2961     {
2962         yaz_log(YLOG_LOG, "%sSending file %s to client", m_session_str,
2963                  fname);
2964     }
2965     int len;
2966     send_GDU(gdu, &len);
2967     recv_GDU_more(true);
2968     return 1;
2969 }
2970
2971 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2972 {
2973     if (m_s2z_odr_init)
2974     {
2975         odr_destroy(m_s2z_odr_init);
2976         m_s2z_odr_init = 0;
2977     }
2978     if (m_s2z_odr_search)
2979     {
2980         odr_destroy(m_s2z_odr_search);
2981         m_s2z_odr_search = 0;
2982     }
2983     if (m_s2z_odr_scan)
2984     {
2985         odr_destroy(m_s2z_odr_scan);
2986         m_s2z_odr_scan = 0;
2987     }
2988
2989     m_http_keepalive = 0;
2990     m_http_version = 0;
2991     if (!strcmp(hreq->version, "1.0"))
2992     {
2993         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2994         if (v && !strcmp(v, "Keep-Alive"))
2995             m_http_keepalive = 1;
2996         else
2997             m_http_keepalive = 0;
2998         m_http_version = "1.0";
2999     }
3000     else
3001     {
3002         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
3003         if (v && !strcmp(v, "close"))
3004             m_http_keepalive = 0;
3005         else
3006             m_http_keepalive = 1;
3007         m_http_version = "1.1";
3008     }
3009
3010     const char *a = z_HTTP_header_lookup(hreq->headers, "Authorization");
3011     char authorization_str[255];
3012     *authorization_str = '\0';
3013     if (a && strncasecmp(a, "Basic ", 6) == 0)
3014         base64_decode(a + 6, authorization_str, 254);
3015
3016     Z_SRW_PDU *srw_pdu = 0;
3017     Z_SOAP *soap_package = 0;
3018     char *charset = 0;
3019     Z_SRW_diagnostic *diagnostic = 0;
3020     int num_diagnostic = 0;
3021
3022     yaz_log(YLOG_LOG, "%s%s %s", m_session_str, hreq->method, hreq->path);
3023
3024     if (file_access(hreq))
3025     {
3026         return;
3027     }
3028     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
3029                             &charset) == 0
3030              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
3031                                &charset, &diagnostic, &num_diagnostic) == 0)
3032     {
3033         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
3034         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
3035         m_s2z_odr_scan = odr_createmem(ODR_ENCODE);
3036         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
3037         m_s2z_init_apdu = 0;
3038         m_s2z_search_apdu = 0;
3039         m_s2z_present_apdu = 0;
3040         m_s2z_scan_apdu = 0;
3041
3042         m_s2z_stylesheet = 0;
3043
3044         Z_IdAuthentication *auth = NULL;
3045         if (srw_pdu->username && !srw_pdu->password)
3046         {
3047             yaz_log(YLOG_LOG, "username: %s\n", srw_pdu->username);
3048             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
3049             auth->which = Z_IdAuthentication_open;
3050             auth->u.open = odr_strdup(m_s2z_odr_init, srw_pdu->username);
3051         }
3052         else if (srw_pdu->username && srw_pdu->password)
3053         {
3054             yaz_log(YLOG_LOG, "username/password: %s/%s\n",
3055                     srw_pdu->username, srw_pdu->password);
3056             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
3057             auth->which = Z_IdAuthentication_idPass;
3058             auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
3059             auth->u.idPass->groupId = NULL;
3060             auth->u.idPass->password = odr_strdup(m_s2z_odr_init, srw_pdu->password);
3061             auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, srw_pdu->username);
3062         }
3063         else
3064         {
3065             if (*authorization_str)
3066             {
3067                 yaz_log(YLOG_LOG, "authorization_str present: %s\n", authorization_str);
3068                 auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
3069                 auth->which = Z_IdAuthentication_idPass;
3070                 auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
3071                 auth->u.idPass->groupId = NULL;
3072                 char *p = strchr(authorization_str, ':');
3073                 if (p)
3074                 {
3075                     *p = '\0';
3076                     p++;
3077                     auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
3078                 }
3079                 auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
3080             }
3081         }
3082
3083         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
3084         {
3085
3086             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
3087
3088             const char *backend_db = srw_req->database;
3089             srw_get_client(srw_req->database, &backend_db);
3090
3091             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
3092             // recordXPath unsupported.
3093             if (srw_req->recordXPath)
3094             {
3095                 yaz_add_srw_diagnostic(odr_decode(),
3096                                        &diagnostic, &num_diagnostic,
3097                                        72, 0);
3098             }
3099             // sort unsupported
3100             if (srw_req->sort_type != Z_SRW_sort_type_none)
3101             {
3102                 yaz_add_srw_diagnostic(odr_decode(),
3103                                        &diagnostic, &num_diagnostic,
3104                                        80, 0);
3105             }
3106             // save stylesheet
3107             if (srw_req->stylesheet)
3108                 m_s2z_stylesheet =
3109                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
3110
3111             // set packing for response records ..
3112             if (srw_req->recordPacking &&
3113                 !strcmp(srw_req->recordPacking, "xml"))
3114                 m_s2z_packing = Z_SRW_recordPacking_XML;
3115             else
3116                 m_s2z_packing = Z_SRW_recordPacking_string;
3117
3118             if (num_diagnostic)
3119             {
3120                 Z_SRW_PDU *srw_pdu =
3121                     yaz_srw_get(odr_encode(),
3122                                 Z_SRW_searchRetrieve_response);
3123                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
3124
3125                 srw_res->diagnostics = diagnostic;
3126                 srw_res->num_diagnostics = num_diagnostic;
3127                 send_srw_response(srw_pdu);
3128                 return;
3129             }
3130
3131             // prepare search PDU
3132             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
3133                                           Z_APDU_searchRequest);
3134             Z_SearchRequest *z_searchRequest =
3135                 m_s2z_search_apdu->u.searchRequest;
3136
3137             z_searchRequest->num_databaseNames = 1;
3138             z_searchRequest->databaseNames = (char**)
3139                 odr_malloc(m_s2z_odr_search, sizeof(char *));
3140             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
3141                                                            backend_db);
3142
3143             // query transformation
3144             Z_Query *query = (Z_Query *)
3145                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
3146             z_searchRequest->query = query;
3147
3148             if (srw_req->query_type == Z_SRW_query_type_cql)
3149             {
3150                 Z_External *ext = (Z_External *)
3151                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
3152                 ext->direct_reference =
3153                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
3154                 ext->indirect_reference = 0;
3155                 ext->descriptor = 0;
3156                 ext->which = Z_External_CQL;
3157                 ext->u.cql = srw_req->query.cql;
3158
3159                 query->which = Z_Query_type_104;
3160                 query->u.type_104 =  ext;
3161             }
3162             else if (srw_req->query_type == Z_SRW_query_type_pqf)
3163             {
3164                 Z_RPNQuery *RPNquery;
3165                 YAZ_PQF_Parser pqf_parser;
3166
3167                 pqf_parser = yaz_pqf_create ();
3168
3169                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
3170                                           srw_req->query.pqf);
3171                 if (!RPNquery)
3172                 {
3173                     const char *pqf_msg;
3174                     size_t off;
3175                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
3176                     int ioff = off;
3177                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
3178                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
3179
3180                     send_to_srw_client_error(10, 0);
3181                     return;
3182                 }
3183                 query->which = Z_Query_type_1;
3184                 query->u.type_1 =  RPNquery;
3185
3186                 yaz_pqf_destroy (pqf_parser);
3187             }
3188             else
3189             {
3190                 send_to_srw_client_error(7, "query");
3191                 return;
3192             }
3193
3194             // present
3195             m_s2z_present_apdu = 0;
3196             int max = 0;
3197             if (srw_req->maximumRecords)
3198                 max = *srw_req->maximumRecords;
3199             int start = 1;
3200             if (srw_req->startRecord)
3201                 start = *srw_req->startRecord;
3202             if (max > 0)
3203             {
3204                 // Some backend, such as Voyager doesn't honor piggyback
3205                 // So we use present always (0 &&).
3206                 if (0 && start <= 1)  // Z39.50 piggyback
3207                 {
3208                     *z_searchRequest->smallSetUpperBound = max;
3209                     *z_searchRequest->mediumSetPresentNumber = max;
3210                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
3211
3212                     z_searchRequest->preferredRecordSyntax =
3213                         odr_oiddup(m_s2z_odr_search, yaz_oid_recsyn_xml);
3214
3215                     if (srw_req->recordSchema)
3216                     {
3217                         z_searchRequest->smallSetElementSetNames =
3218                             z_searchRequest->mediumSetElementSetNames =
3219                             mk_esn_from_schema(m_s2z_odr_search,
3220                                                srw_req->recordSchema);
3221                     }
3222                 }
3223                 else   // Z39.50 present
3224                 {
3225                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
3226                                                    Z_APDU_presentRequest);
3227                     Z_PresentRequest *z_presentRequest =
3228                         m_s2z_present_apdu->u.presentRequest;
3229                     *z_presentRequest->resultSetStartPoint = start;
3230                     *z_presentRequest->numberOfRecordsRequested = max;
3231
3232                     z_presentRequest->preferredRecordSyntax =
3233                         odr_oiddup(m_s2z_odr_search, yaz_oid_recsyn_xml);
3234                     if (srw_req->recordSchema)
3235                     {
3236                         z_presentRequest->recordComposition =
3237                             (Z_RecordComposition *)
3238                             odr_malloc(m_s2z_odr_search,
3239                                        sizeof(Z_RecordComposition));
3240                         z_presentRequest->recordComposition->which =
3241                             Z_RecordComp_simple;
3242                         z_presentRequest->recordComposition->u.simple =
3243                             mk_esn_from_schema(m_s2z_odr_search,
3244                                                srw_req->recordSchema);
3245                     }
3246                 }
3247             }
3248             if (!m_client)
3249             {
3250                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
3251                                             Z_APDU_initRequest);
3252
3253                 ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan);
3254                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
3255
3256                 // prevent m_initRequest_apdu memory from being grabbed
3257                 // in Yaz_Proxy::handle_incoming_Z_PDU
3258                 m_initRequest_apdu = m_s2z_init_apdu;
3259                 handle_incoming_Z_PDU(m_s2z_init_apdu);
3260                 return;
3261             }
3262             else
3263             {
3264                 handle_incoming_Z_PDU(m_s2z_search_apdu);
3265                 return;
3266             }
3267         }
3268         else if (srw_pdu->which == Z_SRW_explain_request)
3269         {
3270             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
3271
3272             const char *backend_db = srw_req->database;
3273             srw_get_client(srw_req->database, &backend_db);
3274
3275             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
3276
3277             // save stylesheet
3278             if (srw_req->stylesheet)
3279                 m_s2z_stylesheet =
3280                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
3281
3282             if (srw_req->recordPacking &&
3283                 !strcmp(srw_req->recordPacking, "xml"))
3284                 m_s2z_packing = Z_SRW_recordPacking_XML;
3285             else
3286                 m_s2z_packing = Z_SRW_recordPacking_string;
3287
3288             if (num_diagnostic)
3289             {
3290                 send_srw_explain_response(diagnostic, num_diagnostic);
3291                 return;
3292             }
3293
3294             if (!m_client)
3295             {
3296                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
3297                                             Z_APDU_initRequest);
3298
3299                 ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan);
3300                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
3301
3302                 // prevent m_initRequest_apdu memory from being grabbed
3303                 // in Yaz_Proxy::handle_incoming_Z_PDU
3304                 m_initRequest_apdu = m_s2z_init_apdu;
3305                 handle_incoming_Z_PDU(m_s2z_init_apdu);
3306             }
3307             else
3308                 send_srw_explain_response(0, 0);
3309             return;
3310         }
3311         else if (srw_pdu->which == Z_SRW_scan_request)
3312         {
3313             Z_SRW_scanRequest *srw_req = srw_pdu->u.scan_request;
3314
3315             const char *backend_db = srw_req->database;
3316             srw_get_client(srw_req->database, &backend_db);
3317
3318             m_s2z_database = odr_strdup(m_s2z_odr_init,
3319                                         srw_req->database);
3320             // save stylesheet
3321             if (srw_req->stylesheet)
3322                 m_s2z_stylesheet =
3323                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
3324
3325             // prepare scan PDU
3326             m_s2z_scan_apdu = zget_APDU(m_s2z_odr_scan,
3327                                           Z_APDU_scanRequest);
3328             Z_ScanRequest *z_scanRequest =
3329                 m_s2z_scan_apdu->u.scanRequest;
3330
3331             z_scanRequest->num_databaseNames = 1;
3332             z_scanRequest->databaseNames = (char**)
3333                 odr_malloc(m_s2z_odr_scan, sizeof(char *));
3334             z_scanRequest->databaseNames[0] = odr_strdup(m_s2z_odr_scan,
3335                                                            backend_db);
3336
3337              // query transformation
3338             if (srw_req->query_type == Z_SRW_query_type_cql)
3339             {
3340                 z_scanRequest->termListAndStartPoint =
3341                     (Z_AttributesPlusTerm *)odr_malloc(m_s2z_odr_scan, sizeof(Z_AttributesPlusTerm));
3342                 z_scanRequest->termListAndStartPoint->attributes = NULL;
3343                 z_scanRequest->termListAndStartPoint->term =
3344                    (Z_Term *)odr_malloc(m_s2z_odr_scan, sizeof(Z_Term));
3345                 z_scanRequest->termListAndStartPoint->term->which =
3346                   Z_Term_characterString;
3347                 z_scanRequest->termListAndStartPoint->term->u.characterString =
3348                   odr_strdup(m_s2z_odr_scan, srw_req->scanClause.cql);
3349             }
3350
3351             if (srw_req->responsePosition)
3352                 z_scanRequest->preferredPositionInResponse =
3353                     odr_intdup(m_s2z_odr_scan, *srw_req->responsePosition);
3354             if (srw_req->maximumTerms)
3355                 *z_scanRequest->numberOfTermsRequested = *srw_req->maximumTerms;
3356
3357             if (!m_client)
3358             {
3359                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
3360                                             Z_APDU_initRequest);
3361
3362                 ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan);
3363                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
3364
3365                 // prevent m_initRequest_apdu memory from being grabbed
3366                 // in Yaz_Proxy::handle_incoming_Z_PDU
3367                 m_initRequest_apdu = m_s2z_init_apdu;
3368                 handle_incoming_Z_PDU(m_s2z_init_apdu);
3369                 return;
3370             }
3371             else
3372             {
3373                 handle_incoming_Z_PDU(m_s2z_scan_apdu);
3374                 return;
3375             }
3376         }
3377         else
3378         {
3379             m_s2z_database = 0;
3380
3381             send_to_srw_client_error(4, 0);
3382         }
3383     }
3384     send_http_response(400);
3385 }
3386
3387 void Yaz_Proxy::handle_init(Z_APDU *apdu)
3388 {
3389
3390     Z_OtherInformation **oi;
3391     get_otherInfoAPDU(apdu, &oi);
3392
3393     if (apdu->u.initRequest->implementationId)
3394         yaz_log(YLOG_LOG, "%simplementationId: %s",
3395                 m_session_str, apdu->u.initRequest->implementationId);
3396     if (apdu->u.initRequest->implementationName)
3397         yaz_log(YLOG_LOG, "%simplementationName: %s",
3398                 m_session_str, apdu->u.initRequest->implementationName);
3399     if (apdu->u.initRequest->implementationVersion)
3400         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
3401                 m_session_str, apdu->u.initRequest->implementationVersion);
3402     if (m_initRequest_apdu == 0)
3403     {
3404         if (m_initRequest_mem)
3405             nmem_destroy(m_initRequest_mem);
3406
3407         m_initRequest_apdu = apdu;
3408         m_initRequest_mem = odr_extract_mem(odr_decode());
3409
3410         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
3411             preferredMessageSize;
3412         *apdu->u.initRequest->preferredMessageSize = 64*1024*1024;
3413         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
3414             maximumRecordSize;
3415         *apdu->u.initRequest->maximumRecordSize = 64*1024*1024;
3416
3417         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
3418             yaz_get_charneg_record(*oi);
3419
3420         // Save proposal charsets and langs.
3421         if (ODR_MASK_GET(apdu->u.initRequest->options,
3422                          Z_Options_negotiationModel)
3423             && charSetandLangRecord)
3424         {
3425
3426             yaz_get_proposal_charneg(m_referenceId_mem,
3427                                      charSetandLangRecord,
3428                                      &m_initRequest_oi_negotiation_charsets,
3429                                      &m_initRequest_oi_negotiation_num_charsets,
3430                                      &m_initRequest_oi_negotiation_langs,
3431                                      &m_initRequest_oi_negotiation_num_langs,
3432                                      &m_initRequest_oi_negotiation_selected);
3433
3434             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
3435             {
3436                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
3437                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
3438                         m_initRequest_oi_negotiation_charsets[i]:"none");
3439             }
3440             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
3441             {
3442                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
3443                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
3444                         m_initRequest_oi_negotiation_langs[i]:"none");
3445             }
3446             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
3447                     m_session_str, m_initRequest_oi_negotiation_selected);
3448         }
3449         // save init options for the response..
3450         m_initRequest_options = apdu->u.initRequest->options;
3451
3452         apdu->u.initRequest->options =
3453             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
3454                                        sizeof(Odr_bitmask));
3455         ODR_MASK_ZERO(apdu->u.initRequest->options);
3456         int i;
3457         for (i = 0; i<= 24; i++)
3458             ODR_MASK_SET(apdu->u.initRequest->options, i);
3459         // check negotiation option
3460         if (!ODR_MASK_GET(m_initRequest_options,
3461                           Z_Options_negotiationModel))
3462         {
3463             ODR_MASK_CLEAR(apdu->u.initRequest->options,
3464                            Z_Options_negotiationModel);
3465         }
3466         ODR_MASK_CLEAR(apdu->u.initRequest->options,
3467                        Z_Options_concurrentOperations);
3468         // make new version
3469         m_initRequest_version = apdu->u.initRequest->protocolVersion;
3470         apdu->u.initRequest->protocolVersion =
3471             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
3472                                        sizeof(Odr_bitmask));
3473         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
3474
3475         for (i = 0; i<= 8; i++)
3476             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
3477     }
3478     handle_charset_lang_negotiation(apdu);
3479     if (m_client->m_init_flag)
3480     {
3481         if (handle_init_response_for_invalid_session(apdu))
3482             return;
3483         if (m_client->m_initResponse)
3484         {
3485             Z_APDU *apdu2 = m_client->m_initResponse;
3486             apdu2->u.initResponse->otherInfo = 0;
3487             if (m_client->m_cookie && *m_client->m_cookie)
3488                 set_otherInformationString(apdu2, yaz_oid_userinfo_cookie,
3489                                            1, m_client->m_cookie);
3490             apdu2->u.initResponse->referenceId =
3491                 apdu->u.initRequest->referenceId;
3492             apdu2->u.initResponse->options = m_client->m_initResponse_options;
3493             apdu2->u.initResponse->protocolVersion =
3494                 m_client->m_initResponse_version;
3495
3496             handle_charset_lang_negotiation(apdu2);
3497
3498             if (m_timeout_mode == timeout_busy)
3499                 m_timeout_mode = timeout_normal;
3500             send_to_client(apdu2);
3501             return;
3502         }
3503     }
3504     m_client->m_init_flag = 1;
3505
3506     if (m_num_msg_threads && m_my_thread)
3507     {
3508         Auth_Msg *m = new Auth_Msg;
3509         m->m_proxy = this;
3510         z_APDU(odr_encode(), &apdu, 0, "encode");
3511         char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
3512         m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
3513         memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
3514         odr_reset(odr_encode());
3515
3516         inc_ref();
3517         m_my_thread->put(m);
3518     }
3519     else
3520     {
3521         int ret = handle_authentication(apdu);
3522         result_authentication(apdu, ret);
3523     }
3524 }
3525
3526 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
3527 {
3528     Z_ReferenceId **refid = get_referenceIdP(apdu);
3529     nmem_reset(m_referenceId_mem);
3530     if (refid && *refid)
3531     {
3532         m_referenceId = (Z_ReferenceId *)
3533             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
3534         m_referenceId->len = m_referenceId->size = (*refid)->len;
3535         m_referenceId->buf = (unsigned char *)
3536             nmem_malloc(m_referenceId_mem, (*refid)->len);
3537         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
3538     }
3539     else
3540         m_referenceId = 0;
3541
3542     if (!m_client && m_flag_invalid_session)
3543     {
3544         // Got request for a session that is invalid..
3545         m_apdu_invalid_session = apdu; // save package
3546         m_mem_invalid_session = odr_extract_mem(odr_decode());
3547         apdu = m_initRequest_apdu;     // but throw an init to the target
3548     }
3549
3550     if (apdu->which == Z_APDU_searchRequest)
3551         m_search_stat.add_bytes(1);
3552
3553     // Handle global authentication
3554     if (!handle_global_authentication(apdu))
3555     {
3556         if (m_http_version)
3557         {   // HTTP. Send unauthorized
3558             send_http_response(401);
3559             return;
3560         }
3561         else
3562         {
3563             // Z39.50 just shutdown
3564             timeout(0);
3565             return;
3566         }
3567         return;
3568     }
3569
3570     // Determine our client.
3571     Z_OtherInformation **oi;
3572     get_otherInfoAPDU(apdu, &oi);
3573     int http_code = 404;
3574     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi), &http_code);
3575     if (!m_client)
3576     {
3577         if (m_http_version)
3578         {   // HTTP. Send not found
3579             send_http_response(http_code);
3580             return;
3581         }
3582         else
3583         {
3584             // Z39.50 just shutdown
3585             timeout(0);
3586             return;
3587         }
3588     }
3589
3590     m_client->m_server = this;
3591
3592     if (apdu->which == Z_APDU_initRequest)
3593         handle_init(apdu);
3594     else
3595         handle_incoming_Z_PDU_2(apdu);
3596 }
3597
3598 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
3599 {
3600     handle_max_record_retrieve(apdu);
3601
3602     if (apdu)
3603         apdu = handle_syntax_validation(apdu);
3604
3605     if (apdu)
3606         apdu = handle_query_transformation(apdu);
3607
3608     if (apdu)
3609         apdu = handle_target_charset_conversion(apdu);
3610
3611     if (apdu)
3612         apdu = handle_query_validation(apdu);
3613
3614     if (apdu)
3615         apdu = result_set_optimize(apdu);
3616
3617     if (!apdu)
3618     {
3619         m_client->timeout(m_target_idletime);  // mark it active even
3620         recv_GDU_more(true);
3621         // though we didn't use it
3622         return;
3623     }
3624
3625     // delete other info construct completely if 0 elements
3626     Z_OtherInformation **oi;
3627     get_otherInfoAPDU(apdu, &oi);
3628     if (oi && *oi && (*oi)->num_elements == 0)
3629         *oi = 0;
3630
3631     if (apdu->which == Z_APDU_presentRequest &&
3632         m_client->m_resultSetStartPoint == 0)
3633     {
3634         Z_PresentRequest *pr = apdu->u.presentRequest;
3635         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3636         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3637     } else {
3638         m_client->m_resultSetStartPoint = 0;
3639     }
3640     if (m_client->send_to_target(apdu) < 0)
3641     {
3642         m_client->shutdown();
3643     }
3644     else
3645         m_client->m_waiting = 1;
3646 }
3647
3648 void Yaz_Proxy::connectNotify()
3649 {
3650 }
3651
3652 void Yaz_Proxy::releaseClient()
3653 {
3654     xfree(m_proxyTarget);
3655     m_proxyTarget = 0;
3656     m_flag_invalid_session = 0;
3657     // only keep if keep_alive flag is set...
3658     if (m_client &&
3659         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3660         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3661         m_client->m_waiting == 0)
3662     {
3663         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3664                  m_session_str,
3665                  m_client->get_hostname());
3666         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3667                 m_session_str, m_client->m_pdu_recv,
3668                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3669                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3670         assert (m_client->m_waiting != 2);
3671         // Tell client (if any) that no server connection is there..
3672         m_client->m_server = 0;
3673         m_client = 0;
3674     }
3675     else if (m_client)
3676     {
3677         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) close %s",
3678                  m_session_str,
3679                  m_client->get_hostname());
3680         assert (m_client->m_waiting != 2);
3681         delete m_client;
3682         m_client = 0;
3683     }
3684     else if (!m_parent)
3685     {
3686         yaz_log(YLOG_LOG, "%sshutdown (client to proxy) bad state",
3687                  m_session_str);
3688         assert (m_parent);
3689     }
3690     else
3691     {
3692         yaz_log(YLOG_LOG, "%sShutdown (client to proxy)",
3693                  m_session_str);
3694     }
3695     if (m_parent)
3696         m_parent->pre_init();
3697 }
3698
3699 bool Yaz_Proxy::dec_ref()
3700 {
3701     m_http_keepalive = 0;
3702
3703     --m_ref_count;
3704     if (m_ref_count > 0)
3705         return false;
3706
3707     releaseClient();
3708
3709     delete this;
3710     return true;
3711 }
3712
3713 const char *Yaz_ProxyClient::get_session_str()
3714 {
3715     if (!m_server)
3716         return "0 ";
3717     return m_server->get_session_str();
3718 }
3719
3720 void Yaz_ProxyClient::shutdown()
3721 {
3722     yaz_log(YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3723              get_hostname());
3724
3725     if (m_server)
3726     {
3727         m_waiting = 1;   // ensure it's released from Yaz_Proxy::releaseClient
3728         m_server->dec_ref();
3729     }
3730     else
3731         delete this;
3732 }
3733
3734 void Yaz_Proxy::failNotify()
3735 {
3736     inc_request_no();
3737     yaz_log(YLOG_LOG, "%sConnection closed by client", get_session_str());
3738     dec_ref();
3739 }
3740
3741 void Yaz_Proxy::send_response_fail_client(const char *addr)
3742 {
3743     if (m_http_version)
3744     {
3745         Z_SRW_diagnostic *diagnostic = 0;
3746         int num_diagnostic = 0;
3747
3748         yaz_add_srw_diagnostic(odr_encode(),
3749                                &diagnostic, &num_diagnostic,
3750                                YAZ_SRW_SYSTEM_TEMPORARILY_UNAVAILABLE, addr);
3751         if (m_s2z_search_apdu)
3752             send_srw_search_response(diagnostic, num_diagnostic);
3753         else if (m_s2z_scan_apdu)
3754             send_srw_scan_response(diagnostic, num_diagnostic);
3755         else
3756             send_srw_explain_response(diagnostic, num_diagnostic);
3757     }
3758 }
3759
3760 void Yaz_ProxyClient::failNotify()
3761 {
3762     if (m_server)
3763         m_server->inc_request_no();
3764     yaz_log(YLOG_LOG, "%sConnection closed by target %s",
3765              get_session_str(), get_hostname());
3766
3767     if (m_server)
3768         m_server->send_response_fail_client(get_hostname());
3769     shutdown();
3770 }
3771
3772 void Yaz_ProxyClient::connectNotify()
3773 {
3774     const char *s = get_session_str();
3775     const char *h = get_hostname();
3776     yaz_log(YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3777              m_target_idletime);
3778     timeout(m_target_idletime);
3779     if (!m_server)
3780         pre_init_client();
3781 }
3782
3783 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3784                                               *the_PDU_Observable, int fd)
3785 {
3786     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3787 }
3788
3789 Yaz_ProxyClient::~Yaz_ProxyClient()
3790 {
3791     if (m_prev)
3792         *m_prev = m_next;
3793     if (m_next)
3794         m_next->m_prev = m_prev;
3795     m_waiting = 2;     // for debugging purposes only.
3796     odr_destroy(m_init_odr);
3797     odr_destroy(m_idAuthentication_odr);
3798     delete m_last_query;
3799     xfree(m_last_resultSetId);
3800     xfree(m_cookie);
3801 }
3802
3803 void Yaz_ProxyClient::pre_init_client()
3804 {
3805     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3806     Z_InitRequest *req = apdu->u.initRequest;
3807
3808     int i;
3809     for (i = 0; i<= 24; i++)
3810         ODR_MASK_SET(req->options, i);
3811     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3812                    Z_Options_negotiationModel);
3813     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3814                    Z_Options_concurrentOperations);
3815     for (i = 0; i<= 10; i++)
3816         ODR_MASK_SET(req->protocolVersion, i);
3817
3818     if (send_to_target(apdu) < 0)
3819     {
3820         delete this;
3821     }
3822     else
3823     {
3824         m_waiting = 1;
3825         m_init_flag = 1;
3826     }
3827 }
3828
3829 void Yaz_Proxy::pre_init()
3830 {
3831     int i;
3832     const char *name = 0;
3833     const char *zurl_in_use[MAX_ZURL_PLEX];
3834     int limit_bw, limit_pdu, limit_req, limit_search;
3835     int target_idletime, client_idletime;
3836     int max_sockets = m_max_sockets;
3837     int max_clients;
3838     int keepalive_limit_bw, keepalive_limit_pdu;
3839     int pre_init;
3840     const char *cql2rpn = 0;
3841     const char *authentication = 0;
3842     const char *negotiation_charset = 0;
3843     const char *negotiation_lang = 0;
3844
3845     Yaz_ProxyConfig *cfg = check_reconfigure();
3846
3847     zurl_in_use[0] = 0;
3848
3849     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3850         set_APDU_yazlog(1);
3851     else
3852         set_APDU_yazlog(0);
3853
3854     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3855                                           &limit_bw, &limit_pdu, &limit_req,
3856                                           &limit_search,
3857                                           &target_idletime, &client_idletime,
3858                                           &max_sockets,
3859                                           &max_clients,
3860                                           &keepalive_limit_bw,
3861                                           &keepalive_limit_pdu,
3862                                           &pre_init,
3863                                           &cql2rpn,
3864                                           &authentication,
3865                                           &negotiation_charset,
3866                                           &negotiation_lang,
3867                                           0,
3868                                           0) ; i++)
3869     {
3870         if (pre_init)
3871         {
3872             int j;
3873             for (j = 0; zurl_in_use[j]; j++)
3874             {
3875                 Yaz_ProxyClient *c;
3876                 int spare = 0;
3877                 int spare_waiting = 0;
3878                 int in_use = 0;
3879                 int other = 0;
3880                 for (c = m_clientPool; c; c = c->m_next)
3881                 {
3882                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3883                     {
3884                         if (c->m_cookie == 0)
3885                         {
3886                             if (c->m_server == 0)
3887                                 if (c->m_waiting)
3888                                     spare_waiting++;
3889                                 else
3890                                     spare++;
3891                             else
3892                                 in_use++;
3893                         }
3894                         else
3895                             other++;
3896                     }
3897                 }
3898                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3899                         "sparew=%d preinit=%d",m_session_str,
3900                         name, zurl_in_use[j], in_use, other,
3901                         spare, spare_waiting, pre_init);
3902                 if (spare + spare_waiting < pre_init
3903                     && in_use + spare + spare_waiting + other < max_sockets)
3904                 {
3905                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3906                     c->m_next = m_clientPool;
3907                     if (c->m_next)
3908                         c->m_next->m_prev = &c->m_next;
3909                     m_clientPool = c;
3910                     c->m_prev = &m_clientPool;
3911
3912                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3913                         c->set_APDU_yazlog(1);
3914                     else
3915                         c->set_APDU_yazlog(0);
3916
3917                     if (c->client(zurl_in_use[j]))
3918                     {
3919                         timeout(60);
3920                         delete c;
3921                         return;
3922                     }
3923                     c->timeout(30);
3924                     c->m_waiting = 1;
3925                     c->m_target_idletime = target_idletime;
3926                     c->m_seqno = m_seqno++;
3927                 }
3928             }
3929         }
3930     }
3931 }
3932
3933 void Yaz_Proxy::timeoutNotify()
3934 {
3935     if (m_parent)
3936     {
3937         GDU *gdu;
3938         switch(m_timeout_mode)
3939         {
3940         case timeout_normal:
3941         case timeout_busy:
3942             inc_request_no();
3943             m_in_queue.clear();
3944             yaz_log(YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3945             dec_ref();
3946             break;
3947         case timeout_reduce:
3948             timeout(m_client_idletime);
3949             m_timeout_mode = timeout_busy;
3950             gdu = m_timeout_gdu;
3951             m_timeout_gdu = 0;
3952             recv_GDU_normal(gdu);
3953             break;
3954         case timeout_xsl:
3955             assert(m_stylesheet_nprl);
3956             convert_xsl_delay();
3957             recv_GDU_more(true);
3958         }
3959     }
3960     else
3961     {
3962         timeout(600);
3963         pre_init();
3964     }
3965 }
3966
3967 void Yaz_Proxy::markInvalid()
3968 {
3969     m_client = 0;
3970     m_flag_invalid_session = 1;
3971 }
3972
3973 void Yaz_ProxyClient::timeoutNotify()
3974 {
3975     if (m_server)
3976         m_server->inc_request_no();
3977
3978     yaz_log(YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3979              get_hostname());
3980
3981     if (m_server)
3982         m_server->send_response_fail_client(get_hostname());
3983
3984     Yaz_Proxy *proxy_root = m_root;
3985
3986     shutdown();
3987
3988     proxy_root->pre_init();
3989 }
3990
3991 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
3992                                  Yaz_Proxy *parent) :
3993     Z_Assoc (the_PDU_Observable)
3994 {
3995     m_cookie = 0;
3996     m_next = 0;
3997     m_prev = 0;
3998     m_init_flag = 0;
3999     m_last_query = 0;
4000     m_last_resultSetId = 0;
4001     m_last_resultCount = 0;
4002     m_last_ok = 0;
4003     m_sr_transform = 0;
4004     m_waiting = 0;
4005     m_init_odr = odr_createmem (ODR_DECODE);
4006     m_initResponse = 0;
4007     m_initResponse_options = 0;
4008     m_initResponse_version = 0;
4009     m_initResponse_preferredMessageSize = 0;
4010     m_initResponse_maximumRecordSize = 0;
4011     m_resultSetStartPoint = 0;
4012     m_bytes_sent = m_bytes_recv = 0;
4013     m_pdu_recv = 0;
4014     m_server = 0;
4015     m_seqno = 0;
4016     m_target_idletime = 600;
4017     m_root = parent;
4018     m_idAuthentication_odr = odr_createmem(ODR_ENCODE);
4019     m_idAuthentication_ber_buf = 0;
4020     m_idAuthentication_ber_size = 0;
4021 }
4022
4023 const char *Yaz_Proxy::option(const char *name, const char *value)
4024 {
4025     if (!strcmp (name, "optimize")) {
4026         if (value) {
4027             xfree(m_optimize);
4028             m_optimize = xstrdup(value);
4029         }
4030         return m_optimize;
4031     }
4032     return 0;
4033 }
4034
4035 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
4036 {
4037
4038 }
4039
4040 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
4041 {
4042     if (apdu->which == Z_GDU_Z3950)
4043         recv_Z_PDU(apdu->u.z3950, len);
4044     else if (apdu->which == Z_GDU_HTTP_Response)
4045         recv_HTTP_response(apdu->u.HTTP_Response, len);
4046     else
4047         shutdown();
4048 }
4049
4050 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
4051 {
4052     if (!m_flag_invalid_session)
4053         return 0;
4054     m_flag_invalid_session = 0;
4055     handle_incoming_Z_PDU(m_apdu_invalid_session);
4056     assert (m_mem_invalid_session);
4057     nmem_destroy(m_mem_invalid_session);
4058     m_mem_invalid_session = 0;
4059     return 1;
4060 }
4061
4062 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
4063 {
4064     m_bytes_recv += len;
4065
4066     m_pdu_recv++;
4067     m_waiting = 0;
4068     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
4069         yaz_log(YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
4070                  apdu_name(apdu), get_hostname(), len);
4071     if (apdu->which == Z_APDU_initResponse)
4072     {
4073         if (!m_server)  // if this is a pre init session , check for more
4074             m_root->pre_init();
4075         NMEM nmem = odr_extract_mem (odr_decode());
4076         odr_reset (m_init_odr);
4077         nmem_transfer (m_init_odr->mem, nmem);
4078         m_initResponse = apdu;
4079         m_initResponse_options = apdu->u.initResponse->options;
4080         m_initResponse_version = apdu->u.initResponse->protocolVersion;
4081         m_initResponse_preferredMessageSize =
4082             *apdu->u.initResponse->preferredMessageSize;
4083         m_initResponse_maximumRecordSize =
4084             *apdu->u.initResponse->maximumRecordSize;
4085
4086         Z_InitResponse *ir = apdu->u.initResponse;
4087
4088         // apply YAZ Proxy version
4089         char *imv0 = ir->implementationVersion;
4090         char *imv1 = (char*)
4091             odr_malloc(m_init_odr, 20 + (imv0 ? strlen(imv0) : 0));
4092         *imv1 = '\0';
4093         if (imv0)
4094             strcat(imv1, imv0);
4095 #ifdef VERSION
4096         strcat(imv1, "/" VERSION);
4097 #endif
4098         ir->implementationVersion = imv1;
4099
4100         // apply YAZ Proxy implementation name
4101         char *im0 = ir->implementationName;
4102         char *im1 = (char*)
4103             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
4104         *im1 = '\0';
4105         if (im0)
4106         {
4107             strcat(im1, im0);
4108             strcat(im1, " ");
4109         }
4110         strcat(im1, "(YAZ Proxy)");
4111         ir->implementationName = im1;
4112
4113         nmem_destroy (nmem);
4114
4115         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
4116             return;
4117     }
4118     if (apdu->which == Z_APDU_searchResponse)
4119     {
4120         Z_SearchResponse *sr = apdu->u.searchResponse;
4121         m_last_resultCount = *sr->resultCount;
4122         int status = *sr->searchStatus;
4123         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
4124         {
4125             m_last_ok = 1;
4126
4127             if (sr->records && sr->records->which == Z_Records_DBOSD)
4128             {
4129                 m_cache.add(odr_decode(),
4130                             sr->records->u.databaseOrSurDiagnostics, 1,
4131                             *sr->resultCount);
4132             }
4133         }
4134     }
4135     if (apdu->which == Z_APDU_presentResponse)
4136     {
4137         Z_PresentResponse *pr = apdu->u.presentResponse;
4138         if (m_sr_transform)
4139         {
4140             m_sr_transform = 0;
4141             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
4142             Z_SearchResponse *sr = new_apdu->u.searchResponse;
4143             sr->referenceId = pr->referenceId;
4144             *sr->resultCount = m_last_resultCount;
4145             sr->records = pr->records;
4146             sr->nextResultSetPosition = pr->nextResultSetPosition;
4147             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
4148             apdu = new_apdu;
4149         }
4150         if (pr->records &&
4151             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
4152         {
4153             m_cache.add(odr_decode(),
4154                         pr->records->u.databaseOrSurDiagnostics,
4155                         m_resultSetStartPoint, -1);
4156             m_resultSetStartPoint = 0;
4157         }
4158     }
4159     if (m_cookie)
4160         set_otherInformationString(apdu, yaz_oid_userinfo_cookie, 1, m_cookie);
4161
4162     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
4163
4164     if (server)
4165         server->send_to_client(apdu);
4166     if (apdu->which == Z_APDU_close)
4167         shutdown();
4168     else if (server)
4169         server->recv_GDU_more(true);
4170 }
4171
4172 void Yaz_Proxy::low_socket_close()
4173 {
4174 #if WIN32
4175 #else
4176     int i;
4177     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
4178         if  (m_lo_fd[i] >= 0)
4179             ::close(m_lo_fd[i]);
4180 #endif
4181 }
4182
4183 void Yaz_Proxy::low_socket_open()
4184 {
4185 #if WIN32
4186 #else
4187     int i;
4188     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
4189         m_lo_fd[i] = open("/dev/null", O_RDONLY);
4190 #endif
4191 }
4192
4193 int Yaz_Proxy::server(const char *addr)
4194 {
4195     int r = Z_Assoc::server(addr);
4196     if (!r)
4197     {
4198         yaz_log(YLOG_LOG, "%sStarted proxy "
4199 #ifdef VERSION
4200             VERSION
4201 #endif
4202             " on %s", m_session_str, addr);
4203         timeout(1);
4204     }
4205     return r;
4206 }
4207
4208 void Yaz_Proxy::base64_decode(const char *base64, char *buf, int buf_len)
4209 {
4210     const char *base64_chars =
4211         "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
4212     int len = strlen(base64);
4213     int buf_pos = 0;
4214     int index = 1;
4215
4216     for (int pos = 0; pos <= len; pos++)
4217     {
4218         if (base64[pos] == '=' || buf_pos + 1 >= buf_len)
4219             break;
4220
4221         const char *ch_ptr = strchr(base64_chars, base64[pos]);
4222         if (!ch_ptr)
4223             break;
4224         char ch = (char) (ch_ptr - base64_chars);
4225         switch (index)
4226         {
4227         case 1:
4228             buf[buf_pos] = ch << 2;
4229             break;
4230         case 2:
4231             buf[buf_pos++] += (ch & 0x30) >> 4;
4232             buf[buf_pos] = (ch & 0x0f) << 4;
4233             break;
4234         case 3:
4235             buf[buf_pos++] += (ch & 0x3c) >> 2;
4236             buf[buf_pos] = (ch & 0x03) << 6;
4237             break;
4238         case 4:
4239             buf[buf_pos++] += ch;
4240         }
4241         if (index < 4)
4242             index++;
4243         else
4244             index = 1;
4245     }
4246     buf[buf_pos] = '\0';
4247 }
4248
4249 /*
4250  * Local variables:
4251  * c-basic-offset: 4
4252  * c-file-style: "Stroustrup"
4253  * indent-tabs-mode: nil
4254  * End:
4255  * vim: shiftwidth=4 tabstop=8 expandtab
4256  */
4257