src/yaz-proxy.cpp

   1 /* This file is part of YAZ proxy
   2    Copyright (C) 1998-2011 Index Data
   3
   4 YAZ proxy is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17 */
  18
  19 #ifdef WIN32
  20 #define HAVE_SYS_STAT_H 1
  21 #define HAVE_SYS_TYPES_H 1
  22 #endif
  23
  24 #if HAVE_UNISTD_H
  25 #include <unistd.h>
  26 #endif
  27 #if HAVE_SYS_TIME_H
  28 #include <sys/time.h>
  29 #endif
  30 #if HAVE_SYS_TYPES_H
  31 #include <sys/types.h>
  32 #endif
  33 #if HAVE_SYS_STAT_H
  34 #include <sys/stat.h>
  35 #endif
  36
  37 #include <assert.h>
  38 #include <stdlib.h>
  39 #include <time.h>
  40 #include <fcntl.h>
  41
  42 #include <yaz/srw.h>
  43 #include <yaz/marcdisp.h>
  44 #include <yaz/yaz-iconv.h>
  45 #include <yaz/log.h>
  46 #include <yaz/diagbib1.h>
  47 #include "proxyp.h"
  48 #include <yaz/pquery.h>
  49 #include <yaz/otherinfo.h>
  50 #include <yaz/charneg.h>
  51 #include <yaz/oid_db.h>
  52 #include "msg-thread.h"
  53
  54 #if YAZ_VERSIONL >= 0x03001D
  55 #define YAZ_HAS_MK_SURROGATE 1
  56 #else
  57 #define YAZ_HAS_MK_SURROGATE 0
  58 #endif
  59
  60 using namespace yazpp_1;
  61
  62 #ifdef WIN32
  63 #define strncasecmp _strnicmp
  64 #endif
  65
  66 class YAZ_EXPORT Auth_Msg : public IMsg_Thread {
  67 public:
  68     int m_ret;
  69     IMsg_Thread *handle();
  70     void result();
  71     Yaz_Proxy *m_proxy;
  72     NMEM m_nmem;
  73     char *m_apdu_buf;
  74     int m_apdu_len;
  75     Auth_Msg();
  76     virtual ~Auth_Msg();
  77 };
  78
  79 Auth_Msg::Auth_Msg()
  80 {
  81     m_nmem = nmem_create();
  82 }
  83
  84 Auth_Msg::~Auth_Msg()
  85 {
  86     nmem_destroy(m_nmem);
  87 }
  88
  89 IMsg_Thread *Auth_Msg::handle()
  90 {
  91     ODR decode = odr_createmem(ODR_DECODE);
  92     Z_APDU *apdu;
  93
  94     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
  95     int r = z_APDU(decode, &apdu, 0, 0);
  96     if (!r)
  97     {
  98         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
  99     }
 100     else
 101     {
 102         m_ret = m_proxy->handle_authentication(apdu);
 103     }
 104     odr_destroy(decode);
 105     return this;
 106 }
 107
 108 void Auth_Msg::result()
 109 {
 110     if (m_proxy->dec_ref())
 111     {
 112         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
 113     }
 114     else
 115     {
 116         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
 117         Z_APDU *apdu = 0;
 118         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
 119         if (!r)
 120             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
 121         m_proxy->result_authentication(apdu, m_ret);
 122     }
 123     delete this;
 124 }
 125
 126 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
 127 {
 128     if (apdu == 0 || ret == 0)
 129     {
 130         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
 131         *apdu_reject->u.initResponse->result = 0;
 132         send_to_client(apdu_reject);
 133         dec_ref();
 134     }
 135     else
 136     {
 137         if (apdu->which == Z_APDU_initRequest)
 138         {
 139             Yaz_ProxyConfig *cfg = check_reconfigure();
 140             if (cfg)
 141                 cfg->target_authentication(m_default_target, odr_encode(),
 142                                            apdu->u.initRequest);
 143         }
 144         handle_incoming_Z_PDU_2(apdu);
 145     }
 146 }
 147
 148 static const char *apdu_name(Z_APDU *apdu)
 149 {
 150     switch (apdu->which)
 151     {
 152     case Z_APDU_initRequest:
 153         return "initRequest";
 154     case Z_APDU_initResponse:
 155         return "initResponse";
 156     case Z_APDU_searchRequest:
 157         return "searchRequest";
 158     case Z_APDU_searchResponse:
 159         return "searchResponse";
 160     case Z_APDU_presentRequest:
 161         return "presentRequest";
 162     case Z_APDU_presentResponse:
 163         return "presentResponse";
 164     case Z_APDU_deleteResultSetRequest:
 165         return "deleteResultSetRequest";
 166     case Z_APDU_deleteResultSetResponse:
 167         return "deleteResultSetResponse";
 168     case Z_APDU_scanRequest:
 169         return "scanRequest";
 170     case Z_APDU_scanResponse:
 171         return "scanResponse";
 172     case Z_APDU_sortRequest:
 173         return "sortRequest";
 174     case Z_APDU_sortResponse:
 175         return "sortResponse";
 176     case Z_APDU_extendedServicesRequest:
 177         return "extendedServicesRequest";
 178     case Z_APDU_extendedServicesResponse:
 179         return "extendedServicesResponse";
 180     case Z_APDU_close:
 181         return "close";
 182     }
 183     return "other";
 184 }
 185
 186 static const char *gdu_name(Z_GDU *gdu)
 187 {
 188     switch(gdu->which)
 189     {
 190     case Z_GDU_Z3950:
 191         return apdu_name(gdu->u.z3950);
 192     case Z_GDU_HTTP_Request:
 193         return "HTTP Request";
 194     case Z_GDU_HTTP_Response:
 195         return "HTTP Response";
 196     }
 197     return "Unknown request/response";
 198 }
 199
 200 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
 201                      ISocketObservable *the_socket_observable,
 202                      Yaz_Proxy *parent)
 203     :
 204     Z_Assoc(the_PDU_Observable),
 205     m_bw_stat(60), m_pdu_stat(60), m_search_stat(60)
 206 {
 207     m_PDU_Observable = the_PDU_Observable;
 208     m_socket_observable = the_socket_observable;
 209     m_client = 0;
 210     m_parent = parent;
 211     m_clientPool = 0;
 212     m_seqno = 1;
 213     m_keepalive_limit_bw = 500000;
 214     m_keepalive_limit_pdu = 1000;
 215     m_proxyTarget = 0;
 216     m_default_target = 0;
 217     m_proxy_negotiation_charset = 0;
 218     m_proxy_negotiation_lang = 0;
 219     m_proxy_negotiation_default_charset = 0;
 220     m_charset_converter = new Yaz_CharsetConverter;
 221     m_max_clients = 150;
 222     m_log_mask = 0;
 223     m_seed = time(0);
 224     m_client_idletime = 600;
 225     m_target_idletime = 600;
 226     m_max_sockets = 1024;
 227     m_optimize = xstrdup("1");
 228     strcpy(m_session_str, "0 ");
 229     m_session_no = 0;
 230     m_bytes_sent = 0;
 231     m_bytes_recv = 0;
 232     m_bw_max = 0;
 233     m_pdu_max = 0;
 234     m_search_max = 0;
 235     m_max_connect = 0;
 236     m_max_connect_period = 0;
 237     m_limit_connect = 0;
 238     m_limit_connect_period = 0;
 239     m_timeout_mode = timeout_normal;
 240     m_timeout_gdu = 0;
 241     m_max_record_retrieve = 0;
 242     m_reconfig_flag = 0;
 243     m_config_fname = 0;
 244     m_request_no = 0;
 245     m_flag_invalid_session = 0;
 246     m_referenceId = 0;
 247     m_referenceId_mem = nmem_create();
 248     m_config = 0;
 249     m_marcxml_mode = none;
 250     m_stylesheet_xsp = 0;
 251     m_stylesheet_nprl = 0;
 252     m_stylesheet_apdu = 0;
 253     m_s2z_stylesheet = 0;
 254     m_s2z_database = 0;
 255     m_schema = 0;
 256     m_backend_type = 0;
 257     m_backend_charset = 0;
 258     m_frontend_type[0] = -1;
 259     m_initRequest_apdu = 0;
 260     m_initRequest_mem = 0;
 261     m_initRequest_preferredMessageSize = 0;
 262     m_initRequest_maximumRecordSize = 0;
 263     m_initRequest_options = 0;
 264     m_initRequest_version = 0;
 265     m_initRequest_oi_negotiation_charsets = 0;
 266     m_initRequest_oi_negotiation_num_charsets = 0;
 267     m_initRequest_oi_negotiation_langs = 0;
 268     m_initRequest_oi_negotiation_num_langs = 0;
 269     m_initRequest_oi_negotiation_selected = 0;
 270     m_apdu_invalid_session = 0;
 271     m_mem_invalid_session = 0;
 272     m_s2z_odr_init = 0;
 273     m_s2z_odr_search = 0;
 274     m_s2z_odr_scan = 0;
 275     m_s2z_init_apdu = 0;
 276     m_s2z_search_apdu = 0;
 277     m_s2z_present_apdu = 0;
 278     m_s2z_scan_apdu = 0;
 279     m_http_keepalive = 0;
 280     m_http_version = 0;
 281     m_soap_ns = 0;
 282     m_s2z_packing = Z_SRW_recordPacking_string;
 283 #if HAVE_GETTIMEOFDAY
 284     m_time_tv = xmalloc(sizeof(struct timeval));
 285     struct timeval *tv = (struct timeval *) m_time_tv;
 286     tv->tv_sec = 0;
 287     tv->tv_usec = 0;
 288 #else
 289     m_time_tv = 0;
 290 #endif
 291     m_usemarcon_ini_stage1 = 0;
 292     m_usemarcon_ini_stage2 = 0;
 293     m_usemarcon = new Yaz_usemarcon();
 294     if (!m_parent)
 295         low_socket_open();
 296     m_backend_elementset = 0;
 297     m_my_thread = 0;
 298     m_ref_count = 1;
 299     m_main_ptr_dec = false;
 300     m_peername = 0;
 301     m_num_msg_threads = 0;
 302 }
 303
 304 void Yaz_Proxy::inc_ref()
 305 {
 306     m_ref_count++;
 307 }
 308
 309 Yaz_Proxy::~Yaz_Proxy()
 310 {
 311     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 312             m_bytes_sent, m_bytes_recv);
 313     nmem_destroy(m_initRequest_mem);
 314     nmem_destroy(m_mem_invalid_session);
 315     nmem_destroy(m_referenceId_mem);
 316
 317     xfree(m_proxyTarget);
 318     xfree(m_default_target);
 319     xfree(m_proxy_negotiation_charset);
 320     xfree(m_proxy_negotiation_lang);
 321     xfree(m_proxy_negotiation_default_charset);
 322     delete m_charset_converter;
 323     xfree(m_optimize);
 324
 325 #if YAZ_HAVE_XSLT
 326     if (m_stylesheet_xsp)
 327         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 328 #endif
 329     xfree(m_time_tv);
 330
 331     xfree(m_peername);
 332     xfree(m_schema);
 333     xfree(m_backend_type);
 334     xfree(m_backend_charset);
 335     xfree(m_usemarcon_ini_stage1);
 336     xfree(m_usemarcon_ini_stage2);
 337     xfree(m_backend_elementset);
 338     delete m_usemarcon;
 339     if (m_s2z_odr_init)
 340         odr_destroy(m_s2z_odr_init);
 341     if (m_s2z_odr_search)
 342         odr_destroy(m_s2z_odr_search);
 343     if (m_s2z_odr_scan)
 344         odr_destroy(m_s2z_odr_scan);
 345     if (!m_parent)
 346         low_socket_close();
 347     if (!m_parent)
 348         delete m_my_thread;
 349     delete m_config;
 350 }
 351
 352 void Yaz_Proxy::set_debug_mode(int mode)
 353 {
 354     m_debug_mode = mode;
 355 }
 356
 357 int Yaz_Proxy::set_config(const char *config)
 358 {
 359     delete m_config;
 360     m_config = new Yaz_ProxyConfig();
 361     xfree(m_config_fname);
 362     m_config_fname = xstrdup(config);
 363     int r = m_config->read_xml(config);
 364     if (!r)
 365     {
 366         int period = 60;
 367         m_config->get_generic_info(&m_log_mask, &m_max_clients,
 368                                    &m_max_connect, &m_limit_connect, &period,
 369                                    &m_num_msg_threads);
 370         m_connect.set_period(period);
 371     }
 372     return r;
 373 }
 374
 375 void Yaz_Proxy::set_default_target(const char *target)
 376 {
 377     xfree(m_default_target);
 378     m_default_target = 0;
 379     if (target)
 380         m_default_target = (char *) xstrdup(target);
 381 }
 382
 383 void Yaz_Proxy::set_proxy_negotiation(const char *charset, const char *lang,
 384                                        const char *default_charset)
 385 {
 386     yaz_log(YLOG_DEBUG, "%sSet the proxy negotiation: charset to '%s', "
 387         "default charset to '%s', language to '%s'", m_session_str,
 388         charset?charset:"none",
 389         default_charset?default_charset:"none",
 390         lang?lang:"none");
 391     xfree(m_proxy_negotiation_charset);
 392     xfree(m_proxy_negotiation_lang);
 393     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 394     if (charset)
 395         m_proxy_negotiation_charset = (char *) xstrdup(charset);
 396     if (lang)
 397         m_proxy_negotiation_lang = (char *) xstrdup(lang);
 398     if (default_charset)
 399         m_proxy_negotiation_default_charset =
 400             (char *) xstrdup(default_charset);
 401 }
 402
 403 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 404 {
 405     if (m_parent)
 406         return m_parent->check_reconfigure();
 407
 408     Yaz_ProxyConfig *cfg = m_config;
 409     if (m_reconfig_flag)
 410     {
 411         yaz_log(YLOG_LOG, "reconfigure");
 412         if (m_config_fname && cfg)
 413         {
 414             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 415             int r = cfg->read_xml(m_config_fname);
 416             if (r)
 417                 yaz_log(YLOG_WARN, "reconfigure failed");
 418             else
 419             {
 420                 m_log_mask = 0;
 421                 int period = 60;
 422                 cfg->get_generic_info(&m_log_mask, &m_max_clients,
 423                                       &m_max_connect, &m_limit_connect,
 424                                       &period, &m_num_msg_threads);
 425                 m_connect.set_period(period);
 426             }
 427         }
 428         else
 429             yaz_log(YLOG_LOG, "reconfigure");
 430         m_reconfig_flag = 0;
 431     }
 432     return cfg;
 433 }
 434
 435 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 436                                         *the_PDU_Observable, int fd)
 437 {
 438     check_reconfigure();
 439
 440     char session_str[200];
 441     const char *peername = the_PDU_Observable->getpeername();
 442     if (!peername)
 443         peername = "nullpeer";
 444
 445     if (m_log_mask & PROXY_LOG_IP_CLIENT)
 446         sprintf(session_str, "%ld:%d %.80s %d ",
 447                 (long) time(0), m_session_no, peername, 0);
 448     else
 449         sprintf(session_str, "%ld:%d %d ",
 450                 (long) time(0), m_session_no, 0);
 451     m_session_no++;
 452
 453     yaz_log(YLOG_LOG, "%sNew session %s", session_str, peername);
 454
 455     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 456                                          m_socket_observable, this);
 457
 458     new_proxy->m_config = 0;
 459     new_proxy->m_config_fname = 0;
 460     new_proxy->timeout(m_client_idletime);
 461     new_proxy->m_target_idletime = m_target_idletime;
 462     new_proxy->set_default_target(m_default_target);
 463     new_proxy->m_max_clients = m_max_clients;
 464     new_proxy->m_log_mask = m_log_mask;
 465     new_proxy->m_session_no = m_session_no;
 466     new_proxy->m_num_msg_threads = m_num_msg_threads;
 467
 468 #if 0
 469     // in case we want to watch a particular client..
 470     if (!strcmp(peername, "tcp:163.121.19.82")) // NIS GROUP
 471         new_proxy->m_log_mask = 255;
 472 #endif
 473
 474     new_proxy->set_APDU_log(get_APDU_log());
 475     if (new_proxy->m_log_mask & PROXY_LOG_APDU_CLIENT)
 476         new_proxy->set_APDU_yazlog(1);
 477     else
 478         new_proxy->set_APDU_yazlog(0);
 479     strcpy(new_proxy->m_session_str, session_str);
 480     new_proxy->m_peername = xstrdup(peername);
 481     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 482         m_proxy_negotiation_lang, m_proxy_negotiation_default_charset);
 483     // create thread object the first time we get an incoming connection
 484     if (!m_my_thread && m_num_msg_threads > 0)
 485     {
 486         yaz_log(YLOG_LOG, "%sStarting message thread management. number=%d",
 487                  session_str, m_num_msg_threads);
 488         m_my_thread = new Msg_Thread(m_socket_observable, m_num_msg_threads);
 489     }
 490     new_proxy->m_my_thread = m_my_thread;
 491     return new_proxy;
 492 }
 493
 494 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 495 {
 496     Z_OtherInformationUnit *oi =
 497         update_otherInformation(otherInfo, 0, yaz_oid_userinfo_cookie, 1, 1);
 498
 499     if (oi && oi->which == Z_OtherInfo_characterInfo)
 500         return oi->information.characterInfo;
 501     return 0;
 502 }
 503
 504 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 505 {
 506     Z_OtherInformationUnit *oi =
 507         update_otherInformation(otherInfo, 0, yaz_oid_userinfo_proxy, 1, 1);
 508
 509     if (oi && oi->which == Z_OtherInfo_characterInfo)
 510         return oi->information.characterInfo;
 511     return 0;
 512 }
 513 const char *Yaz_Proxy::load_balance(const char **url)
 514 {
 515     int zurl_in_use[MAX_ZURL_PLEX];
 516     int zurl_in_spare[MAX_ZURL_PLEX];
 517     Yaz_ProxyClient *c;
 518     int i;
 519
 520     for (i = 0; i<MAX_ZURL_PLEX; i++)
 521     {
 522         zurl_in_use[i] = 0;
 523         zurl_in_spare[i] = 0;
 524     }
 525     for (c = m_parent->m_clientPool; c; c = c->m_next)
 526     {
 527         for (i = 0; url[i]; i++)
 528             if (!strcmp(url[i], c->get_hostname()))
 529             {
 530                 zurl_in_use[i]++;
 531                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 532                     zurl_in_spare[i]++;
 533             }
 534     }
 535     int min_use = 100000;
 536     int max_spare = 0;
 537     const char *ret_min = 0;
 538     for (i = 0; url[i]; i++)
 539     {
 540         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 541                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 542         if (min_use > zurl_in_use[i])
 543         {
 544             ret_min = url[i];
 545             min_use = zurl_in_use[i];
 546         }
 547         if (max_spare < zurl_in_spare[i])
 548         {
 549             max_spare = zurl_in_spare[i];
 550         }
 551     }
 552     return ret_min;
 553 }
 554
 555 int Yaz_Proxy::get_number_of_connections()
 556 {
 557     int no_connections = 0;
 558     Yaz_ProxyClient *c;
 559
 560     for (c = m_parent->m_clientPool; c; c = c->m_next)
 561     {
 562         assert(c->m_prev);
 563         assert(*c->m_prev == c);
 564         if (!strcmp(m_proxyTarget, c->get_hostname()))
 565         {
 566             no_connections++;
 567         }
 568     }
 569     yaz_log(YLOG_LOG, "%sExisting %s connections: %d", m_session_str, m_proxyTarget,
 570         no_connections);
 571     return no_connections;
 572 }
 573
 574 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 575                                        const char *proxy_host, int *http_code)
 576 {
 577     assert(m_parent);
 578     Yaz_Proxy *parent = m_parent;
 579     Yaz_ProxyClient *c = m_client;
 580
 581     if (!m_proxyTarget)
 582     {
 583         const char *url[MAX_ZURL_PLEX];
 584         Yaz_ProxyConfig *cfg = check_reconfigure();
 585         if (proxy_host)
 586         {
 587             if (parent && parent->m_debug_mode)
 588             {
 589                 // only to be enabled for debugging...
 590                 if (!strcmp(proxy_host, "stop"))
 591                     exit(0);
 592             }
 593             xfree(m_default_target);
 594             m_default_target = xstrdup(proxy_host);
 595         }
 596         proxy_host = m_default_target;
 597         int client_idletime = -1;
 598         const char *cql2rpn_fname = 0;
 599         const char *negotiation_charset = 0;
 600         const char *negotiation_lang = 0;
 601         const char *query_charset = 0;
 602         const char *default_client_query_charset = 0;
 603         url[0] = m_default_target;
 604         url[1] = 0;
 605         if (cfg)
 606         {
 607             int pre_init = 0;
 608             cfg->get_target_info(proxy_host, url, &m_bw_max,
 609                                  &m_pdu_max, &m_max_record_retrieve,
 610                                  &m_search_max,
 611                                  &m_target_idletime, &client_idletime,
 612                                  &m_max_sockets,
 613                                  &parent->m_max_clients,
 614                                  &m_keepalive_limit_bw,
 615                                  &m_keepalive_limit_pdu,
 616                                  &pre_init,
 617                                  &cql2rpn_fname,
 618                                  &negotiation_charset,
 619                                  &negotiation_lang,
 620                                  &query_charset,
 621                                  &default_client_query_charset);
 622         }
 623         if (client_idletime != -1)
 624         {
 625             m_client_idletime = client_idletime;
 626             timeout(m_client_idletime);
 627         }
 628
 629         // get those FILE descriptors available
 630         m_parent->low_socket_close();
 631         if (cql2rpn_fname)
 632             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 633         // reserve them again
 634         m_parent->low_socket_open();
 635
 636         if (negotiation_charset || negotiation_lang || default_client_query_charset)
 637         {
 638             set_proxy_negotiation(negotiation_charset,
 639                 negotiation_lang, default_client_query_charset);
 640         }
 641         m_charset_converter->set_target_query_charset(query_charset);
 642         if (!url[0])
 643         {
 644             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 645             return 0;
 646         }
 647         // we don't handle multiplexing for cookie session, so we just
 648         // pick the first one in this case (anonymous users will be able
 649         // to use any backend)
 650         if (cookie && *cookie)
 651             m_proxyTarget = (char*) xstrdup(url[0]);
 652         else
 653             m_proxyTarget = (char*) xstrdup(load_balance(url));
 654     }
 655     if (cookie && *cookie)
 656     {   // search in sessions with a cookie
 657         for (c = parent->m_clientPool; c; c = c->m_next)
 658         {
 659             assert(c->m_prev);
 660             assert(*c->m_prev == c);
 661             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 662                 !strcmp(m_proxyTarget, c->get_hostname()))
 663             {
 664                 // Found it in cache
 665                 // The following handles "cancel"
 666                 // If connection is busy (waiting for PDU) and
 667                 // we have an initRequest we can safely do re-open
 668                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 669                 {
 670                     yaz_log(YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 671                              c->get_hostname());
 672                     c->close();
 673                     c->m_init_flag = 0;
 674
 675                     c->m_last_ok = 0;
 676                     c->m_cache.clear();
 677                     c->m_last_resultCount = 0;
 678                     c->m_sr_transform = 0;
 679                     c->m_waiting = 0;
 680                     c->m_resultSetStartPoint = 0;
 681                     c->m_target_idletime = m_target_idletime;
 682                     if (c->client(m_proxyTarget))
 683                     {
 684                         delete c;
 685                         return 0;
 686                     }
 687                     c->timeout(30);
 688                 }
 689                 c->m_seqno = parent->m_seqno;
 690                 if (c->m_server && c->m_server != this)
 691                     c->m_server->m_client = 0;
 692                 c->m_server = this;
 693                 (parent->m_seqno)++;
 694                 yaz_log(YLOG_DEBUG, "get_client 1 %p %p", this, c);
 695                 return c;
 696             }
 697         }
 698     }
 699     else if (!c && apdu->which == Z_APDU_initRequest )
 700     {
 701         // anonymous sessions without cookie.
 702         // if authentication is set it is NOT anonymous se we can't share them.
 703         // If charset and lang negotiation is use it is NOT anonymous session too.
 704         for (c = parent->m_clientPool; c; c = c->m_next)
 705         {
 706             assert(c->m_prev);
 707             assert(*c->m_prev == c);
 708             if (c->m_server == 0 && c->m_cookie == 0 && c->m_waiting == 0
 709                 && c->compare_idAuthentication(apdu)
 710                 && c->compare_charset(apdu)
 711                 && !strcmp(m_proxyTarget, c->get_hostname()))
 712             {
 713                 // found it in cache
 714                 yaz_log(YLOG_LOG, "%sREUSE %d %s",
 715                          m_session_str, parent->m_seqno, c->get_hostname());
 716
 717                 c->m_seqno = parent->m_seqno;
 718                 assert(c->m_server == 0);
 719                 c->m_server = this;
 720
 721                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 722                     c->set_APDU_yazlog(1);
 723                 else
 724                     c->set_APDU_yazlog(0);
 725
 726                 (parent->m_seqno)++;
 727
 728                 parent->pre_init();
 729
 730                 return c;
 731             }
 732         }
 733     }
 734     if (!m_client)
 735     {
 736         if (apdu->which != Z_APDU_initRequest)
 737         {
 738             yaz_log(YLOG_LOG, "%sno init request as first PDU", m_session_str);
 739             *http_code = 500;
 740             return 0;
 741         }
 742
 743         int no_in_use = get_number_of_connections();
 744         if (no_in_use >= m_max_sockets)
 745         {
 746             yaz_log(YLOG_LOG, "%smax sockets reached %d", m_session_str,
 747                 m_max_sockets);
 748             *http_code = 500;
 749             return 0;
 750         }
 751         // go through list of clients - and find the lowest/oldest one.
 752         Yaz_ProxyClient *c_min = 0;
 753         int min_seq = -1;
 754         int no_of_clients = 0;
 755         if (parent->m_clientPool)
 756             yaz_log(YLOG_DEBUG, "Existing sessions");
 757         for (c = parent->m_clientPool; c; c = c->m_next)
 758         {
 759             yaz_log(YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 760                                c->m_waiting, c->get_hostname(),
 761                                c->m_cookie ? c->m_cookie : "");
 762             no_of_clients++;
 763             if (min_seq < 0 || c->m_seqno < min_seq)
 764             {
 765                 min_seq = c->m_seqno;
 766                 c_min = c;
 767             }
 768         }
 769         if (no_of_clients >= parent->m_max_clients)
 770         {
 771             c = c_min;
 772             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 773             {
 774                 yaz_log(YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 775                          m_session_str, parent->m_max_clients, c->m_seqno);
 776                 if (c->m_server && c->m_server != this)
 777                     c->m_server->dec_ref();
 778             }
 779             else
 780             {
 781                 yaz_log(YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 782                          m_session_str, parent->m_max_clients,
 783                          c->m_seqno, parent->m_seqno, c->get_hostname());
 784                 xfree(c->m_cookie);
 785                 c->m_cookie = 0;
 786                 if (cookie)
 787                     c->m_cookie = xstrdup(cookie);
 788                 c->m_seqno = parent->m_seqno;
 789                 if (c->m_server && c->m_server != this)
 790                 {
 791                     c->m_server->m_client = 0;
 792                     c->m_server->dec_ref();
 793                 }
 794                 (parent->m_seqno)++;
 795                 c->m_target_idletime = m_target_idletime;
 796                 c->timeout(m_target_idletime);
 797
 798                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 799                     c->set_APDU_yazlog(1);
 800                 else
 801                     c->set_APDU_yazlog(0);
 802
 803                 return c;
 804             }
 805         }
 806         else
 807         {
 808
 809             yaz_log(YLOG_LOG, "%sNEW %d %s",
 810                      m_session_str, parent->m_seqno, m_proxyTarget);
 811             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 812             c->m_next = parent->m_clientPool;
 813             if (c->m_next)
 814                 c->m_next->m_prev = &c->m_next;
 815             parent->m_clientPool = c;
 816             c->m_prev = &parent->m_clientPool;
 817         }
 818
 819         xfree(c->m_cookie);
 820         c->m_cookie = 0;
 821         if (cookie)
 822             c->m_cookie = xstrdup(cookie);
 823
 824         c->m_seqno = parent->m_seqno;
 825         c->m_init_flag = 0;
 826         c->m_last_resultCount = 0;
 827         c->m_last_ok = 0;
 828         c->m_cache.clear();
 829         c->m_sr_transform = 0;
 830         c->m_waiting = 0;
 831         c->m_resultSetStartPoint = 0;
 832         (parent->m_seqno)++;
 833         if (c->client(m_proxyTarget))
 834         {
 835             delete c;
 836             return 0;
 837         }
 838         c->m_target_idletime = m_target_idletime;
 839         c->timeout(30);
 840
 841         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 842             c->set_APDU_yazlog(1);
 843         else
 844             c->set_APDU_yazlog(0);
 845
 846         c->set_idAuthentication(apdu);
 847     }
 848     yaz_log(YLOG_DEBUG, "get_client 3 %p %p", this, c);
 849     return c;
 850 }
 851
 852 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 853 {
 854     int i;
 855     for (i = 0; i<num; i++)
 856     {
 857         Z_DefaultDiagFormat *r;
 858         Z_DiagRec *p = pp[i];
 859         if (p->which != Z_DiagRec_defaultFormat)
 860         {
 861             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 862             return;
 863         }
 864         else
 865             r = p->u.defaultFormat;
 866         switch (r->which)
 867         {
 868         case Z_DefaultDiagFormat_v2Addinfo:
 869             yaz_log(YLOG_LOG, "%sError " ODR_INT_PRINTF " %s:%s",
 870                     m_session_str,
 871                     *r->condition, diagbib1_str(*r->condition),
 872                     r->u.v2Addinfo);
 873             break;
 874         case Z_DefaultDiagFormat_v3Addinfo:
 875             yaz_log(YLOG_LOG, "%sError " ODR_INT_PRINTF " %s:%s",
 876                     m_session_str,
 877                     *r->condition, diagbib1_str(*r->condition),
 878                     r->u.v3Addinfo);
 879             break;
 880         }
 881     }
 882 }
 883
 884 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 885 {
 886     if (!m_stylesheet_xsp || p->num_records <= 0)
 887     {
 888         return 0;  /* no XSLT to be done ... */
 889     }
 890
 891     m_stylesheet_offset = 0;
 892     m_stylesheet_nprl = p;
 893     m_stylesheet_apdu = apdu;
 894     m_timeout_mode = timeout_xsl;
 895
 896     timeout(0);
 897     return 1;
 898 }
 899
 900 void Yaz_Proxy::convert_xsl_delay()
 901 {
 902 #if YAZ_HAVE_XSLT
 903     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 904     if (npr->which == Z_NamePlusRecord_databaseRecord)
 905     {
 906         Z_External *r = npr->u.databaseRecord;
 907         if (r->which == Z_External_octet)
 908         {
 909 #if 0
 910             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 911 #endif
 912             xmlDocPtr res, doc = xmlParseMemory(
 913                 (char*) r->u.octet_aligned->buf,
 914                 r->u.octet_aligned->len);
 915
 916
 917             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 918                     m_session_str, m_stylesheet_offset);
 919             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 920                                       doc, 0);
 921
 922             if (res)
 923             {
 924                 xmlChar *out_buf;
 925                 int out_len;
 926                 xmlDocDumpFormatMemory(res, &out_buf, &out_len, 1);
 927                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 928                     u.databaseRecord =
 929                     z_ext_record_oid(odr_encode(), yaz_oid_recsyn_xml,
 930                                      (char*) out_buf, out_len);
 931                 xmlFree(out_buf);
 932                 xmlFreeDoc(res);
 933             }
 934
 935             xmlFreeDoc(doc);
 936         }
 937     }
 938 #endif
 939     m_stylesheet_offset++;
 940     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 941     {
 942         m_timeout_mode = timeout_normal;
 943         m_stylesheet_nprl = 0;
 944 #if YAZ_HAVE_XSLT
 945         if (m_stylesheet_xsp)
 946             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 947 #endif
 948         m_stylesheet_xsp = 0;
 949         timeout(m_client_idletime);
 950         send_PDU_convert(m_stylesheet_apdu);
 951     }
 952     else
 953         timeout(0);
 954 }
 955
 956 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 957 {
 958     if (m_frontend_type[0] != -1)
 959     {
 960         int i;
 961         for (i = 0; i < p->num_records; i++)
 962         {
 963             Z_NamePlusRecord *npr = p->records[i];
 964             if (npr->which == Z_NamePlusRecord_databaseRecord)
 965             {
 966                 Z_External *r = npr->u.databaseRecord;
 967                 if (r->which == Z_External_octet)
 968                 {
 969 #if !HAVE_USEMARCON
 970                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 971                         yaz_log(YLOG_LOG, "%sError: USEMARCON requested but not available",
 972                             m_session_str);
 973 #endif
 974 #if HAVE_USEMARCON
 975                     yaz_log(YLOG_DEBUG, "%sUSEMARCON stage1=%s stage2=%s",
 976                         m_session_str,
 977                         m_usemarcon_ini_stage1 ? m_usemarcon_ini_stage1 : "(none)",
 978                         m_usemarcon_ini_stage2 ? m_usemarcon_ini_stage2 : "(none)");
 979                     char *converted;
 980                     int convlen;
 981                     if (m_usemarcon->convert(m_usemarcon_ini_stage1, m_usemarcon_ini_stage2,
 982                         (char*) r->u.octet_aligned->buf, r->u.octet_aligned->len,
 983                         &converted, &convlen))
 984                     {
 985                         npr->u.databaseRecord =
 986                             z_ext_record_oid(odr_encode(),
 987                                              m_frontend_type,
 988                                              converted,
 989                                              strlen(converted));
 990                         free(converted);
 991                     }
 992                     else
 993 #endif
 994 /* HAVE_USEMARCON */
 995                     npr->u.databaseRecord =
 996                         z_ext_record_oid(odr_encode(),
 997                                          m_frontend_type,
 998                                          (char*) r->u.octet_aligned->buf,
 999                                          r->u.octet_aligned->len);
1000                 }
1001             }
1002         }
1003     }
1004 }
1005
1006 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
1007                                         const char *backend_charset)
1008 {
1009     int sel =   m_charset_converter->get_client_charset_selected();
1010     const char *client_record_charset =
1011         m_charset_converter->get_client_query_charset();
1012     if (sel && backend_charset && client_record_charset &&
1013         strcmp(backend_charset, client_record_charset))
1014     {
1015         int i;
1016         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
1017                                         backend_charset);
1018         yaz_marc_t mt = yaz_marc_create();
1019         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
1020         yaz_marc_iconv(mt, cd);
1021         for (i = 0; i < p->num_records; i++)
1022         {
1023             Z_NamePlusRecord *npr = p->records[i];
1024             if (npr->which == Z_NamePlusRecord_databaseRecord)
1025             {
1026                 Z_External *r = npr->u.databaseRecord;
1027                 const Odr_oid *oid = r->direct_reference;
1028                 if (!oid)
1029                     continue;
1030
1031                 if (!oid_oidcmp(oid, yaz_oid_recsyn_sutrs))
1032                 {
1033                     WRBUF w = wrbuf_alloc();
1034
1035                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
1036                                       r->u.octet_aligned->len);
1037                     npr->u.databaseRecord =
1038                         z_ext_record_oid(odr_encode(), oid, wrbuf_buf(w),
1039                                          wrbuf_len(w));
1040                     wrbuf_destroy(w);
1041                 }
1042                 else if (!oid_oidcmp(oid, yaz_oid_recsyn_xml))
1043                 {
1044                     ;
1045                 }
1046                 else if (r->which == Z_External_octet)
1047                 {
1048                     size_t rlen;
1049                     const char *result;
1050                     if (yaz_marc_decode_buf(mt,
1051                                             (char*) r->u.octet_aligned->buf,
1052                                             r->u.octet_aligned->len,
1053                                             &result, &rlen))
1054                     {
1055                         npr->u.databaseRecord =
1056                             z_ext_record_oid(odr_encode(), oid, result, rlen);
1057                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1058                                 m_session_str);
1059                     }
1060                 }
1061             }
1062         }
1063         if (cd)
1064             yaz_iconv_close(cd);
1065         yaz_marc_destroy(mt);
1066     }
1067 }
1068
1069 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1070                                    const char *backend_charset)
1071 {
1072     int i;
1073     if (!backend_charset)
1074         backend_charset = "MARC-8";
1075     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1076     yaz_marc_t mt = yaz_marc_create();
1077     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1078     yaz_marc_iconv(mt, cd);
1079     for (i = 0; i < p->num_records; i++)
1080     {
1081         Z_NamePlusRecord *npr = p->records[i];
1082         if (npr->which == Z_NamePlusRecord_databaseRecord)
1083         {
1084             Z_External *r = npr->u.databaseRecord;
1085             if (r->which == Z_External_OPAC)
1086             {
1087                 WRBUF w = wrbuf_alloc();
1088
1089                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
1090                 npr->u.databaseRecord = z_ext_record_oid(
1091                     odr_encode(), yaz_oid_recsyn_xml,
1092                     wrbuf_buf(w), wrbuf_len(w));
1093                 wrbuf_destroy(w);
1094             }
1095             else if (r->which == Z_External_octet)
1096             {
1097                 size_t rlen;
1098                 const char *result;
1099                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1100                                         r->u.octet_aligned->len,
1101                                         &result, &rlen))
1102                 {
1103                     npr->u.databaseRecord =
1104                         z_ext_record_oid(odr_encode(), yaz_oid_recsyn_xml,
1105                                          result, rlen);
1106                 }
1107             }
1108         }
1109     }
1110     if (cd)
1111         yaz_iconv_close(cd);
1112     yaz_marc_destroy(mt);
1113 }
1114
1115 void Yaz_Proxy::logtime()
1116 {
1117 #if HAVE_GETTIMEOFDAY
1118     struct timeval *tv = (struct timeval*) m_time_tv;
1119     if (tv->tv_sec)
1120     {
1121         struct timeval tv1;
1122         gettimeofday(&tv1, 0);
1123         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1124             (tv1.tv_usec - tv->tv_usec);
1125         if (diff >= 0)
1126             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1127                     diff/1000000, (diff/1000)%1000);
1128     }
1129     tv->tv_sec = 0;
1130     tv->tv_usec = 0;
1131 #endif
1132 }
1133
1134 int Yaz_Proxy::send_http_response(int code)
1135 {
1136     ODR o = odr_encode();
1137     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1138     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1139     if (m_http_version)
1140         hres->version = odr_strdup(o, m_http_version);
1141     if (m_http_keepalive)
1142         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1143     else
1144         timeout(0);
1145     if (code == 401)
1146         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate",
1147                           "Basic realm=\"YAZ Proxy\"");
1148
1149
1150     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1151     {
1152         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1153                  gdu_name(gdu));
1154     }
1155     int len;
1156     int r = send_GDU(gdu, &len);
1157     m_bytes_sent += len;
1158     m_bw_stat.add_bytes(len);
1159     logtime();
1160
1161     recv_GDU_more(true);
1162
1163     return r;
1164 }
1165
1166 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */)
1167 {
1168     ODR o = odr_encode();
1169     const char *ctype = "text/xml";
1170     Z_GDU *gdu = z_get_HTTP_Response(o, http_code);
1171     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1172     if (m_http_version)
1173         hres->version = odr_strdup(o, m_http_version);
1174     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1175     if (m_http_keepalive)
1176         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1177     else
1178         timeout(0);
1179     if (http_code == 401)
1180         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\"");
1181
1182     static Z_SOAP_Handler soap_handlers[2] = {
1183 #if YAZ_HAVE_XSLT
1184         { (char*) "http://www.loc.gov/zing/srw/", 0,
1185          (Z_SOAP_fun) yaz_srw_codec},
1186 #endif
1187         {0, 0, 0}
1188     };
1189
1190     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1191     soap_package->which = Z_SOAP_generic;
1192     soap_package->u.generic =
1193         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1194     soap_package->u.generic->no = 0;
1195     soap_package->u.generic->ns = soap_handlers[0].ns;
1196     soap_package->u.generic->p = (void *) srw_pdu;
1197     soap_package->ns = m_soap_ns;
1198     z_soap_codec_enc_xsl(o, &soap_package,
1199                          &hres->content_buf, &hres->content_len,
1200                          soap_handlers, 0, m_s2z_stylesheet);
1201     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1202     {
1203         yaz_log(YLOG_LOG, "%sSending %s to client", m_session_str,
1204                 gdu_name(gdu));
1205     }
1206     int len;
1207     int r = send_GDU(gdu, &len);
1208     m_bytes_sent += len;
1209     m_bw_stat.add_bytes(len);
1210     logtime();
1211
1212     recv_GDU_more(true);
1213
1214     return r;
1215 }
1216
1217 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1218 {
1219     ODR o = odr_encode();
1220     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1221         odr_malloc(o, sizeof(*diagnostic));
1222     int num_diagnostic = 1;
1223     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1224     return send_srw_search_response(diagnostic, num_diagnostic, srw_error == 3 ? 401 : 200);
1225 }
1226
1227 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1228                              Z_DefaultDiagFormat *ddf)
1229 {
1230     int bib1_code = *ddf->condition;
1231     if (bib1_code == 109)
1232         return 404;
1233     srw_res->num_diagnostics = 1;
1234     srw_res->diagnostics = (Z_SRW_diagnostic *)
1235         odr_malloc(o, sizeof(*srw_res->diagnostics));
1236     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1237                           yaz_diag_bib1_to_srw(*ddf->condition),
1238                           ddf->u.v2Addinfo);
1239     return 0;
1240 }
1241
1242 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_scanResponse *srw_res,
1243                              Z_DiagRec *dr)
1244 {
1245     if (dr->which == Z_DiagRec_defaultFormat)
1246     {
1247         int bib1_code = *dr->u.defaultFormat->condition;
1248         if (bib1_code == 109)
1249             return 404;
1250         srw_res->num_diagnostics = 1;
1251         srw_res->diagnostics = (Z_SRW_diagnostic *)
1252             odr_malloc(o, sizeof(*srw_res->diagnostics));
1253         yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1254                           yaz_diag_bib1_to_srw(*dr->u.defaultFormat->condition),
1255                           dr->u.defaultFormat->u.v2Addinfo);
1256     }
1257     return 0;
1258 }
1259
1260 #if YAZ_HAS_MK_SURROGATE
1261 #else
1262 static void yazproxy_mk_sru_surrogate(ODR o, Z_SRW_record *record, int pos,
1263                           int code, const char *details)
1264 {
1265     const char *message = yaz_diag_srw_str(code);
1266     int len = 200;
1267     if (message)
1268         len += strlen(message);
1269     if (details)
1270         len += strlen(details);
1271
1272     record->recordData_buf = (char *) odr_malloc(o, len);
1273
1274     sprintf(record->recordData_buf, "<diagnostic "
1275             "xmlns=\"http://www.loc.gov/zing/srw/diagnostic/\">\n"
1276             " <uri>info:srw/diagnostic/1/%d</uri>\n", code);
1277     if (details)
1278         sprintf(record->recordData_buf + strlen(record->recordData_buf),
1279                 " <details>%s</details>\n", details);
1280     if (message)
1281         sprintf(record->recordData_buf + strlen(record->recordData_buf),
1282                 " <message>%s</message>\n", message);
1283     sprintf(record->recordData_buf + strlen(record->recordData_buf),
1284             "</diagnostic>\n");
1285     record->recordData_len = strlen(record->recordData_buf);
1286     record->recordPosition = odr_intdup(o, pos);
1287     record->recordSchema = odr_strdup(o, "info:srw/schema/1/diagnostics-v1.1");
1288 }
1289 #endif
1290
1291 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1292 {
1293     ODR o = odr_encode();
1294     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1295     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1296
1297     srw_res->numberOfRecords = odr_intdup (o, hits);
1298     if (records && records->which == Z_Records_DBOSD)
1299     {
1300         srw_res->num_records =
1301             records->u.databaseOrSurDiagnostics->num_records;
1302         int i;
1303         srw_res->records = (Z_SRW_record *)
1304             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1305         for (i = 0; i < srw_res->num_records; i++)
1306         {
1307             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1308             if (npr->which != Z_NamePlusRecord_databaseRecord)
1309             {
1310 #if YAZ_HAS_MK_SURROGATE
1311                 yaz_mk_sru_surrogate(
1312                     o, srw_res->records + i, i+start,
1313                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1314 #else
1315                 yazproxy_mk_sru_surrogate(
1316                     o, srw_res->records + i, i+start,
1317                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1318 #endif
1319                 continue;
1320             }
1321             Z_External *r = npr->u.databaseRecord;
1322
1323             if (r->which == Z_External_octet
1324                 && !oid_oidcmp(r->direct_reference, yaz_oid_recsyn_xml))
1325             {
1326                 srw_res->records[i].recordSchema = m_schema;
1327                 srw_res->records[i].recordPacking = m_s2z_packing;
1328                 srw_res->records[i].recordData_buf = (char*)
1329                     r->u.octet_aligned->buf;
1330                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1331                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1332             }
1333             else
1334             {
1335 #if YAZ_HAS_MK_SURROGATE
1336                 yaz_mk_sru_surrogate(
1337                     o, srw_res->records + i, i+start,
1338                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1339 #else
1340                 yazproxy_mk_sru_surrogate(
1341                     o, srw_res->records + i, i+start,
1342                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1343 #endif
1344             }
1345         }
1346     }
1347     if (records && records->which == Z_Records_NSD)
1348     {
1349         int http_code;
1350         http_code = z_to_srw_diag(odr_encode(), srw_res,
1351                                    records->u.nonSurrogateDiagnostic);
1352         if (http_code)
1353             return send_http_response(http_code);
1354     }
1355     return send_srw_response(srw_pdu);
1356
1357 }
1358
1359 int Yaz_Proxy::send_to_srw_client_ok(Z_ListEntries *entries)
1360 {
1361     ODR o = odr_encode();
1362     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_scan_response);
1363     Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1364
1365     if (entries && entries->num_entries > 0)
1366     {
1367         srw_res->num_terms = entries->num_entries;
1368         int i;
1369         srw_res->terms = (Z_SRW_scanTerm *)
1370             odr_malloc(o, srw_res->num_terms * sizeof(Z_SRW_scanTerm));
1371         for (i = 0; i < srw_res->num_terms; i++)
1372         {
1373             if (entries->entries[i]->which == Z_Entry_termInfo)
1374             {
1375                 switch(entries->entries[i]->u.termInfo->term->which)
1376                 {
1377                 case Z_Term_general:
1378                     srw_res->terms[i].value = odr_strdupn(o,
1379                         (char *) entries->entries[i]->u.termInfo->term->u.general->buf,
1380                         entries->entries[i]->u.termInfo->term->u.general->len);
1381                     break;
1382                 default:
1383                     srw_res->terms[i].value = NULL;
1384                 }
1385                 if (entries->entries[i]->u.termInfo->globalOccurrences != NULL)
1386                     srw_res->terms[i].numberOfRecords = odr_intdup(o,
1387                         *entries->entries[i]->u.termInfo->globalOccurrences);
1388                 else
1389                     srw_res->terms[i].numberOfRecords = NULL;
1390                 if (entries->entries[i]->u.termInfo->displayTerm != NULL)
1391                     srw_res->terms[i].displayTerm = odr_strdup(o,
1392                         entries->entries[i]->u.termInfo->displayTerm);
1393                 else
1394                     srw_res->terms[i].displayTerm = NULL;
1395                 srw_res->terms[i].whereInList = NULL;
1396             }
1397         }
1398     }
1399     if (entries && entries->num_nonsurrogateDiagnostics > 0)
1400     {
1401         int http_code;
1402         http_code = z_to_srw_diag(odr_encode(), srw_res,
1403                                    entries->nonsurrogateDiagnostics[0]);
1404         if (http_code)
1405             return send_http_response(http_code);
1406     }
1407     return send_srw_response(srw_pdu);
1408
1409 }
1410
1411 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1412                                         int num_diagnostics, int http_code /* = 200 */)
1413 {
1414     ODR o = odr_encode();
1415     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1416     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1417
1418     srw_res->num_diagnostics = num_diagnostics;
1419     srw_res->diagnostics = diagnostics;
1420     return send_srw_response(srw_pdu, http_code);
1421 }
1422
1423 int Yaz_Proxy::send_srw_scan_response(Z_SRW_diagnostic *diagnostics,
1424                                         int num_diagnostics, int http_code /* = 200 */)
1425 {
1426     ODR o = odr_encode();
1427     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_scan_response);
1428     Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1429
1430     srw_res->num_diagnostics = num_diagnostics;
1431     srw_res->diagnostics = diagnostics;
1432     return send_srw_response(srw_pdu, http_code);
1433 }
1434
1435 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1436                                         int num_diagnostics)
1437 {
1438     int http_status = 404;
1439     Yaz_ProxyConfig *cfg = check_reconfigure();
1440     if (cfg)
1441     {
1442         int len;
1443         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1444                                        m_s2z_database, &len, &http_status);
1445         if (b)
1446         {
1447             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1448             Z_SRW_explainResponse *er = res->u.explain_response;
1449
1450             er->record.recordData_buf = b;
1451             er->record.recordData_len = len;
1452             er->record.recordPacking = m_s2z_packing;
1453             er->record.recordSchema = odr_strdup(odr_encode(),
1454                                                  "http://explain.z3950.org/dtd/2.0/");
1455
1456             er->diagnostics = diagnostics;
1457             er->num_diagnostics = num_diagnostics;
1458             return send_srw_response(res);
1459         }
1460     }
1461     return send_http_response(http_status);
1462 }
1463
1464 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1465 {
1466     if (m_http_version)
1467     {
1468         if (apdu->which == Z_APDU_initResponse)
1469         {
1470             Z_InitResponse *res = apdu->u.initResponse;
1471             if (*res->result == 0)
1472             {
1473                 send_to_srw_client_error(3, 0);
1474             }
1475             else if (m_s2z_search_apdu)
1476             {
1477                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1478             }
1479             else if (m_s2z_scan_apdu)
1480             {
1481                 handle_incoming_Z_PDU(m_s2z_scan_apdu);
1482             }
1483             else
1484             {
1485                 send_srw_explain_response(0, 0);
1486             }
1487         }
1488         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1489         {
1490             m_s2z_search_apdu = 0;
1491             Z_SearchResponse *res = apdu->u.searchResponse;
1492             m_s2z_hit_count = *res->resultCount;
1493             if (res->records && res->records->which == Z_Records_NSD)
1494             {
1495                 send_to_srw_client_ok(0, res->records, 1);
1496             }
1497             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1498             {
1499                 // adjust
1500                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1501
1502                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1503                 {
1504                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1505                         > m_s2z_hit_count)
1506                         *pr->numberOfRecordsRequested =
1507                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1508                 }
1509                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1510             }
1511             else
1512             {
1513                 m_s2z_present_apdu = 0;
1514                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1515             }
1516         }
1517         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1518         {
1519             int start =
1520                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1521
1522             m_s2z_present_apdu = 0;
1523             Z_PresentResponse *res = apdu->u.presentResponse;
1524             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1525         }
1526         else if (m_s2z_scan_apdu && apdu->which == Z_APDU_scanResponse)
1527         {
1528             Z_ScanResponse *res = apdu->u.scanResponse;
1529             send_to_srw_client_ok(res->entries);
1530         }
1531     }
1532     else
1533     {
1534         int len = 0;
1535         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1536             yaz_log(YLOG_LOG, "%sSending %s to client", m_session_str,
1537                      apdu_name(apdu));
1538         int r = send_Z_PDU(apdu, &len);
1539         m_bytes_sent += len;
1540         m_bw_stat.add_bytes(len);
1541         logtime();
1542         return r;
1543     }
1544     return 0;
1545 }
1546
1547 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1548 {
1549     int kill_session = 0;
1550     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1551
1552     if (new_id)
1553         *new_id = m_referenceId;
1554
1555     if (apdu->which == Z_APDU_searchResponse)
1556     {
1557         Z_SearchResponse *sr = apdu->u.searchResponse;
1558         Z_Records *p = sr->records;
1559         if (p && p->which == Z_Records_NSD)
1560         {
1561             Z_DiagRec dr, *dr_p = &dr;
1562             dr.which = Z_DiagRec_defaultFormat;
1563             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1564
1565             *sr->searchStatus = 0;
1566             display_diagrecs(&dr_p, 1);
1567         }
1568         else
1569         {
1570             if (p && p->which == Z_Records_DBOSD)
1571             {
1572                 if (m_backend_type
1573 #if HAVE_USEMARCON
1574                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1575 #endif
1576                     )
1577                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1578                 if (m_marcxml_mode == marcxml)
1579                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1580                                        m_backend_charset);
1581                 else
1582                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1583                                             m_backend_charset);
1584                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1585                     return 0;
1586
1587             }
1588             if (sr->resultCount)
1589             {
1590                 yaz_log(YLOG_LOG, "%s" ODR_INT_PRINTF " hits", m_session_str,
1591                         *sr->resultCount);
1592                 if (*sr->resultCount < 0)
1593                 {
1594                     m_flag_invalid_session = 1;
1595                     kill_session = 1;
1596
1597                     *sr->searchStatus = 0;
1598                     sr->records =
1599                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1600                     *sr->resultCount = 0;
1601                 }
1602             }
1603         }
1604     }
1605     else if (apdu->which == Z_APDU_presentResponse)
1606     {
1607         Z_PresentResponse *sr = apdu->u.presentResponse;
1608         Z_Records *p = sr->records;
1609         if (p && p->which == Z_Records_NSD)
1610         {
1611             Z_DiagRec dr, *dr_p = &dr;
1612             dr.which = Z_DiagRec_defaultFormat;
1613             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1614             if (*sr->presentStatus == Z_PresentStatus_success)
1615                 *sr->presentStatus = Z_PresentStatus_failure;
1616             display_diagrecs(&dr_p, 1);
1617         }
1618         if (p && p->which == Z_Records_DBOSD)
1619         {
1620             if (m_backend_type
1621 #if HAVE_USEMARCON
1622                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1623 #endif
1624                 )
1625                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1626             if (m_marcxml_mode == marcxml)
1627                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1628                                    m_backend_charset);
1629             else
1630                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1631                                         m_backend_charset);
1632             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1633                 return 0;
1634         }
1635     }
1636     else if (apdu->which == Z_APDU_initResponse)
1637     {
1638         //Get and check negotiation record
1639         //from init response.
1640         handle_charset_lang_negotiation(apdu);
1641
1642         if (m_initRequest_options)
1643         {
1644             Z_Options *nopt =
1645                 (Odr_bitmask *)odr_malloc(odr_encode(),
1646                                           sizeof(Odr_bitmask));
1647             ODR_MASK_ZERO(nopt);
1648
1649             int i;
1650             for (i = 0; i<24; i++)
1651                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1652                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1653                     ODR_MASK_SET(nopt, i);
1654             apdu->u.initResponse->options = nopt;
1655         }
1656         if (m_initRequest_version)
1657         {
1658             Z_ProtocolVersion *nopt =
1659                 (Odr_bitmask *)odr_malloc(odr_encode(),
1660                                           sizeof(Odr_bitmask));
1661             ODR_MASK_ZERO(nopt);
1662
1663             int i;
1664             for (i = 0; i<8; i++)
1665                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1666                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1667                     ODR_MASK_SET(nopt, i);
1668             apdu->u.initResponse->protocolVersion = nopt;
1669         }
1670         apdu->u.initResponse->preferredMessageSize =
1671             odr_intdup(odr_encode(),
1672                        m_client->m_initResponse_preferredMessageSize >
1673                        m_initRequest_preferredMessageSize ?
1674                        m_initRequest_preferredMessageSize :
1675                        m_client->m_initResponse_preferredMessageSize);
1676         apdu->u.initResponse->maximumRecordSize =
1677             odr_intdup(odr_encode(),
1678                        m_client->m_initResponse_maximumRecordSize >
1679                        m_initRequest_maximumRecordSize ?
1680                        m_initRequest_maximumRecordSize :
1681                        m_client->m_initResponse_maximumRecordSize);
1682     }
1683
1684     int r = send_PDU_convert(apdu);
1685     if (r)
1686         return r;
1687     if (kill_session)
1688     {
1689         delete m_client;
1690         m_client = 0;
1691         m_parent->pre_init();
1692     }
1693     return r;
1694 }
1695
1696 void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu)
1697 {
1698     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1699
1700     odr_reset(m_idAuthentication_odr);
1701     z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0);
1702     m_idAuthentication_ber_buf =
1703         odr_getbuf(m_idAuthentication_odr,
1704                    &m_idAuthentication_ber_size, 0);
1705 }
1706
1707 bool Yaz_ProxyClient::compare_charset(Z_APDU *apdu)
1708 {
1709     return true;
1710 }
1711
1712 bool Yaz_ProxyClient::compare_idAuthentication(Z_APDU *apdu)
1713 {
1714     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1715     ODR odr = odr_createmem(ODR_ENCODE);
1716
1717     z_IdAuthentication(odr, &t, 1, 0);
1718     int sz;
1719     char *buf = odr_getbuf(odr, &sz, 0);
1720     if (buf && m_idAuthentication_ber_buf
1721         && sz == m_idAuthentication_ber_size
1722         && !memcmp(m_idAuthentication_ber_buf, buf, sz))
1723     {
1724         odr_destroy(odr);
1725         return true;
1726     }
1727     odr_destroy(odr);
1728     if (!buf && !m_idAuthentication_ber_buf)
1729         return true;
1730     return false;
1731 }
1732
1733 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1734 {
1735     int len = 0;
1736     const char *apdu_name_tmp = apdu_name(apdu);
1737     int r = send_Z_PDU(apdu, &len);
1738     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1739         yaz_log(YLOG_LOG, "%sSending %s to %s %d bytes",
1740                  get_session_str(),
1741                  apdu_name_tmp, get_hostname(), len);
1742     m_bytes_sent += len;
1743     return r;
1744 }
1745
1746 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1747 {
1748     if (apdu->which == Z_APDU_presentRequest)
1749     {
1750         Z_PresentRequest *pr = apdu->u.presentRequest;
1751         int toget = *pr->numberOfRecordsRequested;
1752         int start = *pr->resultSetStartPoint;
1753
1754         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1755                 pr->resultSetId, start, toget);
1756
1757         if (*m_parent->m_optimize == '0')
1758             return apdu;
1759
1760         if (!m_client->m_last_resultSetId)
1761         {
1762             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1763             new_apdu->u.presentResponse->records =
1764                 create_nonSurrogateDiagnostics(
1765                     odr_encode(),
1766                     YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
1767                     pr->resultSetId);
1768             send_to_client(new_apdu);
1769             return 0;
1770         }
1771         if (start < 1 || toget < 0)
1772         {
1773             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1774             new_apdu->u.presentResponse->records =
1775                 create_nonSurrogateDiagnostics(
1776                     odr_encode(),
1777                     YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
1778                     0);
1779             send_to_client(new_apdu);
1780             return 0;
1781         }
1782         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1783         {
1784             if (start+toget-1 > m_client->m_last_resultCount)
1785             {
1786                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1787                 new_apdu->u.presentResponse->records =
1788                     create_nonSurrogateDiagnostics(
1789                         odr_encode(),
1790                         YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
1791                         0);
1792                 send_to_client(new_apdu);
1793                 return 0;
1794             }
1795             Z_NamePlusRecordList *npr;
1796 #if 0
1797             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1798                     m_session_str, start, toget, yaz_z3950oid_to_str(
1799                         pr->preferredRecordSyntax, &oclass));
1800 #endif
1801             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1802                                           pr->preferredRecordSyntax,
1803                                           pr->recordComposition))
1804             {
1805                 yaz_log(YLOG_LOG, "%sReturned cached records for present request",
1806                          m_session_str);
1807                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1808                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1809
1810                 new_apdu->u.presentResponse->numberOfRecordsReturned
1811                     = odr_intdup(odr_encode(), toget);
1812
1813                 new_apdu->u.presentResponse->records = (Z_Records*)
1814                     odr_malloc(odr_encode(), sizeof(Z_Records));
1815                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1816                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1817                 new_apdu->u.presentResponse->nextResultSetPosition =
1818                     odr_intdup(odr_encode(), start+toget);
1819
1820                 send_to_client(new_apdu);
1821                 return 0;
1822             }
1823         }
1824     }
1825
1826     if (apdu->which != Z_APDU_searchRequest)
1827         return apdu;
1828     Z_SearchRequest *sr = apdu->u.searchRequest;
1829     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1830     Yaz_Z_Databases this_databases;
1831
1832     this_databases.set(sr->num_databaseNames, (const char **)
1833                        sr->databaseNames);
1834
1835     this_query->set_Z_Query(sr->query);
1836
1837     // Check for non-negative piggyback params.
1838     if (*sr->smallSetUpperBound < 0
1839         || *sr->largeSetLowerBound < 0
1840         || *sr->mediumSetPresentNumber < 0)
1841     {
1842         Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1843         // Not a present request.. But can't find better diagnostic
1844         new_apdu->u.searchResponse->records =
1845             create_nonSurrogateDiagnostics(
1846                 odr_encode(),
1847                 YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, 0);
1848         send_to_client(new_apdu);
1849         return 0;
1850     }
1851
1852     char query_str[4096];
1853     this_query->print(query_str, sizeof(query_str)-1);
1854     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1855
1856     if (*m_parent->m_optimize != '0' &&
1857         m_client->m_last_ok && m_client->m_last_query &&
1858         m_client->m_last_query->match(this_query) &&
1859         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1860         m_client->m_last_databases.match(this_databases))
1861     {
1862         delete this_query;
1863         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1864             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1865         {
1866             Z_NamePlusRecordList *npr;
1867             int toget = *sr->mediumSetPresentNumber;
1868             Z_RecordComposition *comp = 0;
1869
1870             if (toget > m_client->m_last_resultCount)
1871                 toget = m_client->m_last_resultCount;
1872
1873             if (sr->mediumSetElementSetNames)
1874             {
1875                 comp = (Z_RecordComposition *)
1876                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1877                 comp->which = Z_RecordComp_simple;
1878                 comp->u.simple = sr->mediumSetElementSetNames;
1879             }
1880
1881             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1882                                           sr->preferredRecordSyntax, comp))
1883             {
1884                 yaz_log(YLOG_LOG, "%sReturned cached records for medium set",
1885                          m_session_str);
1886                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1887                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1888                 new_apdu->u.searchResponse->resultCount =
1889                     &m_client->m_last_resultCount;
1890
1891                 new_apdu->u.searchResponse->numberOfRecordsReturned
1892                     = odr_intdup(odr_encode(), toget);
1893
1894                 new_apdu->u.searchResponse->presentStatus =
1895                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1896                 new_apdu->u.searchResponse->records = (Z_Records*)
1897                     odr_malloc(odr_encode(), sizeof(Z_Records));
1898                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1899                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1900                 new_apdu->u.searchResponse->nextResultSetPosition =
1901                     odr_intdup(odr_encode(), toget+1);
1902                 send_to_client(new_apdu);
1903                 return 0;
1904             }
1905             else
1906             {
1907                 // medium Set
1908                 // send present request (medium size)
1909                 yaz_log(YLOG_LOG, "%sOptimizing search for medium set",
1910                          m_session_str);
1911
1912                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1913                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1914                 pr->referenceId = sr->referenceId;
1915                 pr->resultSetId = sr->resultSetName;
1916                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1917                 *pr->numberOfRecordsRequested = toget;
1918                 pr->recordComposition = comp;
1919                 m_client->m_sr_transform = 1;
1920                 return new_apdu;
1921             }
1922         }
1923         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1924             m_client->m_last_resultCount <= 0)
1925         {
1926             // large set. Return pseudo-search response immediately
1927             yaz_log(YLOG_LOG, "%sOptimizing search for large set",
1928                      m_session_str);
1929             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1930             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1931             new_apdu->u.searchResponse->resultCount =
1932                 &m_client->m_last_resultCount;
1933             send_to_client(new_apdu);
1934             return 0;
1935         }
1936         else
1937         {
1938             Z_NamePlusRecordList *npr;
1939             int toget = m_client->m_last_resultCount;
1940             Z_RecordComposition *comp = 0;
1941             // small set
1942             // send a present request (small set)
1943
1944             if (sr->smallSetElementSetNames)
1945             {
1946                 comp = (Z_RecordComposition *)
1947                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1948                 comp->which = Z_RecordComp_simple;
1949                 comp->u.simple = sr->smallSetElementSetNames;
1950             }
1951
1952             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1953                                           sr->preferredRecordSyntax, comp))
1954             {
1955                 yaz_log(YLOG_LOG, "%sReturned cached records for small set",
1956                          m_session_str);
1957                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1958                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1959                 new_apdu->u.searchResponse->resultCount =
1960                     &m_client->m_last_resultCount;
1961
1962                 new_apdu->u.searchResponse->numberOfRecordsReturned
1963                     = odr_intdup(odr_encode(), toget);
1964
1965                 new_apdu->u.searchResponse->presentStatus =
1966                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1967                 new_apdu->u.searchResponse->records = (Z_Records*)
1968                     odr_malloc(odr_encode(), sizeof(Z_Records));
1969                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1970                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1971                 new_apdu->u.searchResponse->nextResultSetPosition =
1972                     odr_intdup(odr_encode(), toget+1);
1973                 send_to_client(new_apdu);
1974                 return 0;
1975             }
1976             else
1977             {
1978                 yaz_log(YLOG_LOG, "%sOptimizing search for small set",
1979                          m_session_str);
1980                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1981                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1982                 pr->referenceId = sr->referenceId;
1983                 pr->resultSetId = sr->resultSetName;
1984                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1985                 *pr->numberOfRecordsRequested = toget;
1986                 pr->recordComposition = comp;
1987                 m_client->m_sr_transform = 1;
1988                 return new_apdu;
1989             }
1990         }
1991     }
1992     else  // query doesn't match
1993     {
1994         delete m_client->m_last_query;
1995         m_client->m_last_query = this_query;
1996         m_client->m_last_ok = 0;
1997         m_client->m_cache.clear();
1998         m_client->m_resultSetStartPoint = 0;
1999
2000         xfree(m_client->m_last_resultSetId);
2001         m_client->m_last_resultSetId = xstrdup(sr->resultSetName);
2002
2003         m_client->m_last_databases.set(sr->num_databaseNames,
2004                                        (const char **) sr->databaseNames);
2005     }
2006     return apdu;
2007 }
2008
2009
2010 void Yaz_Proxy::inc_request_no()
2011 {
2012     m_request_no++;
2013     char *cp = m_session_str + strlen(m_session_str)-1;
2014     if (*cp == ' ')
2015         cp--;
2016     while (*cp && *cp != ' ')
2017         cp--;
2018     if (*cp)
2019         sprintf(cp+1, "%d ", m_request_no);
2020 }
2021
2022 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
2023 {
2024     inc_request_no();
2025
2026     m_bytes_recv += len;
2027
2028     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2029         yaz_log(YLOG_LOG, "%sReceiving %s from client %d bytes",
2030                  m_session_str, gdu_name(apdu), len);
2031
2032 #if 0
2033     // try to make a _bad_ attribute set ID .. Don't enable this in prod.
2034     if (apdu->which == Z_GDU_Z3950
2035         && apdu->u.z3950->which == Z_APDU_searchRequest)
2036     {
2037         Z_SearchRequest *req = apdu->u.z3950->u.searchRequest;
2038         if (req->query && req->query->which == Z_Query_type_1)
2039         {
2040             Z_RPNQuery *rpnquery = req->query->u.type_1;
2041             if (rpnquery->attributeSetId)
2042             {
2043                 rpnquery->attributeSetId[0] = -2;
2044                 rpnquery->attributeSetId[1] = -1;
2045                 yaz_log(YLOG_WARN, "%sBAD FIXUP TEST", m_session_str);
2046             }
2047         }
2048     }
2049 #endif
2050
2051 #if HAVE_GETTIMEOFDAY
2052     gettimeofday((struct timeval *) m_time_tv, 0);
2053 #endif
2054     m_bw_stat.add_bytes(len);
2055     m_pdu_stat.add_bytes(1);
2056
2057     GDU *gdu = new GDU(apdu);
2058
2059     if (gdu->get() == 0)
2060     {
2061         delete gdu;
2062         yaz_log(YLOG_LOG, "%sUnable to encode package", m_session_str);
2063         m_in_queue.clear();
2064         dec_ref();
2065         return;
2066     }
2067     m_in_queue.enqueue(gdu);
2068     recv_GDU_more(false);
2069 }
2070
2071 void Yaz_Proxy::HTTP_Forwarded(Z_GDU *z_gdu)
2072 {
2073     if (z_gdu->which == Z_GDU_HTTP_Request)
2074     {
2075         Z_HTTP_Request *hreq = z_gdu->u.HTTP_Request;
2076         const char *x_forwarded_for =
2077             z_HTTP_header_lookup(hreq->headers, "X-Forwarded-For");
2078         if (x_forwarded_for)
2079         {
2080             xfree(m_peername);
2081             m_peername = (char*) xmalloc(strlen(x_forwarded_for)+5);
2082             sprintf(m_peername, "tcp:%s", x_forwarded_for);
2083
2084             yaz_log(YLOG_LOG, "%sHTTP Forwarded from %s", m_session_str,
2085                     m_peername);
2086             if (m_log_mask & PROXY_LOG_IP_CLIENT)
2087                 sprintf(m_session_str, "%ld:%d %.80s %d ",
2088                         (long) time(0), m_session_no, m_peername, m_request_no);
2089             else
2090                 sprintf(m_session_str, "%ld:%d %d ",
2091                         (long) time(0), m_session_no, m_request_no);
2092         }
2093     }
2094 }
2095
2096 void Yaz_Proxy::connect_stat(bool &block, int &reduce)
2097 {
2098
2099     m_parent->m_connect.cleanup(false);
2100     m_parent->m_connect.add_connect(m_peername);
2101
2102     int connect_total = m_parent->m_connect.get_total(m_peername);
2103     int max_connect = m_parent->m_max_connect;
2104
2105     if (max_connect && connect_total > max_connect)
2106     {
2107         yaz_log(YLOG_LOG, "%sconnect not accepted total=%d max=%d",
2108                 m_session_str, connect_total, max_connect);
2109         block = true;
2110     }
2111     else
2112         block = false;
2113     yaz_log(YLOG_LOG, "%sconnect accepted total=%d", m_session_str,
2114             connect_total);
2115
2116     int limit_connect = m_parent->m_limit_connect;
2117     if (limit_connect)
2118         reduce = connect_total / limit_connect;
2119     else
2120         reduce = 0;
2121 }
2122
2123 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
2124 {
2125     HTTP_Forwarded(gdu->get());
2126
2127     int reduce = 0;
2128
2129     if (m_request_no == 1)
2130     {
2131         bool block = false;
2132
2133         connect_stat(block, reduce);
2134
2135         if (block)
2136         {
2137             m_timeout_mode = timeout_busy;
2138             timeout(0);
2139             return;
2140         }
2141     }
2142
2143     int bw_total = m_bw_stat.get_total();
2144     int pdu_total = m_pdu_stat.get_total();
2145     int search_total = m_search_stat.get_total();
2146
2147     assert(m_timeout_mode == timeout_busy);
2148     assert(m_timeout_gdu == 0);
2149
2150     if (m_search_max)
2151         reduce += search_total / m_search_max;
2152     if (m_bw_max)
2153         reduce += (bw_total/m_bw_max);
2154     if (m_pdu_max)
2155     {
2156         if (pdu_total > m_pdu_max)
2157         {
2158             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
2159             reduce = (reduce > nreduce) ? reduce : nreduce;
2160         }
2161     }
2162     m_http_version = 0;
2163
2164 #if 0
2165     /* uncomment to force a big reduce */
2166     m_timeout_mode = timeout_reduce;
2167     m_timeout_gdu = gdu;
2168     timeout(3);       // call us reduce seconds later
2169     return;
2170 #endif
2171     if (reduce)
2172     {
2173         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d search=%d limit-bw=%d limit-pdu=%d limit-search=%d",
2174                 m_session_str, reduce, bw_total, pdu_total, search_total,
2175                 m_bw_max, m_pdu_max, m_search_max);
2176
2177         m_timeout_mode = timeout_reduce;
2178         m_timeout_gdu = gdu;
2179         timeout(reduce);       // call us reduce seconds later
2180     }
2181     else
2182         recv_GDU_normal(gdu);
2183 }
2184
2185 void Yaz_Proxy::recv_GDU_more(bool normal)
2186 {
2187     GDU *g;
2188     if (normal && m_timeout_mode == timeout_busy)
2189         m_timeout_mode = timeout_normal;
2190     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
2191     {
2192         m_timeout_mode = timeout_busy;
2193         inc_ref();
2194         recv_GDU_reduce(g);
2195         if (dec_ref())
2196             break;
2197     }
2198 }
2199
2200 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
2201 {
2202     Z_GDU *apdu = 0;
2203     gdu->move_away_gdu(odr_decode(), &apdu);
2204     delete gdu;
2205
2206     if (apdu->which == Z_GDU_Z3950)
2207         handle_incoming_Z_PDU(apdu->u.z3950);
2208     else if (apdu->which == Z_GDU_HTTP_Request)
2209         handle_incoming_HTTP(apdu->u.HTTP_Request);
2210 }
2211
2212 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
2213 {
2214     if (m_max_record_retrieve)
2215     {
2216         if (apdu->which == Z_APDU_presentRequest)
2217         {
2218             Z_PresentRequest *pr = apdu->u.presentRequest;
2219             if (pr->numberOfRecordsRequested &&
2220                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
2221                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
2222         }
2223     }
2224 }
2225
2226 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
2227 {
2228     if (apdu->which == Z_APDU_initRequest)
2229     {
2230         if (m_initRequest_options &&
2231             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
2232             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
2233         {
2234             // There is no negotiation proposal from
2235             // client's side. OK. The proxy negotiation
2236             // in use, only.
2237             Z_InitRequest *initRequest = apdu->u.initRequest;
2238             Z_OtherInformation **otherInfo;
2239             Z_OtherInformationUnit *oi;
2240             get_otherInfoAPDU(apdu, &otherInfo);
2241             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2242             if (oi)
2243             {
2244                 ODR_MASK_SET(initRequest->options,
2245                     Z_Options_negotiationModel);
2246                 oi->which = Z_OtherInfo_externallyDefinedInfo;
2247                 oi->information.externallyDefinedInfo =
2248                 yaz_set_proposal_charneg(odr_encode(),
2249                     (const char**)&m_proxy_negotiation_charset,
2250                     m_proxy_negotiation_charset ? 1:0,
2251                     (const char**)&m_proxy_negotiation_lang,
2252                     m_proxy_negotiation_lang ? 1:0,
2253                     1);
2254             }
2255         }
2256         else if (m_initRequest_options &&
2257                  ODR_MASK_GET(m_initRequest_options,
2258                               Z_Options_negotiationModel) &&
2259                  m_charset_converter->get_target_query_charset())
2260         {
2261             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
2262                     m_session_str,
2263                     m_charset_converter->get_target_query_charset());
2264             Z_InitRequest *initRequest = apdu->u.initRequest;
2265             Z_CharSetandLanguageNegotiation *negotiation =
2266                 yaz_get_charneg_record (initRequest->otherInfo);
2267             if (negotiation &&
2268                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
2269             {
2270                 NMEM nmem = nmem_create();
2271                 char **charsets = 0;
2272                 int num_charsets = 0;
2273                 char **langs = 0;
2274                 int num_langs = 0;
2275                 int selected = 0;
2276                 yaz_get_proposal_charneg (nmem, negotiation,
2277                                           &charsets, &num_charsets,
2278                                           &langs, &num_langs, &selected);
2279                 int i;
2280                 for (i = 0; i<num_charsets; i++)
2281                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
2282                             charsets[i]);
2283                 for (i = 0; i<num_langs; i++)
2284                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
2285                             langs[i]);
2286
2287                 const char *t_charset =
2288                     m_charset_converter->get_target_query_charset();
2289                 // sweep through charsets and pick the first supported
2290                 // conversion
2291                 for (i = 0; i<num_charsets; i++)
2292                 {
2293                     const char *c_charset = charsets[i];
2294                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
2295                         break;
2296                 }
2297                 if (i != num_charsets)
2298                 {
2299                     // got one .. set up ODR for reverse direction
2300                     const char *c_charset = charsets[i];
2301                     odr_set_charset(odr_encode(), c_charset, t_charset);
2302                     m_charset_converter->set_client_query_charset(c_charset);
2303                     m_charset_converter->set_client_charset_selected(selected);
2304                 }
2305                 nmem_destroy(nmem);
2306                 ODR_MASK_CLEAR(m_initRequest_options,
2307                                Z_Options_negotiationModel);
2308                 yaz_del_charneg_record(&initRequest->otherInfo);
2309             }
2310             else
2311             {
2312                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
2313                         m_session_str);
2314             }
2315         }
2316         else if (m_charset_converter->get_target_query_charset() &&
2317             m_proxy_negotiation_default_charset)
2318         {
2319             m_charset_converter->
2320                 set_client_query_charset(m_proxy_negotiation_default_charset);
2321         }
2322     }
2323     else if (apdu->which == Z_APDU_initResponse)
2324     {
2325         Z_InitResponse *initResponse = apdu->u.initResponse;
2326         Z_OtherInformation **otherInfo;
2327         get_otherInfoAPDU(apdu, &otherInfo);
2328
2329         Z_CharSetandLanguageNegotiation *charneg = 0;
2330
2331         if (otherInfo && *otherInfo &&
2332             ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel)
2333             && (charneg = yaz_get_charneg_record(*otherInfo)))
2334         {
2335             char *charset = 0;
2336             char *lang = 0;
2337             int selected = 0;
2338
2339             yaz_get_response_charneg(m_referenceId_mem, charneg,
2340                 &charset, &lang, &selected);
2341
2342             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
2343                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
2344
2345             if (m_initRequest_options &&
2346                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
2347             {
2348                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
2349                     m_session_str);
2350             }
2351             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2352             {
2353                 // negotiation-charset, negotiation-lang
2354                 // elements of config file in use.
2355
2356                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2357                     m_session_str);
2358
2359                 // clear negotiation option.
2360                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2361
2362                 // Delete negotiation (charneg-3) entry.
2363                 yaz_del_charneg_record(otherInfo);
2364             }
2365         }
2366         else
2367         {
2368             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2369             {
2370                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2371                         m_session_str);
2372             }
2373             else if (m_charset_converter->get_client_query_charset())
2374             {
2375                 Z_OtherInformation **otherInfo;
2376                 Z_OtherInformationUnit *oi;
2377                 get_otherInfoAPDU(apdu, &otherInfo);
2378                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2379                 if (oi)
2380                 {
2381                     ODR_MASK_SET(initResponse->options,
2382                                  Z_Options_negotiationModel);
2383                     if (m_initRequest_options)
2384                         ODR_MASK_SET(m_initRequest_options,
2385                                      Z_Options_negotiationModel);
2386
2387                     oi->which = Z_OtherInfo_externallyDefinedInfo;
2388                     oi->information.externallyDefinedInfo =
2389                         yaz_set_response_charneg(
2390                             odr_encode(),
2391                             m_charset_converter->get_client_query_charset(),
2392                             0 /* no lang */,
2393                             m_charset_converter->get_client_charset_selected());
2394                 }
2395             }
2396         }
2397     }
2398 }
2399
2400 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2401                                                      int error,
2402                                                      const char *addinfo)
2403 {
2404     Z_Records *rec = (Z_Records *)
2405         odr_malloc (odr, sizeof(*rec));
2406     Odr_int *err = (Odr_int *)
2407         odr_malloc (odr, sizeof(*err));
2408     Z_DiagRec *drec = (Z_DiagRec *)
2409         odr_malloc (odr, sizeof(*drec));
2410     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2411         odr_malloc (odr, sizeof(*dr));
2412     *err = error;
2413     rec->which = Z_Records_NSD;
2414     rec->u.nonSurrogateDiagnostic = dr;
2415     dr->diagnosticSetId = odr_oiddup(odr, yaz_oid_diagset_bib_1);
2416     dr->condition = err;
2417     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2418     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2419     return rec;
2420 }
2421
2422 Z_ListEntries *Yaz_Proxy::create_nonSurrogateDiagnostics2(ODR odr,
2423                                                      int error,
2424                                                      const char *addinfo)
2425 {
2426     Z_ListEntries *rec = (Z_ListEntries *)
2427         odr_malloc (odr, sizeof(*rec));
2428     Odr_int *err = (Odr_int *)
2429         odr_malloc (odr, sizeof(*err));
2430     Z_DiagRec *drec = (Z_DiagRec *)
2431         odr_malloc (odr, sizeof(*drec));
2432     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2433         odr_malloc (odr, sizeof(*dr));
2434     *err = error;
2435     drec->which = Z_DiagRec_defaultFormat;
2436     drec->u.defaultFormat = dr;
2437     rec->num_entries = 0;
2438     rec->entries = NULL;
2439     rec->num_nonsurrogateDiagnostics = 1;
2440     rec->nonsurrogateDiagnostics =
2441       (Z_DiagRec **)odr_malloc(odr, sizeof(Z_DiagRec *));
2442     rec->nonsurrogateDiagnostics[0] = drec;
2443     dr->diagnosticSetId = odr_oiddup(odr, yaz_oid_diagset_bib_1);
2444     dr->condition = err;
2445     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2446     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2447     return rec;
2448 }
2449
2450 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2451 {
2452     if (apdu->which == Z_APDU_searchRequest &&
2453         apdu->u.searchRequest->query &&
2454         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2455         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2456     {
2457         Z_RPNQuery *rpnquery = 0;
2458         Z_SearchRequest *sr = apdu->u.searchRequest;
2459         char *addinfo = 0;
2460
2461         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2462                 sr->query->u.type_104->u.cql);
2463
2464         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2465                                           &rpnquery, odr_encode(),
2466                                           &addinfo);
2467         if (r == -3)
2468             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2469         else if (r)
2470         {
2471             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2472             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2473
2474             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2475             new_apdu->u.searchResponse->records =
2476                 create_nonSurrogateDiagnostics(odr_encode(),
2477                                                yaz_diag_srw_to_bib1(r),
2478                                                addinfo);
2479             *new_apdu->u.searchResponse->searchStatus = 0;
2480
2481             send_to_client(new_apdu);
2482
2483             return 0;
2484         }
2485         else
2486         {
2487             sr->query->which = Z_Query_type_1;
2488             sr->query->u.type_1 = rpnquery;
2489         }
2490         return apdu;
2491     }
2492     else if (apdu->which == Z_APDU_scanRequest)
2493     {
2494         Z_RPNQuery *rpnquery = 0;
2495         Z_ScanRequest *sr = apdu->u.scanRequest;
2496         char *addinfo = 0;
2497
2498         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2499                 sr->termListAndStartPoint->term->u.characterString);
2500
2501         int r = m_cql2rpn.query_transform(sr->termListAndStartPoint->term->u.characterString,
2502                                           &rpnquery, odr_encode(),
2503                                           &addinfo);
2504         if (r == -3)
2505             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2506         else if (r)
2507         {
2508             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2509             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_scanResponse);
2510
2511             new_apdu->u.scanResponse->referenceId = sr->referenceId;
2512             new_apdu->u.scanResponse->entries =
2513                 create_nonSurrogateDiagnostics2(odr_encode(),
2514                                                yaz_diag_srw_to_bib1(r),
2515                                                addinfo);
2516             *new_apdu->u.scanResponse->scanStatus = Z_Scan_failure;
2517
2518             send_to_client(new_apdu);
2519
2520             return 0;
2521         }
2522         else
2523         {
2524             sr->attributeSet = rpnquery->attributeSetId;
2525             if (rpnquery->RPNStructure->which == Z_RPNStructure_simple)
2526                 sr->termListAndStartPoint = rpnquery->RPNStructure->u.simple->u.attributesPlusTerm;
2527         }
2528         return apdu;
2529     }
2530     return apdu;
2531 }
2532
2533 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2534 {
2535     if (apdu->which == Z_APDU_searchRequest &&
2536         apdu->u.searchRequest->query)
2537     {
2538         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2539             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2540         {
2541             if (m_http_version)
2542                 m_charset_converter->set_client_query_charset("UTF-8");
2543             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2544             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2545         }
2546     }
2547     else if (apdu->which == Z_APDU_scanRequest &&
2548         apdu->u.scanRequest->termListAndStartPoint)
2549     {
2550         if (apdu->u.scanRequest->termListAndStartPoint->term)
2551             if (m_http_version)
2552                 m_charset_converter->set_client_query_charset("UTF-8");
2553             Z_Term *term = apdu->u.scanRequest->termListAndStartPoint->term;
2554             m_charset_converter->convert_term(term, odr_encode());
2555     }
2556     return apdu;
2557 }
2558
2559
2560 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2561 {
2562     if (apdu->which == Z_APDU_searchRequest)
2563     {
2564         Z_SearchRequest *sr = apdu->u.searchRequest;
2565         int err = 0;
2566         char *addinfo = 0;
2567
2568         Yaz_ProxyConfig *cfg = check_reconfigure();
2569         if (cfg)
2570             err = cfg->check_query(odr_encode(), m_default_target,
2571                                    sr->query, &addinfo);
2572         if (err)
2573         {
2574             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2575
2576             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2577             new_apdu->u.searchResponse->records =
2578                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2579             *new_apdu->u.searchResponse->searchStatus = 0;
2580
2581             send_to_client(new_apdu);
2582
2583             return 0;
2584         }
2585     }
2586     else if (apdu->which == Z_APDU_scanRequest)
2587     {
2588         Z_ScanRequest *sr = apdu->u.scanRequest;
2589         int err = 0;
2590         char *addinfo = 0;
2591
2592 #if 0
2593         Yaz_ProxyConfig *cfg = check_reconfigure();
2594 // Something like this needs to be implemented later:
2595         if (cfg)
2596             err = cfg->check_type_1_attributes(odr_encode(), m_default_target,
2597                                    sr->termListAndStartPoint->attributes, &addinfo);
2598 #endif
2599         if (err)
2600         {
2601             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_scanResponse);
2602
2603             new_apdu->u.scanResponse->referenceId = sr->referenceId;
2604             new_apdu->u.scanResponse->entries =
2605                 create_nonSurrogateDiagnostics2(odr_encode(), err, addinfo);
2606             *new_apdu->u.scanResponse->scanStatus = Z_Scan_failure;
2607
2608             send_to_client(new_apdu);
2609
2610             return 0;
2611         }
2612     }
2613
2614     return apdu;
2615 }
2616
2617 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2618 {
2619     if (apdu->which != Z_APDU_initRequest)
2620         return 1;  // pass if no init request
2621     Z_InitRequest *req = apdu->u.initRequest;
2622
2623     Yaz_ProxyConfig *cfg = check_reconfigure();
2624     if (!cfg)
2625         return 1;  // pass if no config
2626
2627     int ret;
2628     if (req->idAuthentication == 0)
2629     {
2630         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2631                                          m_peername);
2632     }
2633     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2634     {
2635         ret = cfg->client_authentication(
2636             m_default_target,
2637             req->idAuthentication->u.idPass->userId,
2638             req->idAuthentication->u.idPass->groupId,
2639             req->idAuthentication->u.idPass->password,
2640             m_peername);
2641     }
2642     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2643     {
2644         char user[64], pass[64];
2645         *user = '\0';
2646         *pass = '\0';
2647         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2648         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2649                                          m_peername);
2650     }
2651     else
2652         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2653                                          m_peername);
2654     return ret;
2655 }
2656
2657 int Yaz_Proxy::handle_global_authentication(Z_APDU *apdu)
2658 {
2659     if (apdu->which != Z_APDU_initRequest)
2660         return 1;  // pass if no init request
2661     Z_InitRequest *req = apdu->u.initRequest;
2662
2663     Yaz_ProxyConfig *cfg = check_reconfigure();
2664     if (!cfg)
2665         return 1;  // pass if no config
2666
2667     int ret;
2668     if (req->idAuthentication == 0)
2669     {
2670         ret = cfg->global_client_authentication(0, 0, 0,
2671                                                 m_peername);
2672     }
2673     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2674     {
2675         ret = cfg->global_client_authentication(
2676             req->idAuthentication->u.idPass->userId,
2677             req->idAuthentication->u.idPass->groupId,
2678             req->idAuthentication->u.idPass->password,
2679             m_peername);
2680     }
2681     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2682     {
2683         char user[64], pass[64];
2684         *user = '\0';
2685         *pass = '\0';
2686         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2687         ret = cfg->global_client_authentication(user, 0, pass,
2688                                                 m_peername);
2689     }
2690     else
2691         ret = cfg->global_client_authentication(0, 0, 0, m_peername);
2692     return ret;
2693 }
2694
2695 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2696 {
2697     m_marcxml_mode = none;
2698     if (apdu->which == Z_APDU_searchRequest)
2699     {
2700         Z_SearchRequest *sr = apdu->u.searchRequest;
2701         int err = 0;
2702         char *addinfo = 0;
2703         Yaz_ProxyConfig *cfg = check_reconfigure();
2704
2705         Z_RecordComposition rc_temp, *rc = 0;
2706         if (sr->smallSetElementSetNames)
2707         {
2708             rc_temp.which = Z_RecordComp_simple;
2709             rc_temp.u.simple = sr->smallSetElementSetNames;
2710             rc = &rc_temp;
2711         }
2712
2713         if (sr->preferredRecordSyntax)
2714             oid_oidcpy(m_frontend_type, sr->preferredRecordSyntax);
2715         else
2716             m_frontend_type[0] = -1;
2717
2718         char *stylesheet_name = 0;
2719         if (cfg)
2720             err = cfg->check_syntax(odr_encode(),
2721                                     m_default_target,
2722                                     sr->preferredRecordSyntax, rc,
2723                                     &addinfo, &stylesheet_name, &m_schema,
2724                                     &m_backend_type, &m_backend_charset,
2725                                     &m_usemarcon_ini_stage1,
2726                                     &m_usemarcon_ini_stage2,
2727                                     &m_backend_elementset);
2728         if (stylesheet_name)
2729         {
2730             m_parent->low_socket_close();
2731
2732 #if YAZ_HAVE_XSLT
2733             if (m_stylesheet_xsp)
2734                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2735             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2736                                                        stylesheet_name);
2737 #endif
2738             m_stylesheet_offset = 0;
2739             xfree(stylesheet_name);
2740
2741             m_parent->low_socket_open();
2742         }
2743         if (err == -1)
2744         {
2745             sr->smallSetElementSetNames = 0;
2746             sr->mediumSetElementSetNames = 0;
2747             m_marcxml_mode = marcxml;
2748             sr->preferredRecordSyntax =
2749                 yaz_string_to_oid_odr(
2750                     yaz_oid_std(), CLASS_RECSYN,
2751                     m_backend_type ? m_backend_type : "usmarc",
2752                     odr_encode());
2753         }
2754         else if (err)
2755         {
2756             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2757
2758             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2759             new_apdu->u.searchResponse->records =
2760                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2761             *new_apdu->u.searchResponse->searchStatus = 0;
2762
2763             send_to_client(new_apdu);
2764
2765             return 0;
2766         }
2767         else if (m_backend_type)
2768         {
2769             sr->preferredRecordSyntax =
2770                 yaz_string_to_oid_odr(yaz_oid_std(), CLASS_RECSYN,
2771                                       m_backend_type, odr_encode());
2772         }
2773         if (m_backend_elementset)
2774         {
2775             Z_ElementSetNames *esn =
2776                 mk_esn_from_schema(
2777                     odr_encode(),
2778                     *m_backend_elementset ? m_backend_elementset : 0);
2779             sr->smallSetElementSetNames = esn;
2780             sr->mediumSetElementSetNames = esn;
2781         }
2782     }
2783     else if (apdu->which == Z_APDU_presentRequest)
2784     {
2785         Z_PresentRequest *pr = apdu->u.presentRequest;
2786         int err = 0;
2787         char *addinfo = 0;
2788         Yaz_ProxyConfig *cfg = check_reconfigure();
2789
2790         if (pr->preferredRecordSyntax)
2791             oid_oidcpy(m_frontend_type, pr->preferredRecordSyntax);
2792         else
2793             m_frontend_type[0] = -1;
2794
2795         char *stylesheet_name = 0;
2796         if (cfg)
2797             err = cfg->check_syntax(odr_encode(), m_default_target,
2798                                     pr->preferredRecordSyntax,
2799                                     pr->recordComposition,
2800                                     &addinfo, &stylesheet_name, &m_schema,
2801                                     &m_backend_type, &m_backend_charset,
2802                                     &m_usemarcon_ini_stage1,
2803                                     &m_usemarcon_ini_stage2,
2804                                     &m_backend_elementset);
2805         if (stylesheet_name)
2806         {
2807             m_parent->low_socket_close();
2808
2809 #if YAZ_HAVE_XSLT
2810             if (m_stylesheet_xsp)
2811                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2812             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2813                                                        stylesheet_name);
2814 #endif
2815             m_stylesheet_offset = 0;
2816             xfree(stylesheet_name);
2817
2818             m_parent->low_socket_open();
2819         }
2820         if (err == -1)
2821         {
2822             pr->recordComposition = 0;
2823             m_marcxml_mode = marcxml;
2824
2825             pr->preferredRecordSyntax =
2826                 yaz_string_to_oid_odr(
2827                     yaz_oid_std(), CLASS_RECSYN,
2828                     m_backend_type ? m_backend_type : "usmarc",
2829                     odr_encode());
2830         }
2831         else if (err)
2832         {
2833             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2834
2835             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2836             new_apdu->u.presentResponse->records =
2837                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2838             *new_apdu->u.presentResponse->presentStatus =
2839                 Z_PresentStatus_failure;
2840
2841             send_to_client(new_apdu);
2842
2843             return 0;
2844         }
2845         else if (m_backend_type)
2846         {
2847             pr->preferredRecordSyntax =
2848                 yaz_string_to_oid_odr(yaz_oid_std(),
2849                                       CLASS_RECSYN, m_backend_type,
2850                                       odr_encode());
2851         }
2852         if (m_backend_elementset)
2853         {
2854             Z_ElementSetNames *esn =
2855                 mk_esn_from_schema(
2856                     odr_encode(),
2857                     *m_backend_elementset ? m_backend_elementset : 0);
2858             Z_RecordComposition *comp = (Z_RecordComposition *)
2859                 odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
2860             comp->which = Z_RecordComp_simple;
2861             comp->u.simple = esn;
2862             pr->recordComposition = comp;
2863         }
2864     }
2865     return apdu;
2866 }
2867
2868 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2869 {
2870     if (!schema)
2871         return 0;
2872     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2873         odr_malloc(o, sizeof(Z_ElementSetNames));
2874     esn->which = Z_ElementSetNames_generic;
2875     esn->u.generic = odr_strdup(o, schema);
2876     return esn;
2877 }
2878
2879 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2880 {
2881     const char *t = 0;
2882     Yaz_ProxyConfig *cfg = check_reconfigure();
2883     if (cfg)
2884         t = cfg->get_explain_name(db, backend_db);
2885
2886     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2887     {
2888         releaseClient();
2889     }
2890
2891     if (t)
2892     {
2893         xfree(m_default_target);
2894         m_default_target = xstrdup(t);
2895     }
2896 }
2897
2898 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2899 {
2900     struct stat sbuf;
2901     if (strcmp(hreq->method, "GET"))
2902         return 0;
2903     if (hreq->path[0] != '/')
2904         return 0;
2905     const char *cp = hreq->path;
2906     while (*cp)
2907     {
2908         if (*cp == '/' && strchr("/.", cp[1]))
2909             return 0;
2910         cp++;
2911     }
2912
2913     Yaz_ProxyConfig *cfg = check_reconfigure();
2914
2915     if (!cfg->get_file_access_info(hreq->path+1))
2916         return 0;
2917
2918     const char *fname = hreq->path+1;
2919     if (stat(fname, &sbuf))
2920     {
2921         yaz_log(YLOG_LOG|YLOG_ERRNO, "%sstat failed for %s", m_session_str,
2922                 fname);
2923         return 0;
2924     }
2925     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2926     {
2927         yaz_log(YLOG_LOG, "%sNot a regular file %s", m_session_str, fname);
2928         return 0;
2929     }
2930     if (sbuf.st_size > (off_t) 1000000)
2931     {
2932         yaz_log(YLOG_WARN, "%sFile %s too large for transfer", m_session_str,
2933                 fname);
2934         return 0;
2935     }
2936
2937     ODR o = odr_encode();
2938
2939     const char *ctype = cfg->check_mime_type(fname);
2940     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2941     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2942     if (m_http_version)
2943         hres->version = odr_strdup(o, m_http_version);
2944     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2945     if (m_http_keepalive)
2946         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2947     else
2948         timeout(0);
2949
2950     hres->content_len = sbuf.st_size;
2951     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2952     FILE *f = fopen(fname, "rb");
2953     if (f)
2954     {
2955         fread(hres->content_buf, 1, hres->content_len, f);
2956         fclose(f);
2957     }
2958     else
2959     {
2960         return 0;
2961     }
2962     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2963     {
2964         yaz_log(YLOG_LOG, "%sSending file %s to client", m_session_str,
2965                  fname);
2966     }
2967     int len;
2968     send_GDU(gdu, &len);
2969     recv_GDU_more(true);
2970     return 1;
2971 }
2972
2973 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2974 {
2975     if (m_s2z_odr_init)
2976     {
2977         odr_destroy(m_s2z_odr_init);
2978         m_s2z_odr_init = 0;
2979     }
2980     if (m_s2z_odr_search)
2981     {
2982         odr_destroy(m_s2z_odr_search);
2983         m_s2z_odr_search = 0;
2984     }
2985     if (m_s2z_odr_scan)
2986     {
2987         odr_destroy(m_s2z_odr_scan);
2988         m_s2z_odr_scan = 0;
2989     }
2990
2991     m_http_keepalive = 0;
2992     m_http_version = 0;
2993     if (!strcmp(hreq->version, "1.0"))
2994     {
2995         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2996         if (v && !strcmp(v, "Keep-Alive"))
2997             m_http_keepalive = 1;
2998         else
2999             m_http_keepalive = 0;
3000         m_http_version = "1.0";
3001     }
3002     else
3003     {
3004         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
3005         if (v && !strcmp(v, "close"))
3006             m_http_keepalive = 0;
3007         else
3008             m_http_keepalive = 1;
3009         m_http_version = "1.1";
3010     }
3011
3012     const char *a = z_HTTP_header_lookup(hreq->headers, "Authorization");
3013     char authorization_str[255];
3014     *authorization_str = '\0';
3015     if (a && strncasecmp(a, "Basic ", 6) == 0)
3016         base64_decode(a + 6, authorization_str, 254);
3017
3018     Z_SRW_PDU *srw_pdu = 0;
3019     Z_SOAP *soap_package = 0;
3020     char *charset = 0;
3021     Z_SRW_diagnostic *diagnostic = 0;
3022     int num_diagnostic = 0;
3023
3024     yaz_log(YLOG_LOG, "%s%s %s", m_session_str, hreq->method, hreq->path);
3025
3026     if (file_access(hreq))
3027     {
3028         return;
3029     }
3030     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
3031                             &charset) == 0
3032              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
3033                                &charset, &diagnostic, &num_diagnostic) == 0)
3034     {
3035         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
3036         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
3037         m_s2z_odr_scan = odr_createmem(ODR_ENCODE);
3038         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
3039         m_s2z_init_apdu = 0;
3040         m_s2z_search_apdu = 0;
3041         m_s2z_present_apdu = 0;
3042         m_s2z_scan_apdu = 0;
3043
3044         m_s2z_stylesheet = 0;
3045
3046         Z_IdAuthentication *auth = NULL;
3047         if (srw_pdu->username && !srw_pdu->password)
3048         {
3049             yaz_log(YLOG_LOG, "username: %s\n", srw_pdu->username);
3050             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
3051             auth->which = Z_IdAuthentication_open;
3052             auth->u.open = odr_strdup(m_s2z_odr_init, srw_pdu->username);
3053         }
3054         else if (srw_pdu->username && srw_pdu->password)
3055         {
3056             yaz_log(YLOG_LOG, "username/password: %s/%s\n",
3057                     srw_pdu->username, srw_pdu->password);
3058             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
3059             auth->which = Z_IdAuthentication_idPass;
3060             auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
3061             auth->u.idPass->groupId = NULL;
3062             auth->u.idPass->password = odr_strdup(m_s2z_odr_init, srw_pdu->password);
3063             auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, srw_pdu->username);
3064         }
3065         else
3066         {
3067             if (*authorization_str)
3068             {
3069                 yaz_log(YLOG_LOG, "authorization_str present: %s\n", authorization_str);
3070                 auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
3071                 auth->which = Z_IdAuthentication_idPass;
3072                 auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
3073                 auth->u.idPass->groupId = NULL;
3074                 char *p = strchr(authorization_str, ':');
3075                 if (p)
3076                 {
3077                     *p = '\0';
3078                     p++;
3079                     auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
3080                 }
3081                 auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
3082             }
3083         }
3084
3085         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
3086         {
3087
3088             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
3089
3090             const char *backend_db = srw_req->database;
3091             srw_get_client(srw_req->database, &backend_db);
3092
3093             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
3094             // recordXPath unsupported.
3095             if (srw_req->recordXPath)
3096             {
3097                 yaz_add_srw_diagnostic(odr_decode(),
3098                                        &diagnostic, &num_diagnostic,
3099                                        72, 0);
3100             }
3101             // sort unsupported
3102             if (srw_req->sort_type != Z_SRW_sort_type_none)
3103             {
3104                 yaz_add_srw_diagnostic(odr_decode(),
3105                                        &diagnostic, &num_diagnostic,
3106                                        80, 0);
3107             }
3108             // save stylesheet
3109             if (srw_req->stylesheet)
3110                 m_s2z_stylesheet =
3111                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
3112
3113             // set packing for response records ..
3114             if (srw_req->recordPacking &&
3115                 !strcmp(srw_req->recordPacking, "xml"))
3116                 m_s2z_packing = Z_SRW_recordPacking_XML;
3117             else
3118                 m_s2z_packing = Z_SRW_recordPacking_string;
3119
3120             if (num_diagnostic)
3121             {
3122                 Z_SRW_PDU *srw_pdu =
3123                     yaz_srw_get(odr_encode(),
3124                                 Z_SRW_searchRetrieve_response);
3125                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
3126
3127                 srw_res->diagnostics = diagnostic;
3128                 srw_res->num_diagnostics = num_diagnostic;
3129                 send_srw_response(srw_pdu);
3130                 return;
3131             }
3132
3133             // prepare search PDU
3134             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
3135                                           Z_APDU_searchRequest);
3136             Z_SearchRequest *z_searchRequest =
3137                 m_s2z_search_apdu->u.searchRequest;
3138
3139             z_searchRequest->num_databaseNames = 1;
3140             z_searchRequest->databaseNames = (char**)
3141                 odr_malloc(m_s2z_odr_search, sizeof(char *));
3142             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
3143                                                            backend_db);
3144
3145             // query transformation
3146             Z_Query *query = (Z_Query *)
3147                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
3148             z_searchRequest->query = query;
3149
3150             if (
3151 #ifdef Z_SRW_query_type_cql
3152                 srw_req->query_type == Z_SRW_query_type_cql
3153 #else
3154                 !srw_req->queryType || !strcmp(srw_req->queryType, "cql")
3155 #endif
3156                 )
3157             {
3158                 Z_External *ext = (Z_External *)
3159                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
3160                 ext->direct_reference =
3161                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
3162                 ext->indirect_reference = 0;
3163                 ext->descriptor = 0;
3164                 ext->which = Z_External_CQL;
3165 #ifdef Z_SRW_query_type_cql
3166                 ext->u.cql = srw_req->query.cql;
3167 #else
3168                 ext->u.cql = srw_req->query;
3169 #endif
3170
3171                 query->which = Z_Query_type_104;
3172                 query->u.type_104 =  ext;
3173             }
3174             else if (
3175 #ifdef Z_SRW_query_type_pqf
3176                 srw_req->query_type == Z_SRW_query_type_pqf
3177 #else
3178                 srw_req->queryType && !strcmp(srw_req->queryType, "pqf")
3179 #endif
3180                 )
3181             {
3182                 Z_RPNQuery *RPNquery;
3183                 YAZ_PQF_Parser pqf_parser;
3184
3185                 pqf_parser = yaz_pqf_create ();
3186
3187                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
3188 #ifdef Z_SRW_query_type_pqf
3189                                           srw_req->query.pqf
3190 #else
3191                                           srw_req->query
3192 #endif
3193                     );
3194                 if (!RPNquery)
3195                 {
3196                     const char *pqf_msg;
3197                     size_t off;
3198                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
3199                     int ioff = off;
3200                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
3201                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
3202
3203                     send_to_srw_client_error(10, 0);
3204                     return;
3205                 }
3206                 query->which = Z_Query_type_1;
3207                 query->u.type_1 =  RPNquery;
3208
3209                 yaz_pqf_destroy (pqf_parser);
3210             }
3211             else
3212             {
3213                 send_to_srw_client_error(7, "query");
3214                 return;
3215             }
3216
3217             // present
3218             m_s2z_present_apdu = 0;
3219             int max = 0;
3220             if (srw_req->maximumRecords)
3221                 max = *srw_req->maximumRecords;
3222             int start = 1;
3223             if (srw_req->startRecord)
3224                 start = *srw_req->startRecord;
3225             if (max > 0)
3226             {
3227                 // Some backend, such as Voyager doesn't honor piggyback
3228                 // So we use present always (0 &&).
3229                 if (0 && start <= 1)  // Z39.50 piggyback
3230                 {
3231                     *z_searchRequest->smallSetUpperBound = max;
3232                     *z_searchRequest->mediumSetPresentNumber = max;
3233                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
3234
3235                     z_searchRequest->preferredRecordSyntax =
3236                         odr_oiddup(m_s2z_odr_search, yaz_oid_recsyn_xml);
3237
3238                     if (srw_req->recordSchema)
3239                     {
3240                         z_searchRequest->smallSetElementSetNames =
3241                             z_searchRequest->mediumSetElementSetNames =
3242                             mk_esn_from_schema(m_s2z_odr_search,
3243                                                srw_req->recordSchema);
3244                     }
3245                 }
3246                 else   // Z39.50 present
3247                 {
3248                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
3249                                                    Z_APDU_presentRequest);
3250                     Z_PresentRequest *z_presentRequest =
3251                         m_s2z_present_apdu->u.presentRequest;
3252                     *z_presentRequest->resultSetStartPoint = start;
3253                     *z_presentRequest->numberOfRecordsRequested = max;
3254
3255                     z_presentRequest->preferredRecordSyntax =
3256                         odr_oiddup(m_s2z_odr_search, yaz_oid_recsyn_xml);
3257                     if (srw_req->recordSchema)
3258                     {
3259                         z_presentRequest->recordComposition =
3260                             (Z_RecordComposition *)
3261                             odr_malloc(m_s2z_odr_search,
3262                                        sizeof(Z_RecordComposition));
3263                         z_presentRequest->recordComposition->which =
3264                             Z_RecordComp_simple;
3265                         z_presentRequest->recordComposition->u.simple =
3266                             mk_esn_from_schema(m_s2z_odr_search,
3267                                                srw_req->recordSchema);
3268                     }
3269                 }
3270             }
3271             if (!m_client)
3272             {
3273                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
3274                                             Z_APDU_initRequest);
3275
3276                 ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan);
3277                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
3278
3279                 // prevent m_initRequest_apdu memory from being grabbed
3280                 // in Yaz_Proxy::handle_incoming_Z_PDU
3281                 m_initRequest_apdu = m_s2z_init_apdu;
3282                 handle_incoming_Z_PDU(m_s2z_init_apdu);
3283                 return;
3284             }
3285             else
3286             {
3287                 handle_incoming_Z_PDU(m_s2z_search_apdu);
3288                 return;
3289             }
3290         }
3291         else if (srw_pdu->which == Z_SRW_explain_request)
3292         {
3293             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
3294
3295             const char *backend_db = srw_req->database;
3296             srw_get_client(srw_req->database, &backend_db);
3297
3298             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
3299
3300             // save stylesheet
3301             if (srw_req->stylesheet)
3302                 m_s2z_stylesheet =
3303                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
3304
3305             if (srw_req->recordPacking &&
3306                 !strcmp(srw_req->recordPacking, "xml"))
3307                 m_s2z_packing = Z_SRW_recordPacking_XML;
3308             else
3309                 m_s2z_packing = Z_SRW_recordPacking_string;
3310
3311             if (num_diagnostic)
3312             {
3313                 send_srw_explain_response(diagnostic, num_diagnostic);
3314                 return;
3315             }
3316
3317             if (!m_client)
3318             {
3319                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
3320                                             Z_APDU_initRequest);
3321
3322                 ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan);
3323                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
3324
3325                 // prevent m_initRequest_apdu memory from being grabbed
3326                 // in Yaz_Proxy::handle_incoming_Z_PDU
3327                 m_initRequest_apdu = m_s2z_init_apdu;
3328                 handle_incoming_Z_PDU(m_s2z_init_apdu);
3329             }
3330             else
3331                 send_srw_explain_response(0, 0);
3332             return;
3333         }
3334         else if (srw_pdu->which == Z_SRW_scan_request)
3335         {
3336             Z_SRW_scanRequest *srw_req = srw_pdu->u.scan_request;
3337
3338             const char *backend_db = srw_req->database;
3339             srw_get_client(srw_req->database, &backend_db);
3340
3341             m_s2z_database = odr_strdup(m_s2z_odr_init,
3342                                         srw_req->database);
3343             // save stylesheet
3344             if (srw_req->stylesheet)
3345                 m_s2z_stylesheet =
3346                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
3347
3348             // prepare scan PDU
3349             m_s2z_scan_apdu = zget_APDU(m_s2z_odr_scan,
3350                                           Z_APDU_scanRequest);
3351             Z_ScanRequest *z_scanRequest =
3352                 m_s2z_scan_apdu->u.scanRequest;
3353
3354             z_scanRequest->num_databaseNames = 1;
3355             z_scanRequest->databaseNames = (char**)
3356                 odr_malloc(m_s2z_odr_scan, sizeof(char *));
3357             z_scanRequest->databaseNames[0] = odr_strdup(m_s2z_odr_scan,
3358                                                            backend_db);
3359
3360              // query transformation
3361             if (
3362 #ifdef Z_SRW_query_type_cql
3363                 srw_req->query_type == Z_SRW_query_type_cql
3364 #else
3365                 !srw_req->queryType || !strcmp(srw_req->queryType, "cql")
3366 #endif
3367                 )
3368             {
3369                 z_scanRequest->termListAndStartPoint =
3370                     (Z_AttributesPlusTerm *)odr_malloc(m_s2z_odr_scan, sizeof(Z_AttributesPlusTerm));
3371                 z_scanRequest->termListAndStartPoint->attributes = NULL;
3372                 z_scanRequest->termListAndStartPoint->term =
3373                    (Z_Term *)odr_malloc(m_s2z_odr_scan, sizeof(Z_Term));
3374                 z_scanRequest->termListAndStartPoint->term->which =
3375                   Z_Term_characterString;
3376                 z_scanRequest->termListAndStartPoint->term->u.characterString =
3377                   odr_strdup(m_s2z_odr_scan,
3378 #ifdef Z_SRW_query_type_cql
3379                              srw_req->scanClause.cql
3380 #else
3381                              srw_req->scanClause
3382 #endif
3383                       );
3384             }
3385
3386             if (srw_req->responsePosition)
3387                 z_scanRequest->preferredPositionInResponse =
3388                     odr_intdup(m_s2z_odr_scan, *srw_req->responsePosition);
3389             if (srw_req->maximumTerms)
3390                 *z_scanRequest->numberOfTermsRequested = *srw_req->maximumTerms;
3391
3392             if (!m_client)
3393             {
3394                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
3395                                             Z_APDU_initRequest);
3396
3397                 ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan);
3398                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
3399
3400                 // prevent m_initRequest_apdu memory from being grabbed
3401                 // in Yaz_Proxy::handle_incoming_Z_PDU
3402                 m_initRequest_apdu = m_s2z_init_apdu;
3403                 handle_incoming_Z_PDU(m_s2z_init_apdu);
3404                 return;
3405             }
3406             else
3407             {
3408                 handle_incoming_Z_PDU(m_s2z_scan_apdu);
3409                 return;
3410             }
3411         }
3412         else
3413         {
3414             m_s2z_database = 0;
3415
3416             send_to_srw_client_error(4, 0);
3417         }
3418     }
3419     send_http_response(400);
3420 }
3421
3422 void Yaz_Proxy::handle_init(Z_APDU *apdu)
3423 {
3424
3425     Z_OtherInformation **oi;
3426     get_otherInfoAPDU(apdu, &oi);
3427
3428     if (apdu->u.initRequest->implementationId)
3429         yaz_log(YLOG_LOG, "%simplementationId: %s",
3430                 m_session_str, apdu->u.initRequest->implementationId);
3431     if (apdu->u.initRequest->implementationName)
3432         yaz_log(YLOG_LOG, "%simplementationName: %s",
3433                 m_session_str, apdu->u.initRequest->implementationName);
3434     if (apdu->u.initRequest->implementationVersion)
3435         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
3436                 m_session_str, apdu->u.initRequest->implementationVersion);
3437     if (m_initRequest_apdu == 0)
3438     {
3439         if (m_initRequest_mem)
3440             nmem_destroy(m_initRequest_mem);
3441
3442         m_initRequest_apdu = apdu;
3443         m_initRequest_mem = odr_extract_mem(odr_decode());
3444
3445         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
3446             preferredMessageSize;
3447         *apdu->u.initRequest->preferredMessageSize = 64*1024*1024;
3448         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
3449             maximumRecordSize;
3450         *apdu->u.initRequest->maximumRecordSize = 64*1024*1024;
3451
3452         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
3453             yaz_get_charneg_record(*oi);
3454
3455         // Save proposal charsets and langs.
3456         if (ODR_MASK_GET(apdu->u.initRequest->options,
3457                          Z_Options_negotiationModel)
3458             && charSetandLangRecord)
3459         {
3460
3461             yaz_get_proposal_charneg(m_referenceId_mem,
3462                                      charSetandLangRecord,
3463                                      &m_initRequest_oi_negotiation_charsets,
3464                                      &m_initRequest_oi_negotiation_num_charsets,
3465                                      &m_initRequest_oi_negotiation_langs,
3466                                      &m_initRequest_oi_negotiation_num_langs,
3467                                      &m_initRequest_oi_negotiation_selected);
3468
3469             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
3470             {
3471                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
3472                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
3473                         m_initRequest_oi_negotiation_charsets[i]:"none");
3474             }
3475             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
3476             {
3477                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
3478                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
3479                         m_initRequest_oi_negotiation_langs[i]:"none");
3480             }
3481             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
3482                     m_session_str, m_initRequest_oi_negotiation_selected);
3483         }
3484         // save init options for the response..
3485         m_initRequest_options = apdu->u.initRequest->options;
3486
3487         apdu->u.initRequest->options =
3488             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
3489                                        sizeof(Odr_bitmask));
3490         ODR_MASK_ZERO(apdu->u.initRequest->options);
3491         int i;
3492         for (i = 0; i<= 24; i++)
3493             ODR_MASK_SET(apdu->u.initRequest->options, i);
3494         // check negotiation option
3495         if (!ODR_MASK_GET(m_initRequest_options,
3496                           Z_Options_negotiationModel))
3497         {
3498             ODR_MASK_CLEAR(apdu->u.initRequest->options,
3499                            Z_Options_negotiationModel);
3500         }
3501         ODR_MASK_CLEAR(apdu->u.initRequest->options,
3502                        Z_Options_concurrentOperations);
3503         // make new version
3504         m_initRequest_version = apdu->u.initRequest->protocolVersion;
3505         apdu->u.initRequest->protocolVersion =
3506             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
3507                                        sizeof(Odr_bitmask));
3508         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
3509
3510         for (i = 0; i<= 8; i++)
3511             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
3512     }
3513     handle_charset_lang_negotiation(apdu);
3514     if (m_client->m_init_flag)
3515     {
3516         if (handle_init_response_for_invalid_session(apdu))
3517             return;
3518         if (m_client->m_initResponse)
3519         {
3520             Z_APDU *apdu2 = m_client->m_initResponse;
3521             apdu2->u.initResponse->otherInfo = 0;
3522             if (m_client->m_cookie && *m_client->m_cookie)
3523                 set_otherInformationString(apdu2, yaz_oid_userinfo_cookie,
3524                                            1, m_client->m_cookie);
3525             apdu2->u.initResponse->referenceId =
3526                 apdu->u.initRequest->referenceId;
3527             apdu2->u.initResponse->options = m_client->m_initResponse_options;
3528             apdu2->u.initResponse->protocolVersion =
3529                 m_client->m_initResponse_version;
3530
3531             handle_charset_lang_negotiation(apdu2);
3532
3533             if (m_timeout_mode == timeout_busy)
3534                 m_timeout_mode = timeout_normal;
3535             send_to_client(apdu2);
3536             return;
3537         }
3538     }
3539     m_client->m_init_flag = 1;
3540
3541     if (m_num_msg_threads && m_my_thread)
3542     {
3543         Auth_Msg *m = new Auth_Msg;
3544         m->m_proxy = this;
3545         z_APDU(odr_encode(), &apdu, 0, "encode");
3546         char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
3547         m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
3548         memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
3549         odr_reset(odr_encode());
3550
3551         inc_ref();
3552         m_my_thread->put(m);
3553     }
3554     else
3555     {
3556         int ret = handle_authentication(apdu);
3557         result_authentication(apdu, ret);
3558     }
3559 }
3560
3561 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
3562 {
3563     Z_ReferenceId **refid = get_referenceIdP(apdu);
3564     nmem_reset(m_referenceId_mem);
3565     if (refid && *refid)
3566     {
3567         m_referenceId = (Z_ReferenceId *)
3568             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
3569         m_referenceId->len = (*refid)->len;
3570 #if YAZ_VERSIONL < 0x50000
3571         m_referenceId->size = m_referenceId->len;
3572         m_referenceId->buf = (unsigned char *)
3573             nmem_strdupn(m_referenceId_mem, (const char *)
3574                                     (*refid)->buf, (*refid)->len);
3575 #else
3576         m_referenceId->buf =
3577             nmem_strdupn(m_referenceId_mem, (*refid)->buf, (*refid)->len);
3578 #endif
3579     }
3580     else
3581         m_referenceId = 0;
3582
3583     if (!m_client && m_flag_invalid_session)
3584     {
3585         // Got request for a session that is invalid..
3586         m_apdu_invalid_session = apdu; // save package
3587         m_mem_invalid_session = odr_extract_mem(odr_decode());
3588         apdu = m_initRequest_apdu;     // but throw an init to the target
3589     }
3590
3591     if (apdu->which == Z_APDU_searchRequest)
3592         m_search_stat.add_bytes(1);
3593
3594     // Handle global authentication
3595     if (!handle_global_authentication(apdu))
3596     {
3597         if (m_http_version)
3598         {   // HTTP. Send unauthorized
3599             send_http_response(401);
3600             return;
3601         }
3602         else
3603         {
3604             // Z39.50 just shutdown
3605             timeout(0);
3606             return;
3607         }
3608         return;
3609     }
3610
3611     // Determine our client.
3612     Z_OtherInformation **oi;
3613     get_otherInfoAPDU(apdu, &oi);
3614     int http_code = 404;
3615     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi), &http_code);
3616     if (!m_client)
3617     {
3618         if (m_http_version)
3619         {   // HTTP. Send not found
3620             send_http_response(http_code);
3621             return;
3622         }
3623         else
3624         {
3625             // Z39.50 just shutdown
3626             timeout(0);
3627             return;
3628         }
3629     }
3630
3631     m_client->m_server = this;
3632
3633     if (apdu->which == Z_APDU_initRequest)
3634         handle_init(apdu);
3635     else
3636         handle_incoming_Z_PDU_2(apdu);
3637 }
3638
3639 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
3640 {
3641     handle_max_record_retrieve(apdu);
3642
3643     if (apdu)
3644         apdu = handle_syntax_validation(apdu);
3645
3646     if (apdu)
3647         apdu = handle_query_transformation(apdu);
3648
3649     if (apdu)
3650         apdu = handle_target_charset_conversion(apdu);
3651
3652     if (apdu)
3653         apdu = handle_query_validation(apdu);
3654
3655     if (apdu)
3656         apdu = result_set_optimize(apdu);
3657
3658     if (!apdu)
3659     {
3660         m_client->timeout(m_target_idletime);  // mark it active even
3661         recv_GDU_more(true);
3662         // though we didn't use it
3663         return;
3664     }
3665
3666     // delete other info construct completely if 0 elements
3667     Z_OtherInformation **oi;
3668     get_otherInfoAPDU(apdu, &oi);
3669     if (oi && *oi && (*oi)->num_elements == 0)
3670         *oi = 0;
3671
3672     if (apdu->which == Z_APDU_presentRequest &&
3673         m_client->m_resultSetStartPoint == 0)
3674     {
3675         Z_PresentRequest *pr = apdu->u.presentRequest;
3676         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3677         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3678     } else {
3679         m_client->m_resultSetStartPoint = 0;
3680     }
3681     if (m_client->send_to_target(apdu) < 0)
3682     {
3683         m_client->shutdown();
3684     }
3685     else
3686         m_client->m_waiting = 1;
3687 }
3688
3689 void Yaz_Proxy::connectNotify()
3690 {
3691 }
3692
3693 void Yaz_Proxy::releaseClient()
3694 {
3695     xfree(m_proxyTarget);
3696     m_proxyTarget = 0;
3697     m_flag_invalid_session = 0;
3698     // only keep if keep_alive flag is set...
3699     if (m_client &&
3700         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3701         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3702         m_client->m_waiting == 0)
3703     {
3704         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3705                  m_session_str,
3706                  m_client->get_hostname());
3707         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3708                 m_session_str, m_client->m_pdu_recv,
3709                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3710                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3711         assert (m_client->m_waiting != 2);
3712         // Tell client (if any) that no server connection is there..
3713         m_client->m_server = 0;
3714         m_client = 0;
3715     }
3716     else if (m_client)
3717     {
3718         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) close %s",
3719                  m_session_str,
3720                  m_client->get_hostname());
3721         assert (m_client->m_waiting != 2);
3722         delete m_client;
3723         m_client = 0;
3724     }
3725     else if (!m_parent)
3726     {
3727         yaz_log(YLOG_LOG, "%sshutdown (client to proxy) bad state",
3728                  m_session_str);
3729         assert (m_parent);
3730     }
3731     else
3732     {
3733         yaz_log(YLOG_LOG, "%sShutdown (client to proxy)",
3734                  m_session_str);
3735     }
3736     if (m_parent)
3737         m_parent->pre_init();
3738 }
3739
3740 bool Yaz_Proxy::dec_ref()
3741 {
3742     m_http_keepalive = 0;
3743
3744     --m_ref_count;
3745     if (m_ref_count > 0)
3746         return false;
3747
3748     releaseClient();
3749
3750     delete this;
3751     return true;
3752 }
3753
3754 const char *Yaz_ProxyClient::get_session_str()
3755 {
3756     if (!m_server)
3757         return "0 ";
3758     return m_server->get_session_str();
3759 }
3760
3761 void Yaz_ProxyClient::shutdown()
3762 {
3763     yaz_log(YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3764              get_hostname());
3765
3766     if (m_server)
3767     {
3768         m_waiting = 1;   // ensure it's released from Yaz_Proxy::releaseClient
3769         m_server->dec_ref();
3770     }
3771     else
3772         delete this;
3773 }
3774
3775 void Yaz_Proxy::failNotify()
3776 {
3777     inc_request_no();
3778     yaz_log(YLOG_LOG, "%sConnection closed by client", get_session_str());
3779     dec_ref();
3780 }
3781
3782 void Yaz_Proxy::send_response_fail_client(const char *addr)
3783 {
3784     if (m_http_version)
3785     {
3786         Z_SRW_diagnostic *diagnostic = 0;
3787         int num_diagnostic = 0;
3788
3789         yaz_add_srw_diagnostic(odr_encode(),
3790                                &diagnostic, &num_diagnostic,
3791                                YAZ_SRW_SYSTEM_TEMPORARILY_UNAVAILABLE, addr);
3792         if (m_s2z_search_apdu)
3793             send_srw_search_response(diagnostic, num_diagnostic);
3794         else if (m_s2z_scan_apdu)
3795             send_srw_scan_response(diagnostic, num_diagnostic);
3796         else
3797             send_srw_explain_response(diagnostic, num_diagnostic);
3798     }
3799 }
3800
3801 void Yaz_ProxyClient::failNotify()
3802 {
3803     if (m_server)
3804         m_server->inc_request_no();
3805     yaz_log(YLOG_LOG, "%sConnection closed by target %s",
3806              get_session_str(), get_hostname());
3807
3808     if (m_server)
3809         m_server->send_response_fail_client(get_hostname());
3810     shutdown();
3811 }
3812
3813 void Yaz_ProxyClient::connectNotify()
3814 {
3815     const char *s = get_session_str();
3816     const char *h = get_hostname();
3817     yaz_log(YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3818              m_target_idletime);
3819     timeout(m_target_idletime);
3820     if (!m_server)
3821         pre_init_client();
3822 }
3823
3824 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3825                                               *the_PDU_Observable, int fd)
3826 {
3827     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3828 }
3829
3830 Yaz_ProxyClient::~Yaz_ProxyClient()
3831 {
3832     if (m_prev)
3833         *m_prev = m_next;
3834     if (m_next)
3835         m_next->m_prev = m_prev;
3836     m_waiting = 2;     // for debugging purposes only.
3837     odr_destroy(m_init_odr);
3838     odr_destroy(m_idAuthentication_odr);
3839     delete m_last_query;
3840     xfree(m_last_resultSetId);
3841     xfree(m_cookie);
3842 }
3843
3844 void Yaz_ProxyClient::pre_init_client()
3845 {
3846     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3847     Z_InitRequest *req = apdu->u.initRequest;
3848
3849     int i;
3850     for (i = 0; i<= 24; i++)
3851         ODR_MASK_SET(req->options, i);
3852     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3853                    Z_Options_negotiationModel);
3854     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3855                    Z_Options_concurrentOperations);
3856     for (i = 0; i<= 10; i++)
3857         ODR_MASK_SET(req->protocolVersion, i);
3858
3859     if (send_to_target(apdu) < 0)
3860     {
3861         delete this;
3862     }
3863     else
3864     {
3865         m_waiting = 1;
3866         m_init_flag = 1;
3867     }
3868 }
3869
3870 void Yaz_Proxy::pre_init()
3871 {
3872     int i;
3873     const char *name = 0;
3874     const char *zurl_in_use[MAX_ZURL_PLEX];
3875     int limit_bw, limit_pdu, limit_req, limit_search;
3876     int target_idletime, client_idletime;
3877     int max_sockets = m_max_sockets;
3878     int max_clients;
3879     int keepalive_limit_bw, keepalive_limit_pdu;
3880     int pre_init;
3881     const char *cql2rpn = 0;
3882     const char *authentication = 0;
3883     const char *negotiation_charset = 0;
3884     const char *negotiation_lang = 0;
3885
3886     Yaz_ProxyConfig *cfg = check_reconfigure();
3887
3888     zurl_in_use[0] = 0;
3889
3890     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3891         set_APDU_yazlog(1);
3892     else
3893         set_APDU_yazlog(0);
3894
3895     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3896                                           &limit_bw, &limit_pdu, &limit_req,
3897                                           &limit_search,
3898                                           &target_idletime, &client_idletime,
3899                                           &max_sockets,
3900                                           &max_clients,
3901                                           &keepalive_limit_bw,
3902                                           &keepalive_limit_pdu,
3903                                           &pre_init,
3904                                           &cql2rpn,
3905                                           &authentication,
3906                                           &negotiation_charset,
3907                                           &negotiation_lang,
3908                                           0,
3909                                           0) ; i++)
3910     {
3911         if (pre_init)
3912         {
3913             int j;
3914             for (j = 0; zurl_in_use[j]; j++)
3915             {
3916                 Yaz_ProxyClient *c;
3917                 int spare = 0;
3918                 int spare_waiting = 0;
3919                 int in_use = 0;
3920                 int other = 0;
3921                 for (c = m_clientPool; c; c = c->m_next)
3922                 {
3923                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3924                     {
3925                         if (c->m_cookie == 0)
3926                         {
3927                             if (c->m_server == 0)
3928                                 if (c->m_waiting)
3929                                     spare_waiting++;
3930                                 else
3931                                     spare++;
3932                             else
3933                                 in_use++;
3934                         }
3935                         else
3936                             other++;
3937                     }
3938                 }
3939                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3940                         "sparew=%d preinit=%d",m_session_str,
3941                         name, zurl_in_use[j], in_use, other,
3942                         spare, spare_waiting, pre_init);
3943                 if (spare + spare_waiting < pre_init
3944                     && in_use + spare + spare_waiting + other < max_sockets)
3945                 {
3946                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3947                     c->m_next = m_clientPool;
3948                     if (c->m_next)
3949                         c->m_next->m_prev = &c->m_next;
3950                     m_clientPool = c;
3951                     c->m_prev = &m_clientPool;
3952
3953                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3954                         c->set_APDU_yazlog(1);
3955                     else
3956                         c->set_APDU_yazlog(0);
3957
3958                     if (c->client(zurl_in_use[j]))
3959                     {
3960                         timeout(60);
3961                         delete c;
3962                         return;
3963                     }
3964                     c->timeout(30);
3965                     c->m_waiting = 1;
3966                     c->m_target_idletime = target_idletime;
3967                     c->m_seqno = m_seqno++;
3968                 }
3969             }
3970         }
3971     }
3972 }
3973
3974 void Yaz_Proxy::timeoutNotify()
3975 {
3976     if (m_parent)
3977     {
3978         GDU *gdu;
3979         switch(m_timeout_mode)
3980         {
3981         case timeout_normal:
3982         case timeout_busy:
3983             inc_request_no();
3984             m_in_queue.clear();
3985             yaz_log(YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3986             dec_ref();
3987             break;
3988         case timeout_reduce:
3989             timeout(m_client_idletime);
3990             m_timeout_mode = timeout_busy;
3991             gdu = m_timeout_gdu;
3992             m_timeout_gdu = 0;
3993             recv_GDU_normal(gdu);
3994             break;
3995         case timeout_xsl:
3996             assert(m_stylesheet_nprl);
3997             convert_xsl_delay();
3998             recv_GDU_more(true);
3999         }
4000     }
4001     else
4002     {
4003         timeout(600);
4004         pre_init();
4005     }
4006 }
4007
4008 void Yaz_Proxy::markInvalid()
4009 {
4010     m_client = 0;
4011     m_flag_invalid_session = 1;
4012 }
4013
4014 void Yaz_ProxyClient::timeoutNotify()
4015 {
4016     if (m_server)
4017         m_server->inc_request_no();
4018
4019     yaz_log(YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
4020              get_hostname());
4021
4022     if (m_server)
4023         m_server->send_response_fail_client(get_hostname());
4024
4025     Yaz_Proxy *proxy_root = m_root;
4026
4027     shutdown();
4028
4029     proxy_root->pre_init();
4030 }
4031
4032 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
4033                                  Yaz_Proxy *parent) :
4034     Z_Assoc (the_PDU_Observable)
4035 {
4036     m_cookie = 0;
4037     m_next = 0;
4038     m_prev = 0;
4039     m_init_flag = 0;
4040     m_last_query = 0;
4041     m_last_resultSetId = 0;
4042     m_last_resultCount = 0;
4043     m_last_ok = 0;
4044     m_sr_transform = 0;
4045     m_waiting = 0;
4046     m_init_odr = odr_createmem (ODR_DECODE);
4047     m_initResponse = 0;
4048     m_initResponse_options = 0;
4049     m_initResponse_version = 0;
4050     m_initResponse_preferredMessageSize = 0;
4051     m_initResponse_maximumRecordSize = 0;
4052     m_resultSetStartPoint = 0;
4053     m_bytes_sent = m_bytes_recv = 0;
4054     m_pdu_recv = 0;
4055     m_server = 0;
4056     m_seqno = 0;
4057     m_target_idletime = 600;
4058     m_root = parent;
4059     m_idAuthentication_odr = odr_createmem(ODR_ENCODE);
4060     m_idAuthentication_ber_buf = 0;
4061     m_idAuthentication_ber_size = 0;
4062 }
4063
4064 const char *Yaz_Proxy::option(const char *name, const char *value)
4065 {
4066     if (!strcmp (name, "optimize")) {
4067         if (value) {
4068             xfree(m_optimize);
4069             m_optimize = xstrdup(value);
4070         }
4071         return m_optimize;
4072     }
4073     return 0;
4074 }
4075
4076 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
4077 {
4078
4079 }
4080
4081 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
4082 {
4083     if (apdu->which == Z_GDU_Z3950)
4084         recv_Z_PDU(apdu->u.z3950, len);
4085     else if (apdu->which == Z_GDU_HTTP_Response)
4086         recv_HTTP_response(apdu->u.HTTP_Response, len);
4087     else
4088         shutdown();
4089 }
4090
4091 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
4092 {
4093     if (!m_flag_invalid_session)
4094         return 0;
4095     m_flag_invalid_session = 0;
4096     handle_incoming_Z_PDU(m_apdu_invalid_session);
4097     assert (m_mem_invalid_session);
4098     nmem_destroy(m_mem_invalid_session);
4099     m_mem_invalid_session = 0;
4100     return 1;
4101 }
4102
4103 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
4104 {
4105     m_bytes_recv += len;
4106
4107     m_pdu_recv++;
4108     m_waiting = 0;
4109     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
4110         yaz_log(YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
4111                  apdu_name(apdu), get_hostname(), len);
4112     if (apdu->which == Z_APDU_initResponse)
4113     {
4114         if (!m_server)  // if this is a pre init session , check for more
4115             m_root->pre_init();
4116         NMEM nmem = odr_extract_mem (odr_decode());
4117         odr_reset (m_init_odr);
4118         nmem_transfer (m_init_odr->mem, nmem);
4119         m_initResponse = apdu;
4120         m_initResponse_options = apdu->u.initResponse->options;
4121         m_initResponse_version = apdu->u.initResponse->protocolVersion;
4122         m_initResponse_preferredMessageSize =
4123             *apdu->u.initResponse->preferredMessageSize;
4124         m_initResponse_maximumRecordSize =
4125             *apdu->u.initResponse->maximumRecordSize;
4126
4127         Z_InitResponse *ir = apdu->u.initResponse;
4128
4129         // apply YAZ Proxy version
4130         char *imv0 = ir->implementationVersion;
4131         char *imv1 = (char*)
4132             odr_malloc(m_init_odr, 20 + (imv0 ? strlen(imv0) : 0));
4133         *imv1 = '\0';
4134         if (imv0)
4135             strcat(imv1, imv0);
4136 #ifdef VERSION
4137         strcat(imv1, "/" VERSION);
4138 #endif
4139         ir->implementationVersion = imv1;
4140
4141         // apply YAZ Proxy implementation name
4142         char *im0 = ir->implementationName;
4143         char *im1 = (char*)
4144             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
4145         *im1 = '\0';
4146         if (im0)
4147         {
4148             strcat(im1, im0);
4149             strcat(im1, " ");
4150         }
4151         strcat(im1, "(YAZ Proxy)");
4152         ir->implementationName = im1;
4153
4154         nmem_destroy (nmem);
4155
4156         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
4157             return;
4158     }
4159     if (apdu->which == Z_APDU_searchResponse)
4160     {
4161         Z_SearchResponse *sr = apdu->u.searchResponse;
4162         m_last_resultCount = *sr->resultCount;
4163         int status = *sr->searchStatus;
4164         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
4165         {
4166             m_last_ok = 1;
4167
4168             if (sr->records && sr->records->which == Z_Records_DBOSD)
4169             {
4170                 m_cache.add(odr_decode(),
4171                             sr->records->u.databaseOrSurDiagnostics, 1,
4172                             *sr->resultCount);
4173             }
4174         }
4175     }
4176     if (apdu->which == Z_APDU_presentResponse)
4177     {
4178         Z_PresentResponse *pr = apdu->u.presentResponse;
4179         if (m_sr_transform)
4180         {
4181             m_sr_transform = 0;
4182             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
4183             Z_SearchResponse *sr = new_apdu->u.searchResponse;
4184             sr->referenceId = pr->referenceId;
4185             *sr->resultCount = m_last_resultCount;
4186             sr->records = pr->records;
4187             sr->nextResultSetPosition = pr->nextResultSetPosition;
4188             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
4189             apdu = new_apdu;
4190         }
4191         if (pr->records &&
4192             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
4193         {
4194             m_cache.add(odr_decode(),
4195                         pr->records->u.databaseOrSurDiagnostics,
4196                         m_resultSetStartPoint, -1);
4197             m_resultSetStartPoint = 0;
4198         }
4199     }
4200     if (m_cookie)
4201         set_otherInformationString(apdu, yaz_oid_userinfo_cookie, 1, m_cookie);
4202
4203     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
4204
4205     if (server)
4206         server->send_to_client(apdu);
4207     if (apdu->which == Z_APDU_close)
4208         shutdown();
4209     else if (server)
4210         server->recv_GDU_more(true);
4211 }
4212
4213 void Yaz_Proxy::low_socket_close()
4214 {
4215 #if WIN32
4216 #else
4217     int i;
4218     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
4219         if  (m_lo_fd[i] >= 0)
4220             ::close(m_lo_fd[i]);
4221 #endif
4222 }
4223
4224 void Yaz_Proxy::low_socket_open()
4225 {
4226 #if WIN32
4227 #else
4228     int i;
4229     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
4230         m_lo_fd[i] = open("/dev/null", O_RDONLY);
4231 #endif
4232 }
4233
4234 int Yaz_Proxy::server(const char *addr)
4235 {
4236     int r = Z_Assoc::server(addr);
4237     if (!r)
4238     {
4239         yaz_log(YLOG_LOG, "%sStarted proxy "
4240 #ifdef VERSION
4241             VERSION
4242 #endif
4243             " on %s", m_session_str, addr);
4244         timeout(1);
4245     }
4246     return r;
4247 }
4248
4249 void Yaz_Proxy::base64_decode(const char *base64, char *buf, int buf_len)
4250 {
4251     const char *base64_chars =
4252         "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
4253     int len = strlen(base64);
4254     int buf_pos = 0;
4255     int index = 1;
4256
4257     for (int pos = 0; pos <= len; pos++)
4258     {
4259         if (base64[pos] == '=' || buf_pos + 1 >= buf_len)
4260             break;
4261
4262         const char *ch_ptr = strchr(base64_chars, base64[pos]);
4263         if (!ch_ptr)
4264             break;
4265         char ch = (char) (ch_ptr - base64_chars);
4266         switch (index)
4267         {
4268         case 1:
4269             buf[buf_pos] = ch << 2;
4270             break;
4271         case 2:
4272             buf[buf_pos++] += (ch & 0x30) >> 4;
4273             buf[buf_pos] = (ch & 0x0f) << 4;
4274             break;
4275         case 3:
4276             buf[buf_pos++] += (ch & 0x3c) >> 2;
4277             buf[buf_pos] = (ch & 0x03) << 6;
4278             break;
4279         case 4:
4280             buf[buf_pos++] += ch;
4281         }
4282         if (index < 4)
4283             index++;
4284         else
4285             index = 1;
4286     }
4287     buf[buf_pos] = '\0';
4288 }
4289
4290 /*
4291  * Local variables:
4292  * c-basic-offset: 4
4293  * c-file-style: "Stroustrup"
4294  * indent-tabs-mode: nil
4295  * End:
4296  * vim: shiftwidth=4 tabstop=8 expandtab
4297  */
4298