src/yaz-proxy.cpp

   1 /* This file is part of YAZ proxy
   2    Copyright (C) 1998-2011 Index Data
   3
   4 YAZ proxy is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17 */
  18
  19 #ifdef WIN32
  20 #define HAVE_SYS_STAT_H 1
  21 #define HAVE_SYS_TYPES_H 1
  22 #endif
  23
  24 #if HAVE_UNISTD_H
  25 #include <unistd.h>
  26 #endif
  27 #if HAVE_SYS_TIME_H
  28 #include <sys/time.h>
  29 #endif
  30 #if HAVE_SYS_TYPES_H
  31 #include <sys/types.h>
  32 #endif
  33 #if HAVE_SYS_STAT_H
  34 #include <sys/stat.h>
  35 #endif
  36
  37 #include <assert.h>
  38 #include <stdlib.h>
  39 #include <time.h>
  40 #include <fcntl.h>
  41
  42 #include <yaz/srw.h>
  43 #include <yaz/marcdisp.h>
  44 #include <yaz/yaz-iconv.h>
  45 #include <yaz/log.h>
  46 #include <yaz/diagbib1.h>
  47 #include "proxyp.h"
  48 #include <yaz/pquery.h>
  49 #include <yaz/otherinfo.h>
  50 #include <yaz/charneg.h>
  51 #include <yaz/oid_db.h>
  52 #include "msg-thread.h"
  53
  54 #if YAZ_VERSIONL >= 0x03001D
  55 #define YAZ_HAS_MK_SURROGATE 1
  56 #else
  57 #define YAZ_HAS_MK_SURROGATE 0
  58 #endif
  59
  60 using namespace yazpp_1;
  61
  62 #ifdef WIN32
  63 #define strncasecmp _strnicmp
  64 #endif
  65
  66 class YAZ_EXPORT Auth_Msg : public IMsg_Thread {
  67 public:
  68     int m_ret;
  69     IMsg_Thread *handle();
  70     void result();
  71     Yaz_Proxy *m_proxy;
  72     NMEM m_nmem;
  73     char *m_apdu_buf;
  74     int m_apdu_len;
  75     Auth_Msg();
  76     virtual ~Auth_Msg();
  77 };
  78
  79 Auth_Msg::Auth_Msg()
  80 {
  81     m_nmem = nmem_create();
  82 }
  83
  84 Auth_Msg::~Auth_Msg()
  85 {
  86     nmem_destroy(m_nmem);
  87 }
  88
  89 IMsg_Thread *Auth_Msg::handle()
  90 {
  91     ODR decode = odr_createmem(ODR_DECODE);
  92     Z_APDU *apdu;
  93
  94     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
  95     int r = z_APDU(decode, &apdu, 0, 0);
  96     if (!r)
  97     {
  98         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
  99     }
 100     else
 101     {
 102         m_ret = m_proxy->handle_authentication(apdu);
 103     }
 104     odr_destroy(decode);
 105     return this;
 106 }
 107
 108 void Auth_Msg::result()
 109 {
 110     if (m_proxy->dec_ref())
 111     {
 112         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
 113     }
 114     else
 115     {
 116         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
 117         Z_APDU *apdu = 0;
 118         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
 119         if (!r)
 120             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
 121         m_proxy->result_authentication(apdu, m_ret);
 122     }
 123     delete this;
 124 }
 125
 126 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
 127 {
 128     if (apdu == 0 || ret == 0)
 129     {
 130         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
 131         *apdu_reject->u.initResponse->result = 0;
 132         send_to_client(apdu_reject);
 133         dec_ref();
 134     }
 135     else
 136     {
 137         if (apdu->which == Z_APDU_initRequest)
 138         {
 139             Yaz_ProxyConfig *cfg = check_reconfigure();
 140             if (cfg)
 141                 cfg->target_authentication(m_default_target, odr_encode(),
 142                                            apdu->u.initRequest);
 143         }
 144         handle_incoming_Z_PDU_2(apdu);
 145     }
 146 }
 147
 148 static const char *apdu_name(Z_APDU *apdu)
 149 {
 150     switch (apdu->which)
 151     {
 152     case Z_APDU_initRequest:
 153         return "initRequest";
 154     case Z_APDU_initResponse:
 155         return "initResponse";
 156     case Z_APDU_searchRequest:
 157         return "searchRequest";
 158     case Z_APDU_searchResponse:
 159         return "searchResponse";
 160     case Z_APDU_presentRequest:
 161         return "presentRequest";
 162     case Z_APDU_presentResponse:
 163         return "presentResponse";
 164     case Z_APDU_deleteResultSetRequest:
 165         return "deleteResultSetRequest";
 166     case Z_APDU_deleteResultSetResponse:
 167         return "deleteResultSetResponse";
 168     case Z_APDU_scanRequest:
 169         return "scanRequest";
 170     case Z_APDU_scanResponse:
 171         return "scanResponse";
 172     case Z_APDU_sortRequest:
 173         return "sortRequest";
 174     case Z_APDU_sortResponse:
 175         return "sortResponse";
 176     case Z_APDU_extendedServicesRequest:
 177         return "extendedServicesRequest";
 178     case Z_APDU_extendedServicesResponse:
 179         return "extendedServicesResponse";
 180     case Z_APDU_close:
 181         return "close";
 182     }
 183     return "other";
 184 }
 185
 186 static const char *gdu_name(Z_GDU *gdu)
 187 {
 188     switch(gdu->which)
 189     {
 190     case Z_GDU_Z3950:
 191         return apdu_name(gdu->u.z3950);
 192     case Z_GDU_HTTP_Request:
 193         return "HTTP Request";
 194     case Z_GDU_HTTP_Response:
 195         return "HTTP Response";
 196     }
 197     return "Unknown request/response";
 198 }
 199
 200 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
 201                      ISocketObservable *the_socket_observable,
 202                      Yaz_Proxy *parent)
 203     :
 204     Z_Assoc(the_PDU_Observable),
 205     m_bw_stat(60), m_pdu_stat(60), m_search_stat(60)
 206 {
 207     m_PDU_Observable = the_PDU_Observable;
 208     m_socket_observable = the_socket_observable;
 209     m_client = 0;
 210     m_parent = parent;
 211     m_clientPool = 0;
 212     m_seqno = 1;
 213     m_keepalive_limit_bw = 500000;
 214     m_keepalive_limit_pdu = 1000;
 215     m_proxyTarget = 0;
 216     m_default_target = 0;
 217     m_proxy_negotiation_charset = 0;
 218     m_proxy_negotiation_lang = 0;
 219     m_proxy_negotiation_default_charset = 0;
 220     m_charset_converter = new Yaz_CharsetConverter;
 221     m_max_clients = 150;
 222     m_log_mask = 0;
 223     m_seed = time(0);
 224     m_client_idletime = 600;
 225     m_target_idletime = 600;
 226     m_max_sockets = 1024;
 227     m_optimize = xstrdup("1");
 228     strcpy(m_session_str, "0 ");
 229     m_session_no = 0;
 230     m_bytes_sent = 0;
 231     m_bytes_recv = 0;
 232     m_bw_max = 0;
 233     m_pdu_max = 0;
 234     m_search_max = 0;
 235     m_max_connect = 0;
 236     m_max_connect_period = 0;
 237     m_limit_connect = 0;
 238     m_limit_connect_period = 0;
 239     m_timeout_mode = timeout_normal;
 240     m_timeout_gdu = 0;
 241     m_max_record_retrieve = 0;
 242     m_reconfig_flag = 0;
 243     m_config_fname = 0;
 244     m_request_no = 0;
 245     m_flag_invalid_session = 0;
 246     m_referenceId = 0;
 247     m_referenceId_mem = nmem_create();
 248     m_config = 0;
 249     m_marcxml_mode = none;
 250     m_stylesheet_xsp = 0;
 251     m_stylesheet_nprl = 0;
 252     m_stylesheet_apdu = 0;
 253     m_s2z_stylesheet = 0;
 254     m_s2z_database = 0;
 255     m_schema = 0;
 256     m_backend_type = 0;
 257     m_backend_charset = 0;
 258     m_frontend_type[0] = -1;
 259     m_initRequest_apdu = 0;
 260     m_initRequest_mem = 0;
 261     m_initRequest_preferredMessageSize = 0;
 262     m_initRequest_maximumRecordSize = 0;
 263     m_initRequest_options = 0;
 264     m_initRequest_version = 0;
 265     m_initRequest_oi_negotiation_charsets = 0;
 266     m_initRequest_oi_negotiation_num_charsets = 0;
 267     m_initRequest_oi_negotiation_langs = 0;
 268     m_initRequest_oi_negotiation_num_langs = 0;
 269     m_initRequest_oi_negotiation_selected = 0;
 270     m_apdu_invalid_session = 0;
 271     m_mem_invalid_session = 0;
 272     m_s2z_odr_init = 0;
 273     m_s2z_odr_search = 0;
 274     m_s2z_odr_scan = 0;
 275     m_s2z_init_apdu = 0;
 276     m_s2z_search_apdu = 0;
 277     m_s2z_present_apdu = 0;
 278     m_s2z_scan_apdu = 0;
 279     m_http_keepalive = 0;
 280     m_http_version = 0;
 281     m_soap_ns = 0;
 282     m_s2z_packing = Z_SRW_recordPacking_string;
 283 #if HAVE_GETTIMEOFDAY
 284     m_time_tv = xmalloc(sizeof(struct timeval));
 285     struct timeval *tv = (struct timeval *) m_time_tv;
 286     tv->tv_sec = 0;
 287     tv->tv_usec = 0;
 288 #else
 289     m_time_tv = 0;
 290 #endif
 291     m_usemarcon_ini_stage1 = 0;
 292     m_usemarcon_ini_stage2 = 0;
 293     m_usemarcon = new Yaz_usemarcon();
 294     if (!m_parent)
 295         low_socket_open();
 296     m_backend_elementset = 0;
 297     m_my_thread = 0;
 298     m_ref_count = 1;
 299     m_main_ptr_dec = false;
 300     m_peername = 0;
 301     m_num_msg_threads = 0;
 302 }
 303
 304 void Yaz_Proxy::inc_ref()
 305 {
 306     m_ref_count++;
 307 }
 308
 309 Yaz_Proxy::~Yaz_Proxy()
 310 {
 311     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 312             m_bytes_sent, m_bytes_recv);
 313     nmem_destroy(m_initRequest_mem);
 314     nmem_destroy(m_mem_invalid_session);
 315     nmem_destroy(m_referenceId_mem);
 316
 317     xfree(m_proxyTarget);
 318     xfree(m_default_target);
 319     xfree(m_proxy_negotiation_charset);
 320     xfree(m_proxy_negotiation_lang);
 321     xfree(m_proxy_negotiation_default_charset);
 322     delete m_charset_converter;
 323     xfree(m_optimize);
 324
 325 #if YAZ_HAVE_XSLT
 326     if (m_stylesheet_xsp)
 327         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 328 #endif
 329     xfree(m_time_tv);
 330
 331     xfree(m_peername);
 332     xfree(m_schema);
 333     xfree(m_backend_type);
 334     xfree(m_backend_charset);
 335     xfree(m_usemarcon_ini_stage1);
 336     xfree(m_usemarcon_ini_stage2);
 337     xfree(m_backend_elementset);
 338     delete m_usemarcon;
 339     if (m_s2z_odr_init)
 340         odr_destroy(m_s2z_odr_init);
 341     if (m_s2z_odr_search)
 342         odr_destroy(m_s2z_odr_search);
 343     if (m_s2z_odr_scan)
 344         odr_destroy(m_s2z_odr_scan);
 345     if (!m_parent)
 346         low_socket_close();
 347     if (!m_parent)
 348         delete m_my_thread;
 349     delete m_config;
 350 }
 351
 352 void Yaz_Proxy::set_debug_mode(int mode)
 353 {
 354     m_debug_mode = mode;
 355 }
 356
 357 int Yaz_Proxy::set_config(const char *config)
 358 {
 359     delete m_config;
 360     m_config = new Yaz_ProxyConfig();
 361     xfree(m_config_fname);
 362     m_config_fname = xstrdup(config);
 363     int r = m_config->read_xml(config);
 364     if (!r)
 365     {
 366         int period = 60;
 367         m_config->get_generic_info(&m_log_mask, &m_max_clients,
 368                                    &m_max_connect, &m_limit_connect, &period,
 369                                    &m_num_msg_threads);
 370         m_connect.set_period(period);
 371     }
 372     return r;
 373 }
 374
 375 void Yaz_Proxy::set_default_target(const char *target)
 376 {
 377     xfree(m_default_target);
 378     m_default_target = 0;
 379     if (target)
 380         m_default_target = (char *) xstrdup(target);
 381 }
 382
 383 void Yaz_Proxy::set_proxy_negotiation(const char *charset, const char *lang,
 384                                        const char *default_charset)
 385 {
 386     yaz_log(YLOG_DEBUG, "%sSet the proxy negotiation: charset to '%s', "
 387         "default charset to '%s', language to '%s'", m_session_str,
 388         charset?charset:"none",
 389         default_charset?default_charset:"none",
 390         lang?lang:"none");
 391     xfree(m_proxy_negotiation_charset);
 392     xfree(m_proxy_negotiation_lang);
 393     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 394     if (charset)
 395         m_proxy_negotiation_charset = (char *) xstrdup(charset);
 396     if (lang)
 397         m_proxy_negotiation_lang = (char *) xstrdup(lang);
 398     if (default_charset)
 399         m_proxy_negotiation_default_charset =
 400             (char *) xstrdup(default_charset);
 401 }
 402
 403 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 404 {
 405     if (m_parent)
 406         return m_parent->check_reconfigure();
 407
 408     Yaz_ProxyConfig *cfg = m_config;
 409     if (m_reconfig_flag)
 410     {
 411         yaz_log(YLOG_LOG, "reconfigure");
 412         if (m_config_fname && cfg)
 413         {
 414             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 415             int r = cfg->read_xml(m_config_fname);
 416             if (r)
 417                 yaz_log(YLOG_WARN, "reconfigure failed");
 418             else
 419             {
 420                 m_log_mask = 0;
 421                 int period = 60;
 422                 cfg->get_generic_info(&m_log_mask, &m_max_clients,
 423                                       &m_max_connect, &m_limit_connect,
 424                                       &period, &m_num_msg_threads);
 425                 m_connect.set_period(period);
 426             }
 427         }
 428         else
 429             yaz_log(YLOG_LOG, "reconfigure");
 430         m_reconfig_flag = 0;
 431     }
 432     return cfg;
 433 }
 434
 435 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 436                                         *the_PDU_Observable, int fd)
 437 {
 438     check_reconfigure();
 439
 440     char session_str[200];
 441     const char *peername = the_PDU_Observable->getpeername();
 442     if (!peername)
 443         peername = "nullpeer";
 444
 445     if (m_log_mask & PROXY_LOG_IP_CLIENT)
 446         sprintf(session_str, "%ld:%d %.80s %d ",
 447                 (long) time(0), m_session_no, peername, 0);
 448     else
 449         sprintf(session_str, "%ld:%d %d ",
 450                 (long) time(0), m_session_no, 0);
 451     m_session_no++;
 452
 453     yaz_log(YLOG_LOG, "%sNew session %s", session_str, peername);
 454
 455     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 456                                          m_socket_observable, this);
 457
 458     new_proxy->m_config = 0;
 459     new_proxy->m_config_fname = 0;
 460     new_proxy->timeout(m_client_idletime);
 461     new_proxy->m_target_idletime = m_target_idletime;
 462     new_proxy->set_default_target(m_default_target);
 463     new_proxy->m_max_clients = m_max_clients;
 464     new_proxy->m_log_mask = m_log_mask;
 465     new_proxy->m_session_no = m_session_no;
 466     new_proxy->m_num_msg_threads = m_num_msg_threads;
 467
 468 #if 0
 469     // in case we want to watch a particular client..
 470     if (!strcmp(peername, "tcp:163.121.19.82")) // NIS GROUP
 471         new_proxy->m_log_mask = 255;
 472 #endif
 473
 474     new_proxy->set_APDU_log(get_APDU_log());
 475     if (new_proxy->m_log_mask & PROXY_LOG_APDU_CLIENT)
 476         new_proxy->set_APDU_yazlog(1);
 477     else
 478         new_proxy->set_APDU_yazlog(0);
 479     strcpy(new_proxy->m_session_str, session_str);
 480     new_proxy->m_peername = xstrdup(peername);
 481     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 482         m_proxy_negotiation_lang, m_proxy_negotiation_default_charset);
 483     // create thread object the first time we get an incoming connection
 484     if (!m_my_thread && m_num_msg_threads > 0)
 485     {
 486         yaz_log(YLOG_LOG, "%sStarting message thread management. number=%d",
 487                  session_str, m_num_msg_threads);
 488         m_my_thread = new Msg_Thread(m_socket_observable, m_num_msg_threads);
 489     }
 490     new_proxy->m_my_thread = m_my_thread;
 491     return new_proxy;
 492 }
 493
 494 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 495 {
 496     Z_OtherInformationUnit *oi =
 497         update_otherInformation(otherInfo, 0, yaz_oid_userinfo_cookie, 1, 1);
 498
 499     if (oi && oi->which == Z_OtherInfo_characterInfo)
 500         return oi->information.characterInfo;
 501     return 0;
 502 }
 503
 504 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 505 {
 506     Z_OtherInformationUnit *oi =
 507         update_otherInformation(otherInfo, 0, yaz_oid_userinfo_proxy, 1, 1);
 508
 509     if (oi && oi->which == Z_OtherInfo_characterInfo)
 510         return oi->information.characterInfo;
 511     return 0;
 512 }
 513 const char *Yaz_Proxy::load_balance(const char **url)
 514 {
 515     int zurl_in_use[MAX_ZURL_PLEX];
 516     int zurl_in_spare[MAX_ZURL_PLEX];
 517     Yaz_ProxyClient *c;
 518     int i;
 519
 520     for (i = 0; i<MAX_ZURL_PLEX; i++)
 521     {
 522         zurl_in_use[i] = 0;
 523         zurl_in_spare[i] = 0;
 524     }
 525     for (c = m_parent->m_clientPool; c; c = c->m_next)
 526     {
 527         for (i = 0; url[i]; i++)
 528             if (!strcmp(url[i], c->get_hostname()))
 529             {
 530                 zurl_in_use[i]++;
 531                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 532                     zurl_in_spare[i]++;
 533             }
 534     }
 535     int min_use = 100000;
 536     int max_spare = 0;
 537     const char *ret_min = 0;
 538     for (i = 0; url[i]; i++)
 539     {
 540         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 541                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 542         if (min_use > zurl_in_use[i])
 543         {
 544             ret_min = url[i];
 545             min_use = zurl_in_use[i];
 546         }
 547         if (max_spare < zurl_in_spare[i])
 548         {
 549             max_spare = zurl_in_spare[i];
 550         }
 551     }
 552     return ret_min;
 553 }
 554
 555 int Yaz_Proxy::get_number_of_connections()
 556 {
 557     int no_connections = 0;
 558     Yaz_ProxyClient *c;
 559
 560     for (c = m_parent->m_clientPool; c; c = c->m_next)
 561     {
 562         assert(c->m_prev);
 563         assert(*c->m_prev == c);
 564         if (!strcmp(m_proxyTarget, c->get_hostname()))
 565         {
 566             no_connections++;
 567         }
 568     }
 569     yaz_log(YLOG_LOG, "%sExisting %s connections: %d", m_session_str, m_proxyTarget,
 570         no_connections);
 571     return no_connections;
 572 }
 573
 574 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 575                                        const char *proxy_host, int *http_code)
 576 {
 577     assert(m_parent);
 578     Yaz_Proxy *parent = m_parent;
 579     Yaz_ProxyClient *c = m_client;
 580
 581     if (!m_proxyTarget)
 582     {
 583         const char *url[MAX_ZURL_PLEX];
 584         Yaz_ProxyConfig *cfg = check_reconfigure();
 585         if (proxy_host)
 586         {
 587             if (parent && parent->m_debug_mode)
 588             {
 589                 // only to be enabled for debugging...
 590                 if (!strcmp(proxy_host, "stop"))
 591                     exit(0);
 592             }
 593             xfree(m_default_target);
 594             m_default_target = xstrdup(proxy_host);
 595         }
 596         proxy_host = m_default_target;
 597         int client_idletime = -1;
 598         const char *cql2rpn_fname = 0;
 599         const char *negotiation_charset = 0;
 600         const char *negotiation_lang = 0;
 601         const char *query_charset = 0;
 602         const char *default_client_query_charset = 0;
 603         url[0] = m_default_target;
 604         url[1] = 0;
 605         if (cfg)
 606         {
 607             int pre_init = 0;
 608             cfg->get_target_info(proxy_host, url, &m_bw_max,
 609                                  &m_pdu_max, &m_max_record_retrieve,
 610                                  &m_search_max,
 611                                  &m_target_idletime, &client_idletime,
 612                                  &m_max_sockets,
 613                                  &parent->m_max_clients,
 614                                  &m_keepalive_limit_bw,
 615                                  &m_keepalive_limit_pdu,
 616                                  &pre_init,
 617                                  &cql2rpn_fname,
 618                                  &negotiation_charset,
 619                                  &negotiation_lang,
 620                                  &query_charset,
 621                                  &default_client_query_charset);
 622         }
 623         if (client_idletime != -1)
 624         {
 625             m_client_idletime = client_idletime;
 626             timeout(m_client_idletime);
 627         }
 628
 629         // get those FILE descriptors available
 630         m_parent->low_socket_close();
 631         if (cql2rpn_fname)
 632             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 633         // reserve them again
 634         m_parent->low_socket_open();
 635
 636         if (negotiation_charset || negotiation_lang || default_client_query_charset)
 637         {
 638             set_proxy_negotiation(negotiation_charset,
 639                 negotiation_lang, default_client_query_charset);
 640         }
 641         m_charset_converter->set_target_query_charset(query_charset);
 642         if (!url[0])
 643         {
 644             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 645             return 0;
 646         }
 647         // we don't handle multiplexing for cookie session, so we just
 648         // pick the first one in this case (anonymous users will be able
 649         // to use any backend)
 650         if (cookie && *cookie)
 651             m_proxyTarget = (char*) xstrdup(url[0]);
 652         else
 653             m_proxyTarget = (char*) xstrdup(load_balance(url));
 654     }
 655     if (cookie && *cookie)
 656     {   // search in sessions with a cookie
 657         for (c = parent->m_clientPool; c; c = c->m_next)
 658         {
 659             assert(c->m_prev);
 660             assert(*c->m_prev == c);
 661             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 662                 !strcmp(m_proxyTarget, c->get_hostname()))
 663             {
 664                 // Found it in cache
 665                 // The following handles "cancel"
 666                 // If connection is busy (waiting for PDU) and
 667                 // we have an initRequest we can safely do re-open
 668                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 669                 {
 670                     yaz_log(YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 671                              c->get_hostname());
 672                     c->close();
 673                     c->m_init_flag = 0;
 674
 675                     c->m_last_ok = 0;
 676                     c->m_cache.clear();
 677                     c->m_last_resultCount = 0;
 678                     c->m_sr_transform = 0;
 679                     c->m_waiting = 0;
 680                     c->m_resultSetStartPoint = 0;
 681                     c->m_target_idletime = m_target_idletime;
 682                     if (c->client(m_proxyTarget))
 683                     {
 684                         delete c;
 685                         return 0;
 686                     }
 687                     c->timeout(30);
 688                 }
 689                 c->m_seqno = parent->m_seqno;
 690                 if (c->m_server && c->m_server != this)
 691                     c->m_server->m_client = 0;
 692                 c->m_server = this;
 693                 (parent->m_seqno)++;
 694                 yaz_log(YLOG_DEBUG, "get_client 1 %p %p", this, c);
 695                 return c;
 696             }
 697         }
 698     }
 699     else if (!c && apdu->which == Z_APDU_initRequest )
 700     {
 701         // anonymous sessions without cookie.
 702         // if authentication is set it is NOT anonymous se we can't share them.
 703         // If charset and lang negotiation is use it is NOT anonymous session too.
 704         for (c = parent->m_clientPool; c; c = c->m_next)
 705         {
 706             assert(c->m_prev);
 707             assert(*c->m_prev == c);
 708             if (c->m_server == 0 && c->m_cookie == 0 && c->m_waiting == 0
 709                 && c->compare_idAuthentication(apdu)
 710                 && c->compare_charset(apdu)
 711                 && !strcmp(m_proxyTarget, c->get_hostname()))
 712             {
 713                 // found it in cache
 714                 yaz_log(YLOG_LOG, "%sREUSE %d %s",
 715                          m_session_str, parent->m_seqno, c->get_hostname());
 716
 717                 c->m_seqno = parent->m_seqno;
 718                 assert(c->m_server == 0);
 719                 c->m_server = this;
 720
 721                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 722                     c->set_APDU_yazlog(1);
 723                 else
 724                     c->set_APDU_yazlog(0);
 725
 726                 (parent->m_seqno)++;
 727
 728                 parent->pre_init();
 729
 730                 return c;
 731             }
 732         }
 733     }
 734     if (!m_client)
 735     {
 736         if (apdu->which != Z_APDU_initRequest)
 737         {
 738             yaz_log(YLOG_LOG, "%sno init request as first PDU", m_session_str);
 739             *http_code = 500;
 740             return 0;
 741         }
 742
 743         int no_in_use = get_number_of_connections();
 744         if (no_in_use >= m_max_sockets)
 745         {
 746             yaz_log(YLOG_LOG, "%smax sockets reached %d", m_session_str,
 747                 m_max_sockets);
 748             *http_code = 500;
 749             return 0;
 750         }
 751         // go through list of clients - and find the lowest/oldest one.
 752         Yaz_ProxyClient *c_min = 0;
 753         int min_seq = -1;
 754         int no_of_clients = 0;
 755         if (parent->m_clientPool)
 756             yaz_log(YLOG_DEBUG, "Existing sessions");
 757         for (c = parent->m_clientPool; c; c = c->m_next)
 758         {
 759             yaz_log(YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 760                                c->m_waiting, c->get_hostname(),
 761                                c->m_cookie ? c->m_cookie : "");
 762             no_of_clients++;
 763             if (min_seq < 0 || c->m_seqno < min_seq)
 764             {
 765                 min_seq = c->m_seqno;
 766                 c_min = c;
 767             }
 768         }
 769         if (no_of_clients >= parent->m_max_clients)
 770         {
 771             c = c_min;
 772             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 773             {
 774                 yaz_log(YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 775                          m_session_str, parent->m_max_clients, c->m_seqno);
 776                 if (c->m_server && c->m_server != this)
 777                     c->m_server->dec_ref();
 778             }
 779             else
 780             {
 781                 yaz_log(YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 782                          m_session_str, parent->m_max_clients,
 783                          c->m_seqno, parent->m_seqno, c->get_hostname());
 784                 xfree(c->m_cookie);
 785                 c->m_cookie = 0;
 786                 if (cookie)
 787                     c->m_cookie = xstrdup(cookie);
 788                 c->m_seqno = parent->m_seqno;
 789                 if (c->m_server && c->m_server != this)
 790                 {
 791                     c->m_server->m_client = 0;
 792                     c->m_server->dec_ref();
 793                 }
 794                 (parent->m_seqno)++;
 795                 c->m_target_idletime = m_target_idletime;
 796                 c->timeout(m_target_idletime);
 797
 798                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 799                     c->set_APDU_yazlog(1);
 800                 else
 801                     c->set_APDU_yazlog(0);
 802
 803                 return c;
 804             }
 805         }
 806         yaz_log(YLOG_LOG, "%sNEW %d %s",
 807                 m_session_str, parent->m_seqno, m_proxyTarget);
 808         c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 809         c->m_next = parent->m_clientPool;
 810         if (c->m_next)
 811             c->m_next->m_prev = &c->m_next;
 812         parent->m_clientPool = c;
 813         c->m_prev = &parent->m_clientPool;
 814
 815         xfree(c->m_cookie);
 816         c->m_cookie = 0;
 817         if (cookie)
 818             c->m_cookie = xstrdup(cookie);
 819
 820         c->m_seqno = parent->m_seqno;
 821         c->m_init_flag = 0;
 822         c->m_last_resultCount = 0;
 823         c->m_last_ok = 0;
 824         c->m_cache.clear();
 825         c->m_sr_transform = 0;
 826         c->m_waiting = 0;
 827         c->m_resultSetStartPoint = 0;
 828         (parent->m_seqno)++;
 829         if (c->client(m_proxyTarget))
 830         {
 831             delete c;
 832             return 0;
 833         }
 834         c->m_target_idletime = m_target_idletime;
 835         c->timeout(30);
 836
 837         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 838             c->set_APDU_yazlog(1);
 839         else
 840             c->set_APDU_yazlog(0);
 841
 842         c->set_idAuthentication(apdu);
 843     }
 844     yaz_log(YLOG_DEBUG, "get_client 3 %p %p", this, c);
 845     return c;
 846 }
 847
 848 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 849 {
 850     int i;
 851     for (i = 0; i<num; i++)
 852     {
 853         Z_DefaultDiagFormat *r;
 854         Z_DiagRec *p = pp[i];
 855         if (p->which != Z_DiagRec_defaultFormat)
 856         {
 857             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 858             return;
 859         }
 860         else
 861             r = p->u.defaultFormat;
 862         switch (r->which)
 863         {
 864         case Z_DefaultDiagFormat_v2Addinfo:
 865             yaz_log(YLOG_LOG, "%sError " ODR_INT_PRINTF " %s:%s",
 866                     m_session_str,
 867                     *r->condition, diagbib1_str(*r->condition),
 868                     r->u.v2Addinfo);
 869             break;
 870         case Z_DefaultDiagFormat_v3Addinfo:
 871             yaz_log(YLOG_LOG, "%sError " ODR_INT_PRINTF " %s:%s",
 872                     m_session_str,
 873                     *r->condition, diagbib1_str(*r->condition),
 874                     r->u.v3Addinfo);
 875             break;
 876         }
 877     }
 878 }
 879
 880 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 881 {
 882     if (!m_stylesheet_xsp || p->num_records <= 0)
 883     {
 884         return 0;  /* no XSLT to be done ... */
 885     }
 886
 887     m_stylesheet_offset = 0;
 888     m_stylesheet_nprl = p;
 889     m_stylesheet_apdu = apdu;
 890     m_timeout_mode = timeout_xsl;
 891
 892     timeout(0);
 893     return 1;
 894 }
 895
 896 void Yaz_Proxy::convert_xsl_delay()
 897 {
 898 #if YAZ_HAVE_XSLT
 899     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 900     if (npr->which == Z_NamePlusRecord_databaseRecord)
 901     {
 902         Z_External *r = npr->u.databaseRecord;
 903         if (r->which == Z_External_octet)
 904         {
 905 #if 0
 906             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 907 #endif
 908             xmlDocPtr res, doc = xmlParseMemory(
 909                 (char*) r->u.octet_aligned->buf,
 910                 r->u.octet_aligned->len);
 911
 912
 913             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 914                     m_session_str, m_stylesheet_offset);
 915             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 916                                       doc, 0);
 917
 918             if (res)
 919             {
 920                 xmlChar *out_buf;
 921                 int out_len;
 922                 xmlDocDumpFormatMemory(res, &out_buf, &out_len, 1);
 923                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 924                     u.databaseRecord =
 925                     z_ext_record_oid(odr_encode(), yaz_oid_recsyn_xml,
 926                                      (char*) out_buf, out_len);
 927                 xmlFree(out_buf);
 928                 xmlFreeDoc(res);
 929             }
 930
 931             xmlFreeDoc(doc);
 932         }
 933     }
 934 #endif
 935     m_stylesheet_offset++;
 936     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 937     {
 938         m_timeout_mode = timeout_normal;
 939         m_stylesheet_nprl = 0;
 940 #if YAZ_HAVE_XSLT
 941         if (m_stylesheet_xsp)
 942             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 943 #endif
 944         m_stylesheet_xsp = 0;
 945         timeout(m_client_idletime);
 946         send_PDU_convert(m_stylesheet_apdu);
 947     }
 948     else
 949         timeout(0);
 950 }
 951
 952 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 953 {
 954     if (m_frontend_type[0] != -1)
 955     {
 956         int i;
 957         for (i = 0; i < p->num_records; i++)
 958         {
 959             Z_NamePlusRecord *npr = p->records[i];
 960             if (npr->which == Z_NamePlusRecord_databaseRecord)
 961             {
 962                 Z_External *r = npr->u.databaseRecord;
 963                 if (r->which == Z_External_octet)
 964                 {
 965 #if !HAVE_USEMARCON
 966                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 967                         yaz_log(YLOG_LOG, "%sError: USEMARCON requested but not available",
 968                             m_session_str);
 969 #endif
 970 #if HAVE_USEMARCON
 971                     yaz_log(YLOG_DEBUG, "%sUSEMARCON stage1=%s stage2=%s",
 972                         m_session_str,
 973                         m_usemarcon_ini_stage1 ? m_usemarcon_ini_stage1 : "(none)",
 974                         m_usemarcon_ini_stage2 ? m_usemarcon_ini_stage2 : "(none)");
 975                     char *converted;
 976                     int convlen;
 977                     if (m_usemarcon->convert(m_usemarcon_ini_stage1, m_usemarcon_ini_stage2,
 978                         (char*) r->u.octet_aligned->buf, r->u.octet_aligned->len,
 979                         &converted, &convlen))
 980                     {
 981                         npr->u.databaseRecord =
 982                             z_ext_record_oid(odr_encode(),
 983                                              m_frontend_type,
 984                                              converted,
 985                                              strlen(converted));
 986                         free(converted);
 987                     }
 988                     else
 989 #endif
 990 /* HAVE_USEMARCON */
 991                     npr->u.databaseRecord =
 992                         z_ext_record_oid(odr_encode(),
 993                                          m_frontend_type,
 994                                          (char*) r->u.octet_aligned->buf,
 995                                          r->u.octet_aligned->len);
 996                 }
 997             }
 998         }
 999     }
1000 }
1001
1002 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
1003                                         const char *backend_charset)
1004 {
1005     int sel =   m_charset_converter->get_client_charset_selected();
1006     const char *client_record_charset =
1007         m_charset_converter->get_client_query_charset();
1008     if (sel && backend_charset && client_record_charset &&
1009         strcmp(backend_charset, client_record_charset))
1010     {
1011         int i;
1012         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
1013                                         backend_charset);
1014         yaz_marc_t mt = yaz_marc_create();
1015         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
1016         yaz_marc_iconv(mt, cd);
1017         for (i = 0; i < p->num_records; i++)
1018         {
1019             Z_NamePlusRecord *npr = p->records[i];
1020             if (npr->which == Z_NamePlusRecord_databaseRecord)
1021             {
1022                 Z_External *r = npr->u.databaseRecord;
1023                 const Odr_oid *oid = r->direct_reference;
1024                 if (!oid)
1025                     continue;
1026
1027                 if (!oid_oidcmp(oid, yaz_oid_recsyn_sutrs))
1028                 {
1029                     WRBUF w = wrbuf_alloc();
1030
1031                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
1032                                       r->u.octet_aligned->len);
1033                     npr->u.databaseRecord =
1034                         z_ext_record_oid(odr_encode(), oid, wrbuf_buf(w),
1035                                          wrbuf_len(w));
1036                     wrbuf_destroy(w);
1037                 }
1038                 else if (!oid_oidcmp(oid, yaz_oid_recsyn_xml))
1039                 {
1040                     ;
1041                 }
1042                 else if (r->which == Z_External_octet)
1043                 {
1044                     size_t rlen;
1045                     const char *result;
1046                     if (yaz_marc_decode_buf(mt,
1047                                             (char*) r->u.octet_aligned->buf,
1048                                             r->u.octet_aligned->len,
1049                                             &result, &rlen))
1050                     {
1051                         npr->u.databaseRecord =
1052                             z_ext_record_oid(odr_encode(), oid, result, rlen);
1053                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1054                                 m_session_str);
1055                     }
1056                 }
1057             }
1058         }
1059         if (cd)
1060             yaz_iconv_close(cd);
1061         yaz_marc_destroy(mt);
1062     }
1063 }
1064
1065 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1066                                    const char *backend_charset)
1067 {
1068     int i;
1069     if (!backend_charset)
1070         backend_charset = "MARC-8";
1071     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1072     yaz_marc_t mt = yaz_marc_create();
1073     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1074     yaz_marc_iconv(mt, cd);
1075     for (i = 0; i < p->num_records; i++)
1076     {
1077         Z_NamePlusRecord *npr = p->records[i];
1078         if (npr->which == Z_NamePlusRecord_databaseRecord)
1079         {
1080             Z_External *r = npr->u.databaseRecord;
1081             if (r->which == Z_External_OPAC)
1082             {
1083                 WRBUF w = wrbuf_alloc();
1084
1085                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
1086                 npr->u.databaseRecord = z_ext_record_oid(
1087                     odr_encode(), yaz_oid_recsyn_xml,
1088                     wrbuf_buf(w), wrbuf_len(w));
1089                 wrbuf_destroy(w);
1090             }
1091             else if (r->which == Z_External_octet)
1092             {
1093                 size_t rlen;
1094                 const char *result;
1095                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1096                                         r->u.octet_aligned->len,
1097                                         &result, &rlen))
1098                 {
1099                     npr->u.databaseRecord =
1100                         z_ext_record_oid(odr_encode(), yaz_oid_recsyn_xml,
1101                                          result, rlen);
1102                 }
1103             }
1104         }
1105     }
1106     if (cd)
1107         yaz_iconv_close(cd);
1108     yaz_marc_destroy(mt);
1109 }
1110
1111 void Yaz_Proxy::logtime()
1112 {
1113 #if HAVE_GETTIMEOFDAY
1114     struct timeval *tv = (struct timeval*) m_time_tv;
1115     if (tv->tv_sec)
1116     {
1117         struct timeval tv1;
1118         gettimeofday(&tv1, 0);
1119         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1120             (tv1.tv_usec - tv->tv_usec);
1121         if (diff >= 0)
1122             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1123                     diff/1000000, (diff/1000)%1000);
1124     }
1125     tv->tv_sec = 0;
1126     tv->tv_usec = 0;
1127 #endif
1128 }
1129
1130 int Yaz_Proxy::send_http_response(int code)
1131 {
1132     ODR o = odr_encode();
1133     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1134     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1135     if (m_http_version)
1136         hres->version = odr_strdup(o, m_http_version);
1137     if (m_http_keepalive)
1138         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1139     else
1140         timeout(0);
1141     if (code == 401)
1142         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate",
1143                           "Basic realm=\"YAZ Proxy\"");
1144
1145
1146     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1147     {
1148         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1149                  gdu_name(gdu));
1150     }
1151     int len;
1152     int r = send_GDU(gdu, &len);
1153     m_bytes_sent += len;
1154     m_bw_stat.add_bytes(len);
1155     logtime();
1156
1157     recv_GDU_more(true);
1158
1159     return r;
1160 }
1161
1162 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */)
1163 {
1164     ODR o = odr_encode();
1165     const char *ctype = "text/xml";
1166     Z_GDU *gdu = z_get_HTTP_Response(o, http_code);
1167     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1168     if (m_http_version)
1169         hres->version = odr_strdup(o, m_http_version);
1170     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1171     if (m_http_keepalive)
1172         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1173     else
1174         timeout(0);
1175     if (http_code == 401)
1176         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\"");
1177
1178     static Z_SOAP_Handler soap_handlers[2] = {
1179 #if YAZ_HAVE_XSLT
1180         { (char*) "http://www.loc.gov/zing/srw/", 0,
1181          (Z_SOAP_fun) yaz_srw_codec},
1182 #endif
1183         {0, 0, 0}
1184     };
1185
1186     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1187     soap_package->which = Z_SOAP_generic;
1188     soap_package->u.generic =
1189         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1190     soap_package->u.generic->no = 0;
1191     soap_package->u.generic->ns = soap_handlers[0].ns;
1192     soap_package->u.generic->p = (void *) srw_pdu;
1193     soap_package->ns = m_soap_ns;
1194     z_soap_codec_enc_xsl(o, &soap_package,
1195                          &hres->content_buf, &hres->content_len,
1196                          soap_handlers, 0, m_s2z_stylesheet);
1197     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1198     {
1199         yaz_log(YLOG_LOG, "%sSending %s to client", m_session_str,
1200                 gdu_name(gdu));
1201     }
1202     int len;
1203     int r = send_GDU(gdu, &len);
1204     m_bytes_sent += len;
1205     m_bw_stat.add_bytes(len);
1206     logtime();
1207
1208     recv_GDU_more(true);
1209
1210     return r;
1211 }
1212
1213 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1214 {
1215     ODR o = odr_encode();
1216     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1217         odr_malloc(o, sizeof(*diagnostic));
1218     int num_diagnostic = 1;
1219     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1220     return send_srw_search_response(diagnostic, num_diagnostic, srw_error == 3 ? 401 : 200);
1221 }
1222
1223 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1224                              Z_DefaultDiagFormat *ddf)
1225 {
1226     int bib1_code = *ddf->condition;
1227     if (bib1_code == 109)
1228         return 404;
1229     srw_res->num_diagnostics = 1;
1230     srw_res->diagnostics = (Z_SRW_diagnostic *)
1231         odr_malloc(o, sizeof(*srw_res->diagnostics));
1232     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1233                           yaz_diag_bib1_to_srw(*ddf->condition),
1234                           ddf->u.v2Addinfo);
1235     return 0;
1236 }
1237
1238 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_scanResponse *srw_res,
1239                              Z_DiagRec *dr)
1240 {
1241     if (dr->which == Z_DiagRec_defaultFormat)
1242     {
1243         int bib1_code = *dr->u.defaultFormat->condition;
1244         if (bib1_code == 109)
1245             return 404;
1246         srw_res->num_diagnostics = 1;
1247         srw_res->diagnostics = (Z_SRW_diagnostic *)
1248             odr_malloc(o, sizeof(*srw_res->diagnostics));
1249         yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1250                           yaz_diag_bib1_to_srw(*dr->u.defaultFormat->condition),
1251                           dr->u.defaultFormat->u.v2Addinfo);
1252     }
1253     return 0;
1254 }
1255
1256 #if YAZ_HAS_MK_SURROGATE
1257 #else
1258 static void yazproxy_mk_sru_surrogate(ODR o, Z_SRW_record *record, int pos,
1259                           int code, const char *details)
1260 {
1261     const char *message = yaz_diag_srw_str(code);
1262     int len = 200;
1263     if (message)
1264         len += strlen(message);
1265     if (details)
1266         len += strlen(details);
1267
1268     record->recordData_buf = (char *) odr_malloc(o, len);
1269
1270     sprintf(record->recordData_buf, "<diagnostic "
1271             "xmlns=\"http://www.loc.gov/zing/srw/diagnostic/\">\n"
1272             " <uri>info:srw/diagnostic/1/%d</uri>\n", code);
1273     if (details)
1274         sprintf(record->recordData_buf + strlen(record->recordData_buf),
1275                 " <details>%s</details>\n", details);
1276     if (message)
1277         sprintf(record->recordData_buf + strlen(record->recordData_buf),
1278                 " <message>%s</message>\n", message);
1279     sprintf(record->recordData_buf + strlen(record->recordData_buf),
1280             "</diagnostic>\n");
1281     record->recordData_len = strlen(record->recordData_buf);
1282     record->recordPosition = odr_intdup(o, pos);
1283     record->recordSchema = odr_strdup(o, "info:srw/schema/1/diagnostics-v1.1");
1284 }
1285 #endif
1286
1287 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1288 {
1289     ODR o = odr_encode();
1290     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1291     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1292
1293     srw_res->numberOfRecords = odr_intdup (o, hits);
1294     if (records && records->which == Z_Records_DBOSD)
1295     {
1296         srw_res->num_records =
1297             records->u.databaseOrSurDiagnostics->num_records;
1298         int i;
1299         srw_res->records = (Z_SRW_record *)
1300             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1301         for (i = 0; i < srw_res->num_records; i++)
1302         {
1303             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1304             if (npr->which != Z_NamePlusRecord_databaseRecord)
1305             {
1306 #if YAZ_HAS_MK_SURROGATE
1307                 yaz_mk_sru_surrogate(
1308                     o, srw_res->records + i, i+start,
1309                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1310 #else
1311                 yazproxy_mk_sru_surrogate(
1312                     o, srw_res->records + i, i+start,
1313                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1314 #endif
1315                 continue;
1316             }
1317             Z_External *r = npr->u.databaseRecord;
1318
1319             if (r->which == Z_External_octet
1320                 && !oid_oidcmp(r->direct_reference, yaz_oid_recsyn_xml))
1321             {
1322                 srw_res->records[i].recordSchema = m_schema;
1323                 srw_res->records[i].recordPacking = m_s2z_packing;
1324                 srw_res->records[i].recordData_buf = (char*)
1325                     r->u.octet_aligned->buf;
1326                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1327                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1328             }
1329             else
1330             {
1331 #if YAZ_HAS_MK_SURROGATE
1332                 yaz_mk_sru_surrogate(
1333                     o, srw_res->records + i, i+start,
1334                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1335 #else
1336                 yazproxy_mk_sru_surrogate(
1337                     o, srw_res->records + i, i+start,
1338                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1339 #endif
1340             }
1341         }
1342     }
1343     if (records && records->which == Z_Records_NSD)
1344     {
1345         int http_code;
1346         http_code = z_to_srw_diag(odr_encode(), srw_res,
1347                                    records->u.nonSurrogateDiagnostic);
1348         if (http_code)
1349             return send_http_response(http_code);
1350     }
1351     return send_srw_response(srw_pdu);
1352
1353 }
1354
1355 int Yaz_Proxy::send_to_srw_client_ok(Z_ListEntries *entries)
1356 {
1357     ODR o = odr_encode();
1358     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_scan_response);
1359     Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1360
1361     if (entries && entries->num_entries > 0)
1362     {
1363         srw_res->num_terms = entries->num_entries;
1364         int i;
1365         srw_res->terms = (Z_SRW_scanTerm *)
1366             odr_malloc(o, srw_res->num_terms * sizeof(Z_SRW_scanTerm));
1367         for (i = 0; i < srw_res->num_terms; i++)
1368         {
1369             if (entries->entries[i]->which == Z_Entry_termInfo)
1370             {
1371                 switch(entries->entries[i]->u.termInfo->term->which)
1372                 {
1373                 case Z_Term_general:
1374                     srw_res->terms[i].value = odr_strdupn(o,
1375                         (char *) entries->entries[i]->u.termInfo->term->u.general->buf,
1376                         entries->entries[i]->u.termInfo->term->u.general->len);
1377                     break;
1378                 default:
1379                     srw_res->terms[i].value = NULL;
1380                 }
1381                 if (entries->entries[i]->u.termInfo->globalOccurrences != NULL)
1382                     srw_res->terms[i].numberOfRecords = odr_intdup(o,
1383                         *entries->entries[i]->u.termInfo->globalOccurrences);
1384                 else
1385                     srw_res->terms[i].numberOfRecords = NULL;
1386                 if (entries->entries[i]->u.termInfo->displayTerm != NULL)
1387                     srw_res->terms[i].displayTerm = odr_strdup(o,
1388                         entries->entries[i]->u.termInfo->displayTerm);
1389                 else
1390                     srw_res->terms[i].displayTerm = NULL;
1391                 srw_res->terms[i].whereInList = NULL;
1392             }
1393         }
1394     }
1395     if (entries && entries->num_nonsurrogateDiagnostics > 0)
1396     {
1397         int http_code;
1398         http_code = z_to_srw_diag(odr_encode(), srw_res,
1399                                    entries->nonsurrogateDiagnostics[0]);
1400         if (http_code)
1401             return send_http_response(http_code);
1402     }
1403     return send_srw_response(srw_pdu);
1404
1405 }
1406
1407 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1408                                         int num_diagnostics, int http_code /* = 200 */)
1409 {
1410     ODR o = odr_encode();
1411     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1412     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1413
1414     srw_res->num_diagnostics = num_diagnostics;
1415     srw_res->diagnostics = diagnostics;
1416     return send_srw_response(srw_pdu, http_code);
1417 }
1418
1419 int Yaz_Proxy::send_srw_scan_response(Z_SRW_diagnostic *diagnostics,
1420                                         int num_diagnostics, int http_code /* = 200 */)
1421 {
1422     ODR o = odr_encode();
1423     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_scan_response);
1424     Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1425
1426     srw_res->num_diagnostics = num_diagnostics;
1427     srw_res->diagnostics = diagnostics;
1428     return send_srw_response(srw_pdu, http_code);
1429 }
1430
1431 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1432                                         int num_diagnostics)
1433 {
1434     int http_status = 404;
1435     Yaz_ProxyConfig *cfg = check_reconfigure();
1436     if (cfg)
1437     {
1438         int len;
1439         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1440                                        m_s2z_database, &len, &http_status);
1441         if (b)
1442         {
1443             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1444             Z_SRW_explainResponse *er = res->u.explain_response;
1445
1446             er->record.recordData_buf = b;
1447             er->record.recordData_len = len;
1448             er->record.recordPacking = m_s2z_packing;
1449             er->record.recordSchema = odr_strdup(odr_encode(),
1450                                                  "http://explain.z3950.org/dtd/2.0/");
1451
1452             er->diagnostics = diagnostics;
1453             er->num_diagnostics = num_diagnostics;
1454             return send_srw_response(res);
1455         }
1456     }
1457     return send_http_response(http_status);
1458 }
1459
1460 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1461 {
1462     if (m_http_version)
1463     {
1464         if (apdu->which == Z_APDU_initResponse)
1465         {
1466             Z_InitResponse *res = apdu->u.initResponse;
1467             if (*res->result == 0)
1468             {
1469                 send_to_srw_client_error(3, 0);
1470             }
1471             else if (m_s2z_search_apdu)
1472             {
1473                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1474             }
1475             else if (m_s2z_scan_apdu)
1476             {
1477                 handle_incoming_Z_PDU(m_s2z_scan_apdu);
1478             }
1479             else
1480             {
1481                 send_srw_explain_response(0, 0);
1482             }
1483         }
1484         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1485         {
1486             m_s2z_search_apdu = 0;
1487             Z_SearchResponse *res = apdu->u.searchResponse;
1488             m_s2z_hit_count = *res->resultCount;
1489             if (res->records && res->records->which == Z_Records_NSD)
1490             {
1491                 send_to_srw_client_ok(0, res->records, 1);
1492             }
1493             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1494             {
1495                 // adjust
1496                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1497
1498                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1499                 {
1500                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1501                         > m_s2z_hit_count)
1502                         *pr->numberOfRecordsRequested =
1503                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1504                 }
1505                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1506             }
1507             else
1508             {
1509                 m_s2z_present_apdu = 0;
1510                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1511             }
1512         }
1513         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1514         {
1515             int start =
1516                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1517
1518             m_s2z_present_apdu = 0;
1519             Z_PresentResponse *res = apdu->u.presentResponse;
1520             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1521         }
1522         else if (m_s2z_scan_apdu && apdu->which == Z_APDU_scanResponse)
1523         {
1524             Z_ScanResponse *res = apdu->u.scanResponse;
1525             send_to_srw_client_ok(res->entries);
1526         }
1527     }
1528     else
1529     {
1530         int len = 0;
1531         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1532             yaz_log(YLOG_LOG, "%sSending %s to client", m_session_str,
1533                      apdu_name(apdu));
1534         int r = send_Z_PDU(apdu, &len);
1535         m_bytes_sent += len;
1536         m_bw_stat.add_bytes(len);
1537         logtime();
1538         return r;
1539     }
1540     return 0;
1541 }
1542
1543 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1544 {
1545     int kill_session = 0;
1546     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1547
1548     if (new_id)
1549         *new_id = m_referenceId;
1550
1551     if (apdu->which == Z_APDU_searchResponse)
1552     {
1553         Z_SearchResponse *sr = apdu->u.searchResponse;
1554         Z_Records *p = sr->records;
1555         if (p && p->which == Z_Records_NSD)
1556         {
1557             Z_DiagRec dr, *dr_p = &dr;
1558             dr.which = Z_DiagRec_defaultFormat;
1559             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1560
1561             *sr->searchStatus = 0;
1562             display_diagrecs(&dr_p, 1);
1563         }
1564         else
1565         {
1566             if (p && p->which == Z_Records_DBOSD)
1567             {
1568                 if (m_backend_type
1569 #if HAVE_USEMARCON
1570                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1571 #endif
1572                     )
1573                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1574                 if (m_marcxml_mode == marcxml)
1575                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1576                                        m_backend_charset);
1577                 else
1578                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1579                                             m_backend_charset);
1580                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1581                     return 0;
1582
1583             }
1584             if (sr->resultCount)
1585             {
1586                 yaz_log(YLOG_LOG, "%s" ODR_INT_PRINTF " hits", m_session_str,
1587                         *sr->resultCount);
1588                 if (*sr->resultCount < 0)
1589                 {
1590                     m_flag_invalid_session = 1;
1591                     kill_session = 1;
1592
1593                     *sr->searchStatus = 0;
1594                     sr->records =
1595                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1596                     *sr->resultCount = 0;
1597                 }
1598             }
1599         }
1600     }
1601     else if (apdu->which == Z_APDU_presentResponse)
1602     {
1603         Z_PresentResponse *sr = apdu->u.presentResponse;
1604         Z_Records *p = sr->records;
1605         if (p && p->which == Z_Records_NSD)
1606         {
1607             Z_DiagRec dr, *dr_p = &dr;
1608             dr.which = Z_DiagRec_defaultFormat;
1609             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1610             if (*sr->presentStatus == Z_PresentStatus_success)
1611                 *sr->presentStatus = Z_PresentStatus_failure;
1612             display_diagrecs(&dr_p, 1);
1613         }
1614         if (p && p->which == Z_Records_DBOSD)
1615         {
1616             if (m_backend_type
1617 #if HAVE_USEMARCON
1618                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1619 #endif
1620                 )
1621                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1622             if (m_marcxml_mode == marcxml)
1623                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1624                                    m_backend_charset);
1625             else
1626                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1627                                         m_backend_charset);
1628             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1629                 return 0;
1630         }
1631     }
1632     else if (apdu->which == Z_APDU_initResponse)
1633     {
1634         //Get and check negotiation record
1635         //from init response.
1636         handle_charset_lang_negotiation(apdu);
1637
1638         if (m_initRequest_options)
1639         {
1640             Z_Options *nopt =
1641                 (Odr_bitmask *)odr_malloc(odr_encode(),
1642                                           sizeof(Odr_bitmask));
1643             ODR_MASK_ZERO(nopt);
1644
1645             int i;
1646             for (i = 0; i<24; i++)
1647                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1648                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1649                     ODR_MASK_SET(nopt, i);
1650             apdu->u.initResponse->options = nopt;
1651         }
1652         if (m_initRequest_version)
1653         {
1654             Z_ProtocolVersion *nopt =
1655                 (Odr_bitmask *)odr_malloc(odr_encode(),
1656                                           sizeof(Odr_bitmask));
1657             ODR_MASK_ZERO(nopt);
1658
1659             int i;
1660             for (i = 0; i<8; i++)
1661                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1662                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1663                     ODR_MASK_SET(nopt, i);
1664             apdu->u.initResponse->protocolVersion = nopt;
1665         }
1666         apdu->u.initResponse->preferredMessageSize =
1667             odr_intdup(odr_encode(),
1668                        m_client->m_initResponse_preferredMessageSize >
1669                        m_initRequest_preferredMessageSize ?
1670                        m_initRequest_preferredMessageSize :
1671                        m_client->m_initResponse_preferredMessageSize);
1672         apdu->u.initResponse->maximumRecordSize =
1673             odr_intdup(odr_encode(),
1674                        m_client->m_initResponse_maximumRecordSize >
1675                        m_initRequest_maximumRecordSize ?
1676                        m_initRequest_maximumRecordSize :
1677                        m_client->m_initResponse_maximumRecordSize);
1678     }
1679
1680     int r = send_PDU_convert(apdu);
1681     if (r)
1682         return r;
1683     if (kill_session)
1684     {
1685         delete m_client;
1686         m_client = 0;
1687         m_parent->pre_init();
1688     }
1689     return r;
1690 }
1691
1692 void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu)
1693 {
1694     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1695
1696     odr_reset(m_idAuthentication_odr);
1697     z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0);
1698     m_idAuthentication_ber_buf =
1699         odr_getbuf(m_idAuthentication_odr,
1700                    &m_idAuthentication_ber_size, 0);
1701 }
1702
1703 bool Yaz_ProxyClient::compare_charset(Z_APDU *apdu)
1704 {
1705     return true;
1706 }
1707
1708 bool Yaz_ProxyClient::compare_idAuthentication(Z_APDU *apdu)
1709 {
1710     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1711     ODR odr = odr_createmem(ODR_ENCODE);
1712
1713     z_IdAuthentication(odr, &t, 1, 0);
1714     int sz;
1715     char *buf = odr_getbuf(odr, &sz, 0);
1716     if (buf && m_idAuthentication_ber_buf
1717         && sz == m_idAuthentication_ber_size
1718         && !memcmp(m_idAuthentication_ber_buf, buf, sz))
1719     {
1720         odr_destroy(odr);
1721         return true;
1722     }
1723     odr_destroy(odr);
1724     if (!buf && !m_idAuthentication_ber_buf)
1725         return true;
1726     return false;
1727 }
1728
1729 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1730 {
1731     int len = 0;
1732     const char *apdu_name_tmp = apdu_name(apdu);
1733     int r = send_Z_PDU(apdu, &len);
1734     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1735         yaz_log(YLOG_LOG, "%sSending %s to %s %d bytes",
1736                  get_session_str(),
1737                  apdu_name_tmp, get_hostname(), len);
1738     m_bytes_sent += len;
1739     return r;
1740 }
1741
1742 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1743 {
1744     if (apdu->which == Z_APDU_presentRequest)
1745     {
1746         Z_PresentRequest *pr = apdu->u.presentRequest;
1747         int toget = *pr->numberOfRecordsRequested;
1748         int start = *pr->resultSetStartPoint;
1749
1750         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1751                 pr->resultSetId, start, toget);
1752
1753         if (*m_parent->m_optimize == '0')
1754             return apdu;
1755
1756         if (!m_client->m_last_resultSetId)
1757         {
1758             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1759             new_apdu->u.presentResponse->records =
1760                 create_nonSurrogateDiagnostics(
1761                     odr_encode(),
1762                     YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
1763                     pr->resultSetId);
1764             send_to_client(new_apdu);
1765             return 0;
1766         }
1767         if (start < 1 || toget < 0)
1768         {
1769             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1770             new_apdu->u.presentResponse->records =
1771                 create_nonSurrogateDiagnostics(
1772                     odr_encode(),
1773                     YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
1774                     0);
1775             send_to_client(new_apdu);
1776             return 0;
1777         }
1778         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1779         {
1780             if (start+toget-1 > m_client->m_last_resultCount)
1781             {
1782                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1783                 new_apdu->u.presentResponse->records =
1784                     create_nonSurrogateDiagnostics(
1785                         odr_encode(),
1786                         YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
1787                         0);
1788                 send_to_client(new_apdu);
1789                 return 0;
1790             }
1791             Z_NamePlusRecordList *npr;
1792 #if 0
1793             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1794                     m_session_str, start, toget, yaz_z3950oid_to_str(
1795                         pr->preferredRecordSyntax, &oclass));
1796 #endif
1797             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1798                                           pr->preferredRecordSyntax,
1799                                           pr->recordComposition))
1800             {
1801                 yaz_log(YLOG_LOG, "%sReturned cached records for present request",
1802                          m_session_str);
1803                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1804                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1805
1806                 new_apdu->u.presentResponse->numberOfRecordsReturned
1807                     = odr_intdup(odr_encode(), toget);
1808
1809                 new_apdu->u.presentResponse->records = (Z_Records*)
1810                     odr_malloc(odr_encode(), sizeof(Z_Records));
1811                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1812                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1813                 new_apdu->u.presentResponse->nextResultSetPosition =
1814                     odr_intdup(odr_encode(), start+toget);
1815
1816                 send_to_client(new_apdu);
1817                 return 0;
1818             }
1819         }
1820     }
1821
1822     if (apdu->which != Z_APDU_searchRequest)
1823         return apdu;
1824     Z_SearchRequest *sr = apdu->u.searchRequest;
1825     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1826     Yaz_Z_Databases this_databases;
1827
1828     this_databases.set(sr->num_databaseNames, (const char **)
1829                        sr->databaseNames);
1830
1831     this_query->set_Z_Query(sr->query);
1832
1833     // Check for non-negative piggyback params.
1834     if (*sr->smallSetUpperBound < 0
1835         || *sr->largeSetLowerBound < 0
1836         || *sr->mediumSetPresentNumber < 0)
1837     {
1838         Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1839         // Not a present request.. But can't find better diagnostic
1840         new_apdu->u.searchResponse->records =
1841             create_nonSurrogateDiagnostics(
1842                 odr_encode(),
1843                 YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, 0);
1844         send_to_client(new_apdu);
1845         return 0;
1846     }
1847
1848     char query_str[4096];
1849     this_query->print(query_str, sizeof(query_str)-1);
1850     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1851
1852     if (*m_parent->m_optimize != '0' &&
1853         m_client->m_last_ok && m_client->m_last_query &&
1854         m_client->m_last_query->match(this_query) &&
1855         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1856         m_client->m_last_databases.match(this_databases))
1857     {
1858         delete this_query;
1859         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1860             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1861         {
1862             Z_NamePlusRecordList *npr;
1863             int toget = *sr->mediumSetPresentNumber;
1864             Z_RecordComposition *comp = 0;
1865
1866             if (toget > m_client->m_last_resultCount)
1867                 toget = m_client->m_last_resultCount;
1868
1869             if (sr->mediumSetElementSetNames)
1870             {
1871                 comp = (Z_RecordComposition *)
1872                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1873                 comp->which = Z_RecordComp_simple;
1874                 comp->u.simple = sr->mediumSetElementSetNames;
1875             }
1876
1877             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1878                                           sr->preferredRecordSyntax, comp))
1879             {
1880                 yaz_log(YLOG_LOG, "%sReturned cached records for medium set",
1881                          m_session_str);
1882                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1883                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1884                 new_apdu->u.searchResponse->resultCount =
1885                     &m_client->m_last_resultCount;
1886
1887                 new_apdu->u.searchResponse->numberOfRecordsReturned
1888                     = odr_intdup(odr_encode(), toget);
1889
1890                 new_apdu->u.searchResponse->presentStatus =
1891                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1892                 new_apdu->u.searchResponse->records = (Z_Records*)
1893                     odr_malloc(odr_encode(), sizeof(Z_Records));
1894                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1895                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1896                 new_apdu->u.searchResponse->nextResultSetPosition =
1897                     odr_intdup(odr_encode(), toget+1);
1898                 send_to_client(new_apdu);
1899                 return 0;
1900             }
1901             else
1902             {
1903                 // medium Set
1904                 // send present request (medium size)
1905                 yaz_log(YLOG_LOG, "%sOptimizing search for medium set",
1906                          m_session_str);
1907
1908                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1909                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1910                 pr->referenceId = sr->referenceId;
1911                 pr->resultSetId = sr->resultSetName;
1912                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1913                 *pr->numberOfRecordsRequested = toget;
1914                 pr->recordComposition = comp;
1915                 m_client->m_sr_transform = 1;
1916                 return new_apdu;
1917             }
1918         }
1919         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1920             m_client->m_last_resultCount <= 0)
1921         {
1922             // large set. Return pseudo-search response immediately
1923             yaz_log(YLOG_LOG, "%sOptimizing search for large set",
1924                      m_session_str);
1925             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1926             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1927             new_apdu->u.searchResponse->resultCount =
1928                 &m_client->m_last_resultCount;
1929             send_to_client(new_apdu);
1930             return 0;
1931         }
1932         else
1933         {
1934             Z_NamePlusRecordList *npr;
1935             int toget = m_client->m_last_resultCount;
1936             Z_RecordComposition *comp = 0;
1937             // small set
1938             // send a present request (small set)
1939
1940             if (sr->smallSetElementSetNames)
1941             {
1942                 comp = (Z_RecordComposition *)
1943                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1944                 comp->which = Z_RecordComp_simple;
1945                 comp->u.simple = sr->smallSetElementSetNames;
1946             }
1947
1948             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1949                                           sr->preferredRecordSyntax, comp))
1950             {
1951                 yaz_log(YLOG_LOG, "%sReturned cached records for small set",
1952                          m_session_str);
1953                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1954                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1955                 new_apdu->u.searchResponse->resultCount =
1956                     &m_client->m_last_resultCount;
1957
1958                 new_apdu->u.searchResponse->numberOfRecordsReturned
1959                     = odr_intdup(odr_encode(), toget);
1960
1961                 new_apdu->u.searchResponse->presentStatus =
1962                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1963                 new_apdu->u.searchResponse->records = (Z_Records*)
1964                     odr_malloc(odr_encode(), sizeof(Z_Records));
1965                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1966                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1967                 new_apdu->u.searchResponse->nextResultSetPosition =
1968                     odr_intdup(odr_encode(), toget+1);
1969                 send_to_client(new_apdu);
1970                 return 0;
1971             }
1972             else
1973             {
1974                 yaz_log(YLOG_LOG, "%sOptimizing search for small set",
1975                          m_session_str);
1976                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1977                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1978                 pr->referenceId = sr->referenceId;
1979                 pr->resultSetId = sr->resultSetName;
1980                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1981                 *pr->numberOfRecordsRequested = toget;
1982                 pr->recordComposition = comp;
1983                 m_client->m_sr_transform = 1;
1984                 return new_apdu;
1985             }
1986         }
1987     }
1988     else  // query doesn't match
1989     {
1990         delete m_client->m_last_query;
1991         m_client->m_last_query = this_query;
1992         m_client->m_last_ok = 0;
1993         m_client->m_cache.clear();
1994         m_client->m_resultSetStartPoint = 0;
1995
1996         xfree(m_client->m_last_resultSetId);
1997         m_client->m_last_resultSetId = xstrdup(sr->resultSetName);
1998
1999         m_client->m_last_databases.set(sr->num_databaseNames,
2000                                        (const char **) sr->databaseNames);
2001     }
2002     return apdu;
2003 }
2004
2005
2006 void Yaz_Proxy::inc_request_no()
2007 {
2008     m_request_no++;
2009     char *cp = m_session_str + strlen(m_session_str)-1;
2010     if (*cp == ' ')
2011         cp--;
2012     while (*cp && *cp != ' ')
2013         cp--;
2014     if (*cp)
2015         sprintf(cp+1, "%d ", m_request_no);
2016 }
2017
2018 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
2019 {
2020     inc_request_no();
2021
2022     m_bytes_recv += len;
2023
2024     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2025         yaz_log(YLOG_LOG, "%sReceiving %s from client %d bytes",
2026                  m_session_str, gdu_name(apdu), len);
2027
2028 #if 0
2029     // try to make a _bad_ attribute set ID .. Don't enable this in prod.
2030     if (apdu->which == Z_GDU_Z3950
2031         && apdu->u.z3950->which == Z_APDU_searchRequest)
2032     {
2033         Z_SearchRequest *req = apdu->u.z3950->u.searchRequest;
2034         if (req->query && req->query->which == Z_Query_type_1)
2035         {
2036             Z_RPNQuery *rpnquery = req->query->u.type_1;
2037             if (rpnquery->attributeSetId)
2038             {
2039                 rpnquery->attributeSetId[0] = -2;
2040                 rpnquery->attributeSetId[1] = -1;
2041                 yaz_log(YLOG_WARN, "%sBAD FIXUP TEST", m_session_str);
2042             }
2043         }
2044     }
2045 #endif
2046
2047 #if HAVE_GETTIMEOFDAY
2048     gettimeofday((struct timeval *) m_time_tv, 0);
2049 #endif
2050     m_bw_stat.add_bytes(len);
2051     m_pdu_stat.add_bytes(1);
2052
2053     GDU *gdu = new GDU(apdu);
2054
2055     if (gdu->get() == 0)
2056     {
2057         delete gdu;
2058         yaz_log(YLOG_LOG, "%sUnable to encode package", m_session_str);
2059         m_in_queue.clear();
2060         dec_ref();
2061         return;
2062     }
2063     m_in_queue.enqueue(gdu);
2064     recv_GDU_more(false);
2065 }
2066
2067 void Yaz_Proxy::HTTP_Forwarded(Z_GDU *z_gdu)
2068 {
2069     if (z_gdu->which == Z_GDU_HTTP_Request)
2070     {
2071         Z_HTTP_Request *hreq = z_gdu->u.HTTP_Request;
2072         const char *x_forwarded_for =
2073             z_HTTP_header_lookup(hreq->headers, "X-Forwarded-For");
2074         if (x_forwarded_for)
2075         {
2076             xfree(m_peername);
2077             m_peername = (char*) xmalloc(strlen(x_forwarded_for)+5);
2078             sprintf(m_peername, "tcp:%s", x_forwarded_for);
2079
2080             yaz_log(YLOG_LOG, "%sHTTP Forwarded from %s", m_session_str,
2081                     m_peername);
2082             if (m_log_mask & PROXY_LOG_IP_CLIENT)
2083                 sprintf(m_session_str, "%ld:%d %.80s %d ",
2084                         (long) time(0), m_session_no, m_peername, m_request_no);
2085             else
2086                 sprintf(m_session_str, "%ld:%d %d ",
2087                         (long) time(0), m_session_no, m_request_no);
2088         }
2089     }
2090 }
2091
2092 void Yaz_Proxy::connect_stat(bool &block, int &reduce)
2093 {
2094
2095     m_parent->m_connect.cleanup(false);
2096     m_parent->m_connect.add_connect(m_peername);
2097
2098     int connect_total = m_parent->m_connect.get_total(m_peername);
2099     int max_connect = m_parent->m_max_connect;
2100
2101     if (max_connect && connect_total > max_connect)
2102     {
2103         yaz_log(YLOG_LOG, "%sconnect not accepted total=%d max=%d",
2104                 m_session_str, connect_total, max_connect);
2105         block = true;
2106     }
2107     else
2108         block = false;
2109     yaz_log(YLOG_LOG, "%sconnect accepted total=%d", m_session_str,
2110             connect_total);
2111
2112     int limit_connect = m_parent->m_limit_connect;
2113     if (limit_connect)
2114         reduce = connect_total / limit_connect;
2115     else
2116         reduce = 0;
2117 }
2118
2119 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
2120 {
2121     HTTP_Forwarded(gdu->get());
2122
2123     int reduce = 0;
2124
2125     if (m_request_no == 1)
2126     {
2127         bool block = false;
2128
2129         connect_stat(block, reduce);
2130
2131         if (block)
2132         {
2133             m_timeout_mode = timeout_busy;
2134             timeout(0);
2135             return;
2136         }
2137     }
2138
2139     int bw_total = m_bw_stat.get_total();
2140     int pdu_total = m_pdu_stat.get_total();
2141     int search_total = m_search_stat.get_total();
2142
2143     assert(m_timeout_mode == timeout_busy);
2144     assert(m_timeout_gdu == 0);
2145
2146     if (m_search_max)
2147         reduce += search_total / m_search_max;
2148     if (m_bw_max)
2149         reduce += (bw_total/m_bw_max);
2150     if (m_pdu_max)
2151     {
2152         if (pdu_total > m_pdu_max)
2153         {
2154             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
2155             reduce = (reduce > nreduce) ? reduce : nreduce;
2156         }
2157     }
2158     m_http_version = 0;
2159
2160 #if 0
2161     /* uncomment to force a big reduce */
2162     m_timeout_mode = timeout_reduce;
2163     m_timeout_gdu = gdu;
2164     timeout(3);       // call us reduce seconds later
2165     return;
2166 #endif
2167     if (reduce)
2168     {
2169         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d search=%d limit-bw=%d limit-pdu=%d limit-search=%d",
2170                 m_session_str, reduce, bw_total, pdu_total, search_total,
2171                 m_bw_max, m_pdu_max, m_search_max);
2172
2173         m_timeout_mode = timeout_reduce;
2174         m_timeout_gdu = gdu;
2175         timeout(reduce);       // call us reduce seconds later
2176     }
2177     else
2178         recv_GDU_normal(gdu);
2179 }
2180
2181 void Yaz_Proxy::recv_GDU_more(bool normal)
2182 {
2183     GDU *g;
2184     if (normal && m_timeout_mode == timeout_busy)
2185         m_timeout_mode = timeout_normal;
2186     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
2187     {
2188         m_timeout_mode = timeout_busy;
2189         inc_ref();
2190         recv_GDU_reduce(g);
2191         if (dec_ref())
2192             break;
2193     }
2194 }
2195
2196 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
2197 {
2198     Z_GDU *apdu = 0;
2199     gdu->move_away_gdu(odr_decode(), &apdu);
2200     delete gdu;
2201
2202     if (apdu->which == Z_GDU_Z3950)
2203         handle_incoming_Z_PDU(apdu->u.z3950);
2204     else if (apdu->which == Z_GDU_HTTP_Request)
2205         handle_incoming_HTTP(apdu->u.HTTP_Request);
2206 }
2207
2208 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
2209 {
2210     if (m_max_record_retrieve)
2211     {
2212         if (apdu->which == Z_APDU_presentRequest)
2213         {
2214             Z_PresentRequest *pr = apdu->u.presentRequest;
2215             if (pr->numberOfRecordsRequested &&
2216                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
2217                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
2218         }
2219     }
2220 }
2221
2222 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
2223 {
2224     if (apdu->which == Z_APDU_initRequest)
2225     {
2226         if (m_initRequest_options &&
2227             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
2228             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
2229         {
2230             // There is no negotiation proposal from
2231             // client's side. OK. The proxy negotiation
2232             // in use, only.
2233             Z_InitRequest *initRequest = apdu->u.initRequest;
2234             Z_OtherInformation **otherInfo;
2235             Z_OtherInformationUnit *oi;
2236             get_otherInfoAPDU(apdu, &otherInfo);
2237             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2238             if (oi)
2239             {
2240                 ODR_MASK_SET(initRequest->options,
2241                     Z_Options_negotiationModel);
2242                 oi->which = Z_OtherInfo_externallyDefinedInfo;
2243                 oi->information.externallyDefinedInfo =
2244                 yaz_set_proposal_charneg(odr_encode(),
2245                     (const char**)&m_proxy_negotiation_charset,
2246                     m_proxy_negotiation_charset ? 1:0,
2247                     (const char**)&m_proxy_negotiation_lang,
2248                     m_proxy_negotiation_lang ? 1:0,
2249                     1);
2250             }
2251         }
2252         else if (m_initRequest_options &&
2253                  ODR_MASK_GET(m_initRequest_options,
2254                               Z_Options_negotiationModel) &&
2255                  m_charset_converter->get_target_query_charset())
2256         {
2257             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
2258                     m_session_str,
2259                     m_charset_converter->get_target_query_charset());
2260             Z_InitRequest *initRequest = apdu->u.initRequest;
2261             Z_CharSetandLanguageNegotiation *negotiation =
2262                 yaz_get_charneg_record (initRequest->otherInfo);
2263             if (negotiation &&
2264                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
2265             {
2266                 NMEM nmem = nmem_create();
2267                 char **charsets = 0;
2268                 int num_charsets = 0;
2269                 char **langs = 0;
2270                 int num_langs = 0;
2271                 int selected = 0;
2272                 yaz_get_proposal_charneg (nmem, negotiation,
2273                                           &charsets, &num_charsets,
2274                                           &langs, &num_langs, &selected);
2275                 int i;
2276                 for (i = 0; i<num_charsets; i++)
2277                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
2278                             charsets[i]);
2279                 for (i = 0; i<num_langs; i++)
2280                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
2281                             langs[i]);
2282
2283                 const char *t_charset =
2284                     m_charset_converter->get_target_query_charset();
2285                 // sweep through charsets and pick the first supported
2286                 // conversion
2287                 for (i = 0; i<num_charsets; i++)
2288                 {
2289                     const char *c_charset = charsets[i];
2290                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
2291                         break;
2292                 }
2293                 if (i != num_charsets)
2294                 {
2295                     // got one .. set up ODR for reverse direction
2296                     const char *c_charset = charsets[i];
2297                     odr_set_charset(odr_encode(), c_charset, t_charset);
2298                     m_charset_converter->set_client_query_charset(c_charset);
2299                     m_charset_converter->set_client_charset_selected(selected);
2300                 }
2301                 nmem_destroy(nmem);
2302                 ODR_MASK_CLEAR(m_initRequest_options,
2303                                Z_Options_negotiationModel);
2304                 yaz_del_charneg_record(&initRequest->otherInfo);
2305             }
2306             else
2307             {
2308                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
2309                         m_session_str);
2310             }
2311         }
2312         else if (m_charset_converter->get_target_query_charset() &&
2313             m_proxy_negotiation_default_charset)
2314         {
2315             m_charset_converter->
2316                 set_client_query_charset(m_proxy_negotiation_default_charset);
2317         }
2318     }
2319     else if (apdu->which == Z_APDU_initResponse)
2320     {
2321         Z_InitResponse *initResponse = apdu->u.initResponse;
2322         Z_OtherInformation **otherInfo;
2323         get_otherInfoAPDU(apdu, &otherInfo);
2324
2325         Z_CharSetandLanguageNegotiation *charneg = 0;
2326
2327         if (otherInfo && *otherInfo &&
2328             ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel)
2329             && (charneg = yaz_get_charneg_record(*otherInfo)))
2330         {
2331             char *charset = 0;
2332             char *lang = 0;
2333             int selected = 0;
2334
2335             yaz_get_response_charneg(m_referenceId_mem, charneg,
2336                 &charset, &lang, &selected);
2337
2338             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
2339                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
2340
2341             if (m_initRequest_options &&
2342                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
2343             {
2344                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
2345                     m_session_str);
2346             }
2347             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2348             {
2349                 // negotiation-charset, negotiation-lang
2350                 // elements of config file in use.
2351
2352                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2353                     m_session_str);
2354
2355                 // clear negotiation option.
2356                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2357
2358                 // Delete negotiation (charneg-3) entry.
2359                 yaz_del_charneg_record(otherInfo);
2360             }
2361         }
2362         else
2363         {
2364             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2365             {
2366                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2367                         m_session_str);
2368             }
2369             else if (m_charset_converter->get_client_query_charset())
2370             {
2371                 Z_OtherInformation **otherInfo;
2372                 Z_OtherInformationUnit *oi;
2373                 get_otherInfoAPDU(apdu, &otherInfo);
2374                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2375                 if (oi)
2376                 {
2377                     ODR_MASK_SET(initResponse->options,
2378                                  Z_Options_negotiationModel);
2379                     if (m_initRequest_options)
2380                         ODR_MASK_SET(m_initRequest_options,
2381                                      Z_Options_negotiationModel);
2382
2383                     oi->which = Z_OtherInfo_externallyDefinedInfo;
2384                     oi->information.externallyDefinedInfo =
2385                         yaz_set_response_charneg(
2386                             odr_encode(),
2387                             m_charset_converter->get_client_query_charset(),
2388                             0 /* no lang */,
2389                             m_charset_converter->get_client_charset_selected());
2390                 }
2391             }
2392         }
2393     }
2394 }
2395
2396 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2397                                                      int error,
2398                                                      const char *addinfo)
2399 {
2400     Z_Records *rec = (Z_Records *)
2401         odr_malloc (odr, sizeof(*rec));
2402     Odr_int *err = (Odr_int *)
2403         odr_malloc (odr, sizeof(*err));
2404     Z_DiagRec *drec = (Z_DiagRec *)
2405         odr_malloc (odr, sizeof(*drec));
2406     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2407         odr_malloc (odr, sizeof(*dr));
2408     *err = error;
2409     rec->which = Z_Records_NSD;
2410     rec->u.nonSurrogateDiagnostic = dr;
2411     dr->diagnosticSetId = odr_oiddup(odr, yaz_oid_diagset_bib_1);
2412     dr->condition = err;
2413     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2414     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2415     return rec;
2416 }
2417
2418 Z_ListEntries *Yaz_Proxy::create_nonSurrogateDiagnostics2(ODR odr,
2419                                                      int error,
2420                                                      const char *addinfo)
2421 {
2422     Z_ListEntries *rec = (Z_ListEntries *)
2423         odr_malloc (odr, sizeof(*rec));
2424     Odr_int *err = (Odr_int *)
2425         odr_malloc (odr, sizeof(*err));
2426     Z_DiagRec *drec = (Z_DiagRec *)
2427         odr_malloc (odr, sizeof(*drec));
2428     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2429         odr_malloc (odr, sizeof(*dr));
2430     *err = error;
2431     drec->which = Z_DiagRec_defaultFormat;
2432     drec->u.defaultFormat = dr;
2433     rec->num_entries = 0;
2434     rec->entries = NULL;
2435     rec->num_nonsurrogateDiagnostics = 1;
2436     rec->nonsurrogateDiagnostics =
2437       (Z_DiagRec **)odr_malloc(odr, sizeof(Z_DiagRec *));
2438     rec->nonsurrogateDiagnostics[0] = drec;
2439     dr->diagnosticSetId = odr_oiddup(odr, yaz_oid_diagset_bib_1);
2440     dr->condition = err;
2441     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2442     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2443     return rec;
2444 }
2445
2446 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2447 {
2448     if (apdu->which == Z_APDU_searchRequest &&
2449         apdu->u.searchRequest->query &&
2450         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2451         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2452     {
2453         Z_RPNQuery *rpnquery = 0;
2454         Z_SearchRequest *sr = apdu->u.searchRequest;
2455         char *addinfo = 0;
2456
2457         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2458                 sr->query->u.type_104->u.cql);
2459
2460         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2461                                           &rpnquery, odr_encode(),
2462                                           &addinfo);
2463         if (r == -3)
2464             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2465         else if (r)
2466         {
2467             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2468             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2469
2470             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2471             new_apdu->u.searchResponse->records =
2472                 create_nonSurrogateDiagnostics(odr_encode(),
2473                                                yaz_diag_srw_to_bib1(r),
2474                                                addinfo);
2475             *new_apdu->u.searchResponse->searchStatus = 0;
2476
2477             send_to_client(new_apdu);
2478
2479             return 0;
2480         }
2481         else
2482         {
2483             sr->query->which = Z_Query_type_1;
2484             sr->query->u.type_1 = rpnquery;
2485         }
2486         return apdu;
2487     }
2488     else if (apdu->which == Z_APDU_scanRequest)
2489     {
2490         Z_RPNQuery *rpnquery = 0;
2491         Z_ScanRequest *sr = apdu->u.scanRequest;
2492         char *addinfo = 0;
2493
2494         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2495                 sr->termListAndStartPoint->term->u.characterString);
2496
2497         int r = m_cql2rpn.query_transform(sr->termListAndStartPoint->term->u.characterString,
2498                                           &rpnquery, odr_encode(),
2499                                           &addinfo);
2500         if (r == -3)
2501             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2502         else if (r)
2503         {
2504             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2505             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_scanResponse);
2506
2507             new_apdu->u.scanResponse->referenceId = sr->referenceId;
2508             new_apdu->u.scanResponse->entries =
2509                 create_nonSurrogateDiagnostics2(odr_encode(),
2510                                                yaz_diag_srw_to_bib1(r),
2511                                                addinfo);
2512             *new_apdu->u.scanResponse->scanStatus = Z_Scan_failure;
2513
2514             send_to_client(new_apdu);
2515
2516             return 0;
2517         }
2518         else
2519         {
2520             sr->attributeSet = rpnquery->attributeSetId;
2521             if (rpnquery->RPNStructure->which == Z_RPNStructure_simple)
2522                 sr->termListAndStartPoint = rpnquery->RPNStructure->u.simple->u.attributesPlusTerm;
2523         }
2524         return apdu;
2525     }
2526     return apdu;
2527 }
2528
2529 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2530 {
2531     if (apdu->which == Z_APDU_searchRequest &&
2532         apdu->u.searchRequest->query)
2533     {
2534         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2535             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2536         {
2537             if (m_http_version)
2538                 m_charset_converter->set_client_query_charset("UTF-8");
2539             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2540             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2541         }
2542     }
2543     else if (apdu->which == Z_APDU_scanRequest &&
2544         apdu->u.scanRequest->termListAndStartPoint)
2545     {
2546         if (apdu->u.scanRequest->termListAndStartPoint->term)
2547             if (m_http_version)
2548                 m_charset_converter->set_client_query_charset("UTF-8");
2549             Z_Term *term = apdu->u.scanRequest->termListAndStartPoint->term;
2550             m_charset_converter->convert_term(term, odr_encode());
2551     }
2552     return apdu;
2553 }
2554
2555
2556 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2557 {
2558     if (apdu->which == Z_APDU_searchRequest)
2559     {
2560         Z_SearchRequest *sr = apdu->u.searchRequest;
2561         int err = 0;
2562         char *addinfo = 0;
2563
2564         Yaz_ProxyConfig *cfg = check_reconfigure();
2565         if (cfg)
2566             err = cfg->check_query(odr_encode(), m_default_target,
2567                                    sr->query, &addinfo);
2568         if (err)
2569         {
2570             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2571
2572             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2573             new_apdu->u.searchResponse->records =
2574                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2575             *new_apdu->u.searchResponse->searchStatus = 0;
2576
2577             send_to_client(new_apdu);
2578
2579             return 0;
2580         }
2581     }
2582     else if (apdu->which == Z_APDU_scanRequest)
2583     {
2584         Z_ScanRequest *sr = apdu->u.scanRequest;
2585         int err = 0;
2586         char *addinfo = 0;
2587
2588 #if 0
2589         Yaz_ProxyConfig *cfg = check_reconfigure();
2590 // Something like this needs to be implemented later:
2591         if (cfg)
2592             err = cfg->check_type_1_attributes(odr_encode(), m_default_target,
2593                                    sr->termListAndStartPoint->attributes, &addinfo);
2594 #endif
2595         if (err)
2596         {
2597             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_scanResponse);
2598
2599             new_apdu->u.scanResponse->referenceId = sr->referenceId;
2600             new_apdu->u.scanResponse->entries =
2601                 create_nonSurrogateDiagnostics2(odr_encode(), err, addinfo);
2602             *new_apdu->u.scanResponse->scanStatus = Z_Scan_failure;
2603
2604             send_to_client(new_apdu);
2605
2606             return 0;
2607         }
2608     }
2609
2610     return apdu;
2611 }
2612
2613 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2614 {
2615     if (apdu->which != Z_APDU_initRequest)
2616         return 1;  // pass if no init request
2617     Z_InitRequest *req = apdu->u.initRequest;
2618
2619     Yaz_ProxyConfig *cfg = check_reconfigure();
2620     if (!cfg)
2621         return 1;  // pass if no config
2622
2623     int ret;
2624     if (req->idAuthentication == 0)
2625     {
2626         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2627                                          m_peername);
2628     }
2629     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2630     {
2631         ret = cfg->client_authentication(
2632             m_default_target,
2633             req->idAuthentication->u.idPass->userId,
2634             req->idAuthentication->u.idPass->groupId,
2635             req->idAuthentication->u.idPass->password,
2636             m_peername);
2637     }
2638     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2639     {
2640         char user[64], pass[64];
2641         *user = '\0';
2642         *pass = '\0';
2643         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2644         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2645                                          m_peername);
2646     }
2647     else
2648         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2649                                          m_peername);
2650     return ret;
2651 }
2652
2653 int Yaz_Proxy::handle_global_authentication(Z_APDU *apdu)
2654 {
2655     if (apdu->which != Z_APDU_initRequest)
2656         return 1;  // pass if no init request
2657     Z_InitRequest *req = apdu->u.initRequest;
2658
2659     Yaz_ProxyConfig *cfg = check_reconfigure();
2660     if (!cfg)
2661         return 1;  // pass if no config
2662
2663     int ret;
2664     if (req->idAuthentication == 0)
2665     {
2666         ret = cfg->global_client_authentication(0, 0, 0,
2667                                                 m_peername);
2668     }
2669     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2670     {
2671         ret = cfg->global_client_authentication(
2672             req->idAuthentication->u.idPass->userId,
2673             req->idAuthentication->u.idPass->groupId,
2674             req->idAuthentication->u.idPass->password,
2675             m_peername);
2676     }
2677     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2678     {
2679         char user[64], pass[64];
2680         *user = '\0';
2681         *pass = '\0';
2682         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2683         ret = cfg->global_client_authentication(user, 0, pass,
2684                                                 m_peername);
2685     }
2686     else
2687         ret = cfg->global_client_authentication(0, 0, 0, m_peername);
2688     return ret;
2689 }
2690
2691 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2692 {
2693     m_marcxml_mode = none;
2694     if (apdu->which == Z_APDU_searchRequest)
2695     {
2696         Z_SearchRequest *sr = apdu->u.searchRequest;
2697         int err = 0;
2698         char *addinfo = 0;
2699         Yaz_ProxyConfig *cfg = check_reconfigure();
2700
2701         Z_RecordComposition rc_temp, *rc = 0;
2702         if (sr->smallSetElementSetNames)
2703         {
2704             rc_temp.which = Z_RecordComp_simple;
2705             rc_temp.u.simple = sr->smallSetElementSetNames;
2706             rc = &rc_temp;
2707         }
2708
2709         if (sr->preferredRecordSyntax)
2710             oid_oidcpy(m_frontend_type, sr->preferredRecordSyntax);
2711         else
2712             m_frontend_type[0] = -1;
2713
2714         char *stylesheet_name = 0;
2715         if (cfg)
2716             err = cfg->check_syntax(odr_encode(),
2717                                     m_default_target,
2718                                     sr->preferredRecordSyntax, rc,
2719                                     &addinfo, &stylesheet_name, &m_schema,
2720                                     &m_backend_type, &m_backend_charset,
2721                                     &m_usemarcon_ini_stage1,
2722                                     &m_usemarcon_ini_stage2,
2723                                     &m_backend_elementset);
2724         if (stylesheet_name)
2725         {
2726             m_parent->low_socket_close();
2727
2728 #if YAZ_HAVE_XSLT
2729             if (m_stylesheet_xsp)
2730                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2731             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2732                                                        stylesheet_name);
2733 #endif
2734             m_stylesheet_offset = 0;
2735             xfree(stylesheet_name);
2736
2737             m_parent->low_socket_open();
2738         }
2739         if (err == -1)
2740         {
2741             sr->smallSetElementSetNames = 0;
2742             sr->mediumSetElementSetNames = 0;
2743             m_marcxml_mode = marcxml;
2744             sr->preferredRecordSyntax =
2745                 yaz_string_to_oid_odr(
2746                     yaz_oid_std(), CLASS_RECSYN,
2747                     m_backend_type ? m_backend_type : "usmarc",
2748                     odr_encode());
2749         }
2750         else if (err)
2751         {
2752             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2753
2754             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2755             new_apdu->u.searchResponse->records =
2756                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2757             *new_apdu->u.searchResponse->searchStatus = 0;
2758
2759             send_to_client(new_apdu);
2760
2761             return 0;
2762         }
2763         else if (m_backend_type)
2764         {
2765             sr->preferredRecordSyntax =
2766                 yaz_string_to_oid_odr(yaz_oid_std(), CLASS_RECSYN,
2767                                       m_backend_type, odr_encode());
2768         }
2769         if (m_backend_elementset)
2770         {
2771             Z_ElementSetNames *esn =
2772                 mk_esn_from_schema(
2773                     odr_encode(),
2774                     *m_backend_elementset ? m_backend_elementset : 0);
2775             sr->smallSetElementSetNames = esn;
2776             sr->mediumSetElementSetNames = esn;
2777         }
2778     }
2779     else if (apdu->which == Z_APDU_presentRequest)
2780     {
2781         Z_PresentRequest *pr = apdu->u.presentRequest;
2782         int err = 0;
2783         char *addinfo = 0;
2784         Yaz_ProxyConfig *cfg = check_reconfigure();
2785
2786         if (pr->preferredRecordSyntax)
2787             oid_oidcpy(m_frontend_type, pr->preferredRecordSyntax);
2788         else
2789             m_frontend_type[0] = -1;
2790
2791         char *stylesheet_name = 0;
2792         if (cfg)
2793             err = cfg->check_syntax(odr_encode(), m_default_target,
2794                                     pr->preferredRecordSyntax,
2795                                     pr->recordComposition,
2796                                     &addinfo, &stylesheet_name, &m_schema,
2797                                     &m_backend_type, &m_backend_charset,
2798                                     &m_usemarcon_ini_stage1,
2799                                     &m_usemarcon_ini_stage2,
2800                                     &m_backend_elementset);
2801         if (stylesheet_name)
2802         {
2803             m_parent->low_socket_close();
2804
2805 #if YAZ_HAVE_XSLT
2806             if (m_stylesheet_xsp)
2807                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2808             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2809                                                        stylesheet_name);
2810 #endif
2811             m_stylesheet_offset = 0;
2812             xfree(stylesheet_name);
2813
2814             m_parent->low_socket_open();
2815         }
2816         if (err == -1)
2817         {
2818             pr->recordComposition = 0;
2819             m_marcxml_mode = marcxml;
2820
2821             pr->preferredRecordSyntax =
2822                 yaz_string_to_oid_odr(
2823                     yaz_oid_std(), CLASS_RECSYN,
2824                     m_backend_type ? m_backend_type : "usmarc",
2825                     odr_encode());
2826         }
2827         else if (err)
2828         {
2829             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2830
2831             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2832             new_apdu->u.presentResponse->records =
2833                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2834             *new_apdu->u.presentResponse->presentStatus =
2835                 Z_PresentStatus_failure;
2836
2837             send_to_client(new_apdu);
2838
2839             return 0;
2840         }
2841         else if (m_backend_type)
2842         {
2843             pr->preferredRecordSyntax =
2844                 yaz_string_to_oid_odr(yaz_oid_std(),
2845                                       CLASS_RECSYN, m_backend_type,
2846                                       odr_encode());
2847         }
2848         if (m_backend_elementset)
2849         {
2850             Z_ElementSetNames *esn =
2851                 mk_esn_from_schema(
2852                     odr_encode(),
2853                     *m_backend_elementset ? m_backend_elementset : 0);
2854             Z_RecordComposition *comp = (Z_RecordComposition *)
2855                 odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
2856             comp->which = Z_RecordComp_simple;
2857             comp->u.simple = esn;
2858             pr->recordComposition = comp;
2859         }
2860     }
2861     return apdu;
2862 }
2863
2864 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2865 {
2866     if (!schema)
2867         return 0;
2868     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2869         odr_malloc(o, sizeof(Z_ElementSetNames));
2870     esn->which = Z_ElementSetNames_generic;
2871     esn->u.generic = odr_strdup(o, schema);
2872     return esn;
2873 }
2874
2875 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2876 {
2877     const char *t = 0;
2878     Yaz_ProxyConfig *cfg = check_reconfigure();
2879     if (cfg)
2880         t = cfg->get_explain_name(db, backend_db);
2881
2882     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2883     {
2884         releaseClient();
2885     }
2886
2887     if (t)
2888     {
2889         xfree(m_default_target);
2890         m_default_target = xstrdup(t);
2891     }
2892 }
2893
2894 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2895 {
2896     struct stat sbuf;
2897     if (strcmp(hreq->method, "GET"))
2898         return 0;
2899     if (hreq->path[0] != '/')
2900         return 0;
2901     const char *cp = hreq->path;
2902     while (*cp)
2903     {
2904         if (*cp == '/' && strchr("/.", cp[1]))
2905             return 0;
2906         cp++;
2907     }
2908
2909     Yaz_ProxyConfig *cfg = check_reconfigure();
2910
2911     if (!cfg->get_file_access_info(hreq->path+1))
2912         return 0;
2913
2914     const char *fname = hreq->path+1;
2915     if (stat(fname, &sbuf))
2916     {
2917         yaz_log(YLOG_LOG|YLOG_ERRNO, "%sstat failed for %s", m_session_str,
2918                 fname);
2919         return 0;
2920     }
2921     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2922     {
2923         yaz_log(YLOG_LOG, "%sNot a regular file %s", m_session_str, fname);
2924         return 0;
2925     }
2926     if (sbuf.st_size > (off_t) 1000000)
2927     {
2928         yaz_log(YLOG_WARN, "%sFile %s too large for transfer", m_session_str,
2929                 fname);
2930         return 0;
2931     }
2932
2933     ODR o = odr_encode();
2934
2935     const char *ctype = cfg->check_mime_type(fname);
2936     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2937     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2938     if (m_http_version)
2939         hres->version = odr_strdup(o, m_http_version);
2940     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2941     if (m_http_keepalive)
2942         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2943     else
2944         timeout(0);
2945
2946     hres->content_len = sbuf.st_size;
2947     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2948     FILE *f = fopen(fname, "rb");
2949     if (f)
2950     {
2951         fread(hres->content_buf, 1, hres->content_len, f);
2952         fclose(f);
2953     }
2954     else
2955     {
2956         return 0;
2957     }
2958     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2959     {
2960         yaz_log(YLOG_LOG, "%sSending file %s to client", m_session_str,
2961                  fname);
2962     }
2963     int len;
2964     send_GDU(gdu, &len);
2965     recv_GDU_more(true);
2966     return 1;
2967 }
2968
2969 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2970 {
2971     if (m_s2z_odr_init)
2972     {
2973         odr_destroy(m_s2z_odr_init);
2974         m_s2z_odr_init = 0;
2975     }
2976     if (m_s2z_odr_search)
2977     {
2978         odr_destroy(m_s2z_odr_search);
2979         m_s2z_odr_search = 0;
2980     }
2981     if (m_s2z_odr_scan)
2982     {
2983         odr_destroy(m_s2z_odr_scan);
2984         m_s2z_odr_scan = 0;
2985     }
2986
2987     m_http_keepalive = 0;
2988     m_http_version = 0;
2989     if (!strcmp(hreq->version, "1.0"))
2990     {
2991         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2992         if (v && !strcmp(v, "Keep-Alive"))
2993             m_http_keepalive = 1;
2994         else
2995             m_http_keepalive = 0;
2996         m_http_version = "1.0";
2997     }
2998     else
2999     {
3000         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
3001         if (v && !strcmp(v, "close"))
3002             m_http_keepalive = 0;
3003         else
3004             m_http_keepalive = 1;
3005         m_http_version = "1.1";
3006     }
3007
3008     const char *a = z_HTTP_header_lookup(hreq->headers, "Authorization");
3009     char authorization_str[255];
3010     *authorization_str = '\0';
3011     if (a && strncasecmp(a, "Basic ", 6) == 0)
3012         base64_decode(a + 6, authorization_str, 254);
3013
3014     Z_SRW_PDU *srw_pdu = 0;
3015     Z_SOAP *soap_package = 0;
3016     char *charset = 0;
3017     Z_SRW_diagnostic *diagnostic = 0;
3018     int num_diagnostic = 0;
3019
3020     yaz_log(YLOG_LOG, "%s%s %s", m_session_str, hreq->method, hreq->path);
3021
3022     if (file_access(hreq))
3023     {
3024         return;
3025     }
3026     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
3027                             &charset) == 0
3028              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
3029                                &charset, &diagnostic, &num_diagnostic) == 0)
3030     {
3031         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
3032         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
3033         m_s2z_odr_scan = odr_createmem(ODR_ENCODE);
3034         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
3035         m_s2z_init_apdu = 0;
3036         m_s2z_search_apdu = 0;
3037         m_s2z_present_apdu = 0;
3038         m_s2z_scan_apdu = 0;
3039
3040         m_s2z_stylesheet = 0;
3041
3042         Z_IdAuthentication *auth = NULL;
3043         if (srw_pdu->username && !srw_pdu->password)
3044         {
3045             yaz_log(YLOG_LOG, "username: %s\n", srw_pdu->username);
3046             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
3047             auth->which = Z_IdAuthentication_open;
3048             auth->u.open = odr_strdup(m_s2z_odr_init, srw_pdu->username);
3049         }
3050         else if (srw_pdu->username && srw_pdu->password)
3051         {
3052             yaz_log(YLOG_LOG, "username/password: %s/%s\n",
3053                     srw_pdu->username, srw_pdu->password);
3054             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
3055             auth->which = Z_IdAuthentication_idPass;
3056             auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
3057             auth->u.idPass->groupId = NULL;
3058             auth->u.idPass->password = odr_strdup(m_s2z_odr_init, srw_pdu->password);
3059             auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, srw_pdu->username);
3060         }
3061         else
3062         {
3063             if (*authorization_str)
3064             {
3065                 yaz_log(YLOG_LOG, "authorization_str present: %s\n", authorization_str);
3066                 auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
3067                 auth->which = Z_IdAuthentication_idPass;
3068                 auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
3069                 auth->u.idPass->groupId = NULL;
3070                 char *p = strchr(authorization_str, ':');
3071                 if (p)
3072                 {
3073                     *p = '\0';
3074                     p++;
3075                     auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
3076                 }
3077                 auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
3078             }
3079         }
3080
3081         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
3082         {
3083
3084             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
3085
3086             const char *backend_db = srw_req->database;
3087             srw_get_client(srw_req->database, &backend_db);
3088
3089             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
3090             // recordXPath unsupported.
3091             if (srw_req->recordXPath)
3092             {
3093                 yaz_add_srw_diagnostic(odr_decode(),
3094                                        &diagnostic, &num_diagnostic,
3095                                        72, 0);
3096             }
3097             // sort unsupported
3098             if (srw_req->sort_type != Z_SRW_sort_type_none)
3099             {
3100                 yaz_add_srw_diagnostic(odr_decode(),
3101                                        &diagnostic, &num_diagnostic,
3102                                        80, 0);
3103             }
3104             // save stylesheet
3105             if (srw_req->stylesheet)
3106                 m_s2z_stylesheet =
3107                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
3108
3109             // set packing for response records ..
3110             if (srw_req->recordPacking &&
3111                 !strcmp(srw_req->recordPacking, "xml"))
3112                 m_s2z_packing = Z_SRW_recordPacking_XML;
3113             else
3114                 m_s2z_packing = Z_SRW_recordPacking_string;
3115
3116             if (num_diagnostic)
3117             {
3118                 Z_SRW_PDU *srw_pdu =
3119                     yaz_srw_get(odr_encode(),
3120                                 Z_SRW_searchRetrieve_response);
3121                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
3122
3123                 srw_res->diagnostics = diagnostic;
3124                 srw_res->num_diagnostics = num_diagnostic;
3125                 send_srw_response(srw_pdu);
3126                 return;
3127             }
3128
3129             // prepare search PDU
3130             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
3131                                           Z_APDU_searchRequest);
3132             Z_SearchRequest *z_searchRequest =
3133                 m_s2z_search_apdu->u.searchRequest;
3134
3135             z_searchRequest->num_databaseNames = 1;
3136             z_searchRequest->databaseNames = (char**)
3137                 odr_malloc(m_s2z_odr_search, sizeof(char *));
3138             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
3139                                                            backend_db);
3140
3141             // query transformation
3142             Z_Query *query = (Z_Query *)
3143                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
3144             z_searchRequest->query = query;
3145
3146             if (
3147 #ifdef Z_SRW_query_type_cql
3148                 srw_req->query_type == Z_SRW_query_type_cql
3149 #else
3150                 !srw_req->queryType || !strcmp(srw_req->queryType, "cql")
3151 #endif
3152                 )
3153             {
3154                 Z_External *ext = (Z_External *)
3155                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
3156                 ext->direct_reference =
3157                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
3158                 ext->indirect_reference = 0;
3159                 ext->descriptor = 0;
3160                 ext->which = Z_External_CQL;
3161 #ifdef Z_SRW_query_type_cql
3162                 ext->u.cql = srw_req->query.cql;
3163 #else
3164                 ext->u.cql = srw_req->query;
3165 #endif
3166
3167                 query->which = Z_Query_type_104;
3168                 query->u.type_104 =  ext;
3169             }
3170             else if (
3171 #ifdef Z_SRW_query_type_pqf
3172                 srw_req->query_type == Z_SRW_query_type_pqf
3173 #else
3174                 srw_req->queryType && !strcmp(srw_req->queryType, "pqf")
3175 #endif
3176                 )
3177             {
3178                 Z_RPNQuery *RPNquery;
3179                 YAZ_PQF_Parser pqf_parser;
3180
3181                 pqf_parser = yaz_pqf_create ();
3182
3183                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
3184 #ifdef Z_SRW_query_type_pqf
3185                                           srw_req->query.pqf
3186 #else
3187                                           srw_req->query
3188 #endif
3189                     );
3190                 if (!RPNquery)
3191                 {
3192                     const char *pqf_msg;
3193                     size_t off;
3194                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
3195                     int ioff = off;
3196                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
3197                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
3198
3199                     send_to_srw_client_error(10, 0);
3200                     return;
3201                 }
3202                 query->which = Z_Query_type_1;
3203                 query->u.type_1 =  RPNquery;
3204
3205                 yaz_pqf_destroy (pqf_parser);
3206             }
3207             else
3208             {
3209                 send_to_srw_client_error(7, "query");
3210                 return;
3211             }
3212
3213             // present
3214             m_s2z_present_apdu = 0;
3215             int max = 0;
3216             if (srw_req->maximumRecords)
3217                 max = *srw_req->maximumRecords;
3218             int start = 1;
3219             if (srw_req->startRecord)
3220                 start = *srw_req->startRecord;
3221             if (max > 0)
3222             {
3223                 // Some backend, such as Voyager doesn't honor piggyback
3224                 // So we use present always (0 &&).
3225                 if (0 && start <= 1)  // Z39.50 piggyback
3226                 {
3227                     *z_searchRequest->smallSetUpperBound = max;
3228                     *z_searchRequest->mediumSetPresentNumber = max;
3229                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
3230
3231                     z_searchRequest->preferredRecordSyntax =
3232                         odr_oiddup(m_s2z_odr_search, yaz_oid_recsyn_xml);
3233
3234                     if (srw_req->recordSchema)
3235                     {
3236                         z_searchRequest->smallSetElementSetNames =
3237                             z_searchRequest->mediumSetElementSetNames =
3238                             mk_esn_from_schema(m_s2z_odr_search,
3239                                                srw_req->recordSchema);
3240                     }
3241                 }
3242                 else   // Z39.50 present
3243                 {
3244                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
3245                                                    Z_APDU_presentRequest);
3246                     Z_PresentRequest *z_presentRequest =
3247                         m_s2z_present_apdu->u.presentRequest;
3248                     *z_presentRequest->resultSetStartPoint = start;
3249                     *z_presentRequest->numberOfRecordsRequested = max;
3250
3251                     z_presentRequest->preferredRecordSyntax =
3252                         odr_oiddup(m_s2z_odr_search, yaz_oid_recsyn_xml);
3253                     if (srw_req->recordSchema)
3254                     {
3255                         z_presentRequest->recordComposition =
3256                             (Z_RecordComposition *)
3257                             odr_malloc(m_s2z_odr_search,
3258                                        sizeof(Z_RecordComposition));
3259                         z_presentRequest->recordComposition->which =
3260                             Z_RecordComp_simple;
3261                         z_presentRequest->recordComposition->u.simple =
3262                             mk_esn_from_schema(m_s2z_odr_search,
3263                                                srw_req->recordSchema);
3264                     }
3265                 }
3266             }
3267             if (!m_client)
3268             {
3269                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
3270                                             Z_APDU_initRequest);
3271
3272                 ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan);
3273                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
3274
3275                 // prevent m_initRequest_apdu memory from being grabbed
3276                 // in Yaz_Proxy::handle_incoming_Z_PDU
3277                 m_initRequest_apdu = m_s2z_init_apdu;
3278                 handle_incoming_Z_PDU(m_s2z_init_apdu);
3279                 return;
3280             }
3281             else
3282             {
3283                 handle_incoming_Z_PDU(m_s2z_search_apdu);
3284                 return;
3285             }
3286         }
3287         else if (srw_pdu->which == Z_SRW_explain_request)
3288         {
3289             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
3290
3291             const char *backend_db = srw_req->database;
3292             srw_get_client(srw_req->database, &backend_db);
3293
3294             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
3295
3296             // save stylesheet
3297             if (srw_req->stylesheet)
3298                 m_s2z_stylesheet =
3299                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
3300
3301             if (srw_req->recordPacking &&
3302                 !strcmp(srw_req->recordPacking, "xml"))
3303                 m_s2z_packing = Z_SRW_recordPacking_XML;
3304             else
3305                 m_s2z_packing = Z_SRW_recordPacking_string;
3306
3307             if (num_diagnostic)
3308             {
3309                 send_srw_explain_response(diagnostic, num_diagnostic);
3310                 return;
3311             }
3312
3313             if (!m_client)
3314             {
3315                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
3316                                             Z_APDU_initRequest);
3317
3318                 ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan);
3319                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
3320
3321                 // prevent m_initRequest_apdu memory from being grabbed
3322                 // in Yaz_Proxy::handle_incoming_Z_PDU
3323                 m_initRequest_apdu = m_s2z_init_apdu;
3324                 handle_incoming_Z_PDU(m_s2z_init_apdu);
3325             }
3326             else
3327                 send_srw_explain_response(0, 0);
3328             return;
3329         }
3330         else if (srw_pdu->which == Z_SRW_scan_request)
3331         {
3332             Z_SRW_scanRequest *srw_req = srw_pdu->u.scan_request;
3333
3334             const char *backend_db = srw_req->database;
3335             srw_get_client(srw_req->database, &backend_db);
3336
3337             m_s2z_database = odr_strdup(m_s2z_odr_init,
3338                                         srw_req->database);
3339             // save stylesheet
3340             if (srw_req->stylesheet)
3341                 m_s2z_stylesheet =
3342                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
3343
3344             // prepare scan PDU
3345             m_s2z_scan_apdu = zget_APDU(m_s2z_odr_scan,
3346                                           Z_APDU_scanRequest);
3347             Z_ScanRequest *z_scanRequest =
3348                 m_s2z_scan_apdu->u.scanRequest;
3349
3350             z_scanRequest->num_databaseNames = 1;
3351             z_scanRequest->databaseNames = (char**)
3352                 odr_malloc(m_s2z_odr_scan, sizeof(char *));
3353             z_scanRequest->databaseNames[0] = odr_strdup(m_s2z_odr_scan,
3354                                                            backend_db);
3355
3356              // query transformation
3357             if (
3358 #ifdef Z_SRW_query_type_cql
3359                 srw_req->query_type == Z_SRW_query_type_cql
3360 #else
3361                 !srw_req->queryType || !strcmp(srw_req->queryType, "cql")
3362 #endif
3363                 )
3364             {
3365                 z_scanRequest->termListAndStartPoint =
3366                     (Z_AttributesPlusTerm *)odr_malloc(m_s2z_odr_scan, sizeof(Z_AttributesPlusTerm));
3367                 z_scanRequest->termListAndStartPoint->attributes = NULL;
3368                 z_scanRequest->termListAndStartPoint->term =
3369                    (Z_Term *)odr_malloc(m_s2z_odr_scan, sizeof(Z_Term));
3370                 z_scanRequest->termListAndStartPoint->term->which =
3371                   Z_Term_characterString;
3372                 z_scanRequest->termListAndStartPoint->term->u.characterString =
3373                   odr_strdup(m_s2z_odr_scan,
3374 #ifdef Z_SRW_query_type_cql
3375                              srw_req->scanClause.cql
3376 #else
3377                              srw_req->scanClause
3378 #endif
3379                       );
3380             }
3381
3382             if (srw_req->responsePosition)
3383                 z_scanRequest->preferredPositionInResponse =
3384                     odr_intdup(m_s2z_odr_scan, *srw_req->responsePosition);
3385             if (srw_req->maximumTerms)
3386                 *z_scanRequest->numberOfTermsRequested = *srw_req->maximumTerms;
3387
3388             if (!m_client)
3389             {
3390                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
3391                                             Z_APDU_initRequest);
3392
3393                 ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan);
3394                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
3395
3396                 // prevent m_initRequest_apdu memory from being grabbed
3397                 // in Yaz_Proxy::handle_incoming_Z_PDU
3398                 m_initRequest_apdu = m_s2z_init_apdu;
3399                 handle_incoming_Z_PDU(m_s2z_init_apdu);
3400                 return;
3401             }
3402             else
3403             {
3404                 handle_incoming_Z_PDU(m_s2z_scan_apdu);
3405                 return;
3406             }
3407         }
3408         else
3409         {
3410             m_s2z_database = 0;
3411
3412             send_to_srw_client_error(4, 0);
3413         }
3414     }
3415     send_http_response(400);
3416 }
3417
3418 void Yaz_Proxy::handle_init(Z_APDU *apdu)
3419 {
3420
3421     Z_OtherInformation **oi;
3422     get_otherInfoAPDU(apdu, &oi);
3423
3424     if (apdu->u.initRequest->implementationId)
3425         yaz_log(YLOG_LOG, "%simplementationId: %s",
3426                 m_session_str, apdu->u.initRequest->implementationId);
3427     if (apdu->u.initRequest->implementationName)
3428         yaz_log(YLOG_LOG, "%simplementationName: %s",
3429                 m_session_str, apdu->u.initRequest->implementationName);
3430     if (apdu->u.initRequest->implementationVersion)
3431         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
3432                 m_session_str, apdu->u.initRequest->implementationVersion);
3433     if (m_initRequest_apdu == 0)
3434     {
3435         if (m_initRequest_mem)
3436             nmem_destroy(m_initRequest_mem);
3437
3438         m_initRequest_apdu = apdu;
3439         m_initRequest_mem = odr_extract_mem(odr_decode());
3440
3441         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
3442             preferredMessageSize;
3443         *apdu->u.initRequest->preferredMessageSize = 64*1024*1024;
3444         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
3445             maximumRecordSize;
3446         *apdu->u.initRequest->maximumRecordSize = 64*1024*1024;
3447
3448         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
3449             yaz_get_charneg_record(*oi);
3450
3451         // Save proposal charsets and langs.
3452         if (ODR_MASK_GET(apdu->u.initRequest->options,
3453                          Z_Options_negotiationModel)
3454             && charSetandLangRecord)
3455         {
3456
3457             yaz_get_proposal_charneg(m_referenceId_mem,
3458                                      charSetandLangRecord,
3459                                      &m_initRequest_oi_negotiation_charsets,
3460                                      &m_initRequest_oi_negotiation_num_charsets,
3461                                      &m_initRequest_oi_negotiation_langs,
3462                                      &m_initRequest_oi_negotiation_num_langs,
3463                                      &m_initRequest_oi_negotiation_selected);
3464
3465             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
3466             {
3467                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
3468                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
3469                         m_initRequest_oi_negotiation_charsets[i]:"none");
3470             }
3471             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
3472             {
3473                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
3474                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
3475                         m_initRequest_oi_negotiation_langs[i]:"none");
3476             }
3477             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
3478                     m_session_str, m_initRequest_oi_negotiation_selected);
3479         }
3480         // save init options for the response..
3481         m_initRequest_options = apdu->u.initRequest->options;
3482
3483         apdu->u.initRequest->options =
3484             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
3485                                        sizeof(Odr_bitmask));
3486         ODR_MASK_ZERO(apdu->u.initRequest->options);
3487         int i;
3488         for (i = 0; i<= 24; i++)
3489             ODR_MASK_SET(apdu->u.initRequest->options, i);
3490         // check negotiation option
3491         if (!ODR_MASK_GET(m_initRequest_options,
3492                           Z_Options_negotiationModel))
3493         {
3494             ODR_MASK_CLEAR(apdu->u.initRequest->options,
3495                            Z_Options_negotiationModel);
3496         }
3497         ODR_MASK_CLEAR(apdu->u.initRequest->options,
3498                        Z_Options_concurrentOperations);
3499         // make new version
3500         m_initRequest_version = apdu->u.initRequest->protocolVersion;
3501         apdu->u.initRequest->protocolVersion =
3502             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
3503                                        sizeof(Odr_bitmask));
3504         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
3505
3506         for (i = 0; i<= 8; i++)
3507             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
3508     }
3509     handle_charset_lang_negotiation(apdu);
3510     if (m_client->m_init_flag)
3511     {
3512         if (handle_init_response_for_invalid_session(apdu))
3513             return;
3514         if (m_client->m_initResponse)
3515         {
3516             Z_APDU *apdu2 = m_client->m_initResponse;
3517             apdu2->u.initResponse->otherInfo = 0;
3518             if (m_client->m_cookie && *m_client->m_cookie)
3519                 set_otherInformationString(apdu2, yaz_oid_userinfo_cookie,
3520                                            1, m_client->m_cookie);
3521             apdu2->u.initResponse->referenceId =
3522                 apdu->u.initRequest->referenceId;
3523             apdu2->u.initResponse->options = m_client->m_initResponse_options;
3524             apdu2->u.initResponse->protocolVersion =
3525                 m_client->m_initResponse_version;
3526
3527             handle_charset_lang_negotiation(apdu2);
3528
3529             if (m_timeout_mode == timeout_busy)
3530                 m_timeout_mode = timeout_normal;
3531             send_to_client(apdu2);
3532             return;
3533         }
3534     }
3535     m_client->m_init_flag = 1;
3536
3537     if (m_num_msg_threads && m_my_thread)
3538     {
3539         Auth_Msg *m = new Auth_Msg;
3540         m->m_proxy = this;
3541         z_APDU(odr_encode(), &apdu, 0, "encode");
3542         char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
3543         m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
3544         memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
3545         odr_reset(odr_encode());
3546
3547         inc_ref();
3548         m_my_thread->put(m);
3549     }
3550     else
3551     {
3552         int ret = handle_authentication(apdu);
3553         result_authentication(apdu, ret);
3554     }
3555 }
3556
3557 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
3558 {
3559     Z_ReferenceId **refid = get_referenceIdP(apdu);
3560     nmem_reset(m_referenceId_mem);
3561     if (refid && *refid)
3562     {
3563         m_referenceId = (Z_ReferenceId *)
3564             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
3565         m_referenceId->len = (*refid)->len;
3566 #if YAZ_VERSIONL < 0x50000
3567         m_referenceId->size = m_referenceId->len;
3568         m_referenceId->buf = (unsigned char *)
3569             nmem_strdupn(m_referenceId_mem, (const char *)
3570                                     (*refid)->buf, (*refid)->len);
3571 #else
3572         m_referenceId->buf =
3573             nmem_strdupn(m_referenceId_mem, (*refid)->buf, (*refid)->len);
3574 #endif
3575     }
3576     else
3577         m_referenceId = 0;
3578
3579     if (!m_client && m_flag_invalid_session)
3580     {
3581         // Got request for a session that is invalid..
3582         m_apdu_invalid_session = apdu; // save package
3583         m_mem_invalid_session = odr_extract_mem(odr_decode());
3584         apdu = m_initRequest_apdu;     // but throw an init to the target
3585     }
3586
3587     if (apdu->which == Z_APDU_searchRequest)
3588         m_search_stat.add_bytes(1);
3589
3590     // Handle global authentication
3591     if (!handle_global_authentication(apdu))
3592     {
3593         if (m_http_version)
3594         {   // HTTP. Send unauthorized
3595             send_http_response(401);
3596             return;
3597         }
3598         else
3599         {
3600             // Z39.50 just shutdown
3601             timeout(0);
3602             return;
3603         }
3604         return;
3605     }
3606
3607     // Determine our client.
3608     Z_OtherInformation **oi;
3609     get_otherInfoAPDU(apdu, &oi);
3610     int http_code = 404;
3611     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi), &http_code);
3612     if (!m_client)
3613     {
3614         if (m_http_version)
3615         {   // HTTP. Send not found
3616             send_http_response(http_code);
3617             return;
3618         }
3619         else
3620         {
3621             // Z39.50 just shutdown
3622             timeout(0);
3623             return;
3624         }
3625     }
3626
3627     m_client->m_server = this;
3628
3629     if (apdu->which == Z_APDU_initRequest)
3630         handle_init(apdu);
3631     else
3632         handle_incoming_Z_PDU_2(apdu);
3633 }
3634
3635 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
3636 {
3637     handle_max_record_retrieve(apdu);
3638
3639     if (apdu)
3640         apdu = handle_syntax_validation(apdu);
3641
3642     if (apdu)
3643         apdu = handle_query_transformation(apdu);
3644
3645     if (apdu)
3646         apdu = handle_target_charset_conversion(apdu);
3647
3648     if (apdu)
3649         apdu = handle_query_validation(apdu);
3650
3651     if (apdu)
3652         apdu = result_set_optimize(apdu);
3653
3654     if (!apdu)
3655     {
3656         m_client->timeout(m_target_idletime);  // mark it active even
3657         recv_GDU_more(true);
3658         // though we didn't use it
3659         return;
3660     }
3661
3662     // delete other info construct completely if 0 elements
3663     Z_OtherInformation **oi;
3664     get_otherInfoAPDU(apdu, &oi);
3665     if (oi && *oi && (*oi)->num_elements == 0)
3666         *oi = 0;
3667
3668     if (apdu->which == Z_APDU_presentRequest &&
3669         m_client->m_resultSetStartPoint == 0)
3670     {
3671         Z_PresentRequest *pr = apdu->u.presentRequest;
3672         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3673         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3674     } else {
3675         m_client->m_resultSetStartPoint = 0;
3676     }
3677     if (m_client->send_to_target(apdu) < 0)
3678     {
3679         m_client->shutdown();
3680     }
3681     else
3682         m_client->m_waiting = 1;
3683 }
3684
3685 void Yaz_Proxy::connectNotify()
3686 {
3687 }
3688
3689 void Yaz_Proxy::releaseClient()
3690 {
3691     xfree(m_proxyTarget);
3692     m_proxyTarget = 0;
3693     m_flag_invalid_session = 0;
3694     // only keep if keep_alive flag is set...
3695     if (m_client &&
3696         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3697         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3698         m_client->m_waiting == 0)
3699     {
3700         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3701                  m_session_str,
3702                  m_client->get_hostname());
3703         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3704                 m_session_str, m_client->m_pdu_recv,
3705                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3706                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3707         assert (m_client->m_waiting != 2);
3708         // Tell client (if any) that no server connection is there..
3709         m_client->m_server = 0;
3710         m_client = 0;
3711     }
3712     else if (m_client)
3713     {
3714         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) close %s",
3715                  m_session_str,
3716                  m_client->get_hostname());
3717         assert (m_client->m_waiting != 2);
3718         delete m_client;
3719         m_client = 0;
3720     }
3721     else if (!m_parent)
3722     {
3723         yaz_log(YLOG_LOG, "%sshutdown (client to proxy) bad state",
3724                  m_session_str);
3725         assert (m_parent);
3726     }
3727     else
3728     {
3729         yaz_log(YLOG_LOG, "%sShutdown (client to proxy)",
3730                  m_session_str);
3731     }
3732     if (m_parent)
3733         m_parent->pre_init();
3734 }
3735
3736 bool Yaz_Proxy::dec_ref()
3737 {
3738     m_http_keepalive = 0;
3739
3740     --m_ref_count;
3741     if (m_ref_count > 0)
3742         return false;
3743
3744     releaseClient();
3745
3746     delete this;
3747     return true;
3748 }
3749
3750 const char *Yaz_ProxyClient::get_session_str()
3751 {
3752     if (!m_server)
3753         return "0 ";
3754     return m_server->get_session_str();
3755 }
3756
3757 void Yaz_ProxyClient::shutdown()
3758 {
3759     yaz_log(YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3760              get_hostname());
3761
3762     if (m_server)
3763     {
3764         m_waiting = 1;   // ensure it's released from Yaz_Proxy::releaseClient
3765         m_server->dec_ref();
3766     }
3767     else
3768         delete this;
3769 }
3770
3771 void Yaz_Proxy::failNotify()
3772 {
3773     inc_request_no();
3774     yaz_log(YLOG_LOG, "%sConnection closed by client", get_session_str());
3775     dec_ref();
3776 }
3777
3778 void Yaz_Proxy::send_response_fail_client(const char *addr)
3779 {
3780     if (m_http_version)
3781     {
3782         Z_SRW_diagnostic *diagnostic = 0;
3783         int num_diagnostic = 0;
3784
3785         yaz_add_srw_diagnostic(odr_encode(),
3786                                &diagnostic, &num_diagnostic,
3787                                YAZ_SRW_SYSTEM_TEMPORARILY_UNAVAILABLE, addr);
3788         if (m_s2z_search_apdu)
3789             send_srw_search_response(diagnostic, num_diagnostic);
3790         else if (m_s2z_scan_apdu)
3791             send_srw_scan_response(diagnostic, num_diagnostic);
3792         else
3793             send_srw_explain_response(diagnostic, num_diagnostic);
3794     }
3795 }
3796
3797 void Yaz_ProxyClient::failNotify()
3798 {
3799     if (m_server)
3800         m_server->inc_request_no();
3801     yaz_log(YLOG_LOG, "%sConnection closed by target %s",
3802              get_session_str(), get_hostname());
3803
3804     if (m_server)
3805         m_server->send_response_fail_client(get_hostname());
3806     shutdown();
3807 }
3808
3809 void Yaz_ProxyClient::connectNotify()
3810 {
3811     const char *s = get_session_str();
3812     const char *h = get_hostname();
3813     yaz_log(YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3814              m_target_idletime);
3815     timeout(m_target_idletime);
3816     if (!m_server)
3817         pre_init_client();
3818 }
3819
3820 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3821                                               *the_PDU_Observable, int fd)
3822 {
3823     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3824 }
3825
3826 Yaz_ProxyClient::~Yaz_ProxyClient()
3827 {
3828     if (m_prev)
3829         *m_prev = m_next;
3830     if (m_next)
3831         m_next->m_prev = m_prev;
3832     m_waiting = 2;     // for debugging purposes only.
3833     odr_destroy(m_init_odr);
3834     odr_destroy(m_idAuthentication_odr);
3835     delete m_last_query;
3836     xfree(m_last_resultSetId);
3837     xfree(m_cookie);
3838 }
3839
3840 void Yaz_ProxyClient::pre_init_client()
3841 {
3842     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3843     Z_InitRequest *req = apdu->u.initRequest;
3844
3845     int i;
3846     for (i = 0; i<= 24; i++)
3847         ODR_MASK_SET(req->options, i);
3848     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3849                    Z_Options_negotiationModel);
3850     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3851                    Z_Options_concurrentOperations);
3852     for (i = 0; i<= 10; i++)
3853         ODR_MASK_SET(req->protocolVersion, i);
3854
3855     if (send_to_target(apdu) < 0)
3856     {
3857         delete this;
3858     }
3859     else
3860     {
3861         m_waiting = 1;
3862         m_init_flag = 1;
3863     }
3864 }
3865
3866 void Yaz_Proxy::pre_init()
3867 {
3868     int i;
3869     const char *name = 0;
3870     const char *zurl_in_use[MAX_ZURL_PLEX];
3871     int limit_bw, limit_pdu, limit_req, limit_search;
3872     int target_idletime, client_idletime;
3873     int max_sockets = m_max_sockets;
3874     int max_clients;
3875     int keepalive_limit_bw, keepalive_limit_pdu;
3876     int pre_init;
3877     const char *cql2rpn = 0;
3878     const char *authentication = 0;
3879     const char *negotiation_charset = 0;
3880     const char *negotiation_lang = 0;
3881
3882     Yaz_ProxyConfig *cfg = check_reconfigure();
3883
3884     zurl_in_use[0] = 0;
3885
3886     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3887         set_APDU_yazlog(1);
3888     else
3889         set_APDU_yazlog(0);
3890
3891     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3892                                           &limit_bw, &limit_pdu, &limit_req,
3893                                           &limit_search,
3894                                           &target_idletime, &client_idletime,
3895                                           &max_sockets,
3896                                           &max_clients,
3897                                           &keepalive_limit_bw,
3898                                           &keepalive_limit_pdu,
3899                                           &pre_init,
3900                                           &cql2rpn,
3901                                           &authentication,
3902                                           &negotiation_charset,
3903                                           &negotiation_lang,
3904                                           0,
3905                                           0) ; i++)
3906     {
3907         if (pre_init)
3908         {
3909             int j;
3910             for (j = 0; zurl_in_use[j]; j++)
3911             {
3912                 Yaz_ProxyClient *c;
3913                 int spare = 0;
3914                 int spare_waiting = 0;
3915                 int in_use = 0;
3916                 int other = 0;
3917                 for (c = m_clientPool; c; c = c->m_next)
3918                 {
3919                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3920                     {
3921                         if (c->m_cookie == 0)
3922                         {
3923                             if (c->m_server == 0)
3924                                 if (c->m_waiting)
3925                                     spare_waiting++;
3926                                 else
3927                                     spare++;
3928                             else
3929                                 in_use++;
3930                         }
3931                         else
3932                             other++;
3933                     }
3934                 }
3935                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3936                         "sparew=%d preinit=%d",m_session_str,
3937                         name, zurl_in_use[j], in_use, other,
3938                         spare, spare_waiting, pre_init);
3939                 if (spare + spare_waiting < pre_init
3940                     && in_use + spare + spare_waiting + other < max_sockets)
3941                 {
3942                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3943                     c->m_next = m_clientPool;
3944                     if (c->m_next)
3945                         c->m_next->m_prev = &c->m_next;
3946                     m_clientPool = c;
3947                     c->m_prev = &m_clientPool;
3948
3949                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3950                         c->set_APDU_yazlog(1);
3951                     else
3952                         c->set_APDU_yazlog(0);
3953
3954                     if (c->client(zurl_in_use[j]))
3955                     {
3956                         timeout(60);
3957                         delete c;
3958                         return;
3959                     }
3960                     c->timeout(30);
3961                     c->m_waiting = 1;
3962                     c->m_target_idletime = target_idletime;
3963                     c->m_seqno = m_seqno++;
3964                 }
3965             }
3966         }
3967     }
3968 }
3969
3970 void Yaz_Proxy::timeoutNotify()
3971 {
3972     if (m_parent)
3973     {
3974         GDU *gdu;
3975         switch(m_timeout_mode)
3976         {
3977         case timeout_normal:
3978         case timeout_busy:
3979             inc_request_no();
3980             m_in_queue.clear();
3981             yaz_log(YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3982             dec_ref();
3983             break;
3984         case timeout_reduce:
3985             timeout(m_client_idletime);
3986             m_timeout_mode = timeout_busy;
3987             gdu = m_timeout_gdu;
3988             m_timeout_gdu = 0;
3989             recv_GDU_normal(gdu);
3990             break;
3991         case timeout_xsl:
3992             assert(m_stylesheet_nprl);
3993             convert_xsl_delay();
3994             recv_GDU_more(true);
3995         }
3996     }
3997     else
3998     {
3999         timeout(600);
4000         pre_init();
4001     }
4002 }
4003
4004 void Yaz_Proxy::markInvalid()
4005 {
4006     m_client = 0;
4007     m_flag_invalid_session = 1;
4008 }
4009
4010 void Yaz_ProxyClient::timeoutNotify()
4011 {
4012     if (m_server)
4013         m_server->inc_request_no();
4014
4015     yaz_log(YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
4016              get_hostname());
4017
4018     if (m_server)
4019         m_server->send_response_fail_client(get_hostname());
4020
4021     Yaz_Proxy *proxy_root = m_root;
4022
4023     shutdown();
4024
4025     proxy_root->pre_init();
4026 }
4027
4028 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
4029                                  Yaz_Proxy *parent) :
4030     Z_Assoc (the_PDU_Observable)
4031 {
4032     m_cookie = 0;
4033     m_next = 0;
4034     m_prev = 0;
4035     m_init_flag = 0;
4036     m_last_query = 0;
4037     m_last_resultSetId = 0;
4038     m_last_resultCount = 0;
4039     m_last_ok = 0;
4040     m_sr_transform = 0;
4041     m_waiting = 0;
4042     m_init_odr = odr_createmem (ODR_DECODE);
4043     m_initResponse = 0;
4044     m_initResponse_options = 0;
4045     m_initResponse_version = 0;
4046     m_initResponse_preferredMessageSize = 0;
4047     m_initResponse_maximumRecordSize = 0;
4048     m_resultSetStartPoint = 0;
4049     m_bytes_sent = m_bytes_recv = 0;
4050     m_pdu_recv = 0;
4051     m_server = 0;
4052     m_seqno = 0;
4053     m_target_idletime = 600;
4054     m_root = parent;
4055     m_idAuthentication_odr = odr_createmem(ODR_ENCODE);
4056     m_idAuthentication_ber_buf = 0;
4057     m_idAuthentication_ber_size = 0;
4058 }
4059
4060 const char *Yaz_Proxy::option(const char *name, const char *value)
4061 {
4062     if (!strcmp (name, "optimize")) {
4063         if (value) {
4064             xfree(m_optimize);
4065             m_optimize = xstrdup(value);
4066         }
4067         return m_optimize;
4068     }
4069     return 0;
4070 }
4071
4072 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
4073 {
4074
4075 }
4076
4077 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
4078 {
4079     if (apdu->which == Z_GDU_Z3950)
4080         recv_Z_PDU(apdu->u.z3950, len);
4081     else if (apdu->which == Z_GDU_HTTP_Response)
4082         recv_HTTP_response(apdu->u.HTTP_Response, len);
4083     else
4084         shutdown();
4085 }
4086
4087 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
4088 {
4089     if (!m_flag_invalid_session)
4090         return 0;
4091     m_flag_invalid_session = 0;
4092     handle_incoming_Z_PDU(m_apdu_invalid_session);
4093     assert (m_mem_invalid_session);
4094     nmem_destroy(m_mem_invalid_session);
4095     m_mem_invalid_session = 0;
4096     return 1;
4097 }
4098
4099 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
4100 {
4101     m_bytes_recv += len;
4102
4103     m_pdu_recv++;
4104     m_waiting = 0;
4105     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
4106         yaz_log(YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
4107                  apdu_name(apdu), get_hostname(), len);
4108     if (apdu->which == Z_APDU_initResponse)
4109     {
4110         if (!m_server)  // if this is a pre init session , check for more
4111             m_root->pre_init();
4112         NMEM nmem = odr_extract_mem (odr_decode());
4113         odr_reset (m_init_odr);
4114         nmem_transfer (m_init_odr->mem, nmem);
4115         m_initResponse = apdu;
4116         m_initResponse_options = apdu->u.initResponse->options;
4117         m_initResponse_version = apdu->u.initResponse->protocolVersion;
4118         m_initResponse_preferredMessageSize =
4119             *apdu->u.initResponse->preferredMessageSize;
4120         m_initResponse_maximumRecordSize =
4121             *apdu->u.initResponse->maximumRecordSize;
4122
4123         Z_InitResponse *ir = apdu->u.initResponse;
4124
4125         // apply YAZ Proxy version
4126         char *imv0 = ir->implementationVersion;
4127         char *imv1 = (char*)
4128             odr_malloc(m_init_odr, 20 + (imv0 ? strlen(imv0) : 0));
4129         *imv1 = '\0';
4130         if (imv0)
4131             strcat(imv1, imv0);
4132 #ifdef VERSION
4133         strcat(imv1, "/" VERSION);
4134 #endif
4135         ir->implementationVersion = imv1;
4136
4137         // apply YAZ Proxy implementation name
4138         char *im0 = ir->implementationName;
4139         char *im1 = (char*)
4140             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
4141         *im1 = '\0';
4142         if (im0)
4143         {
4144             strcat(im1, im0);
4145             strcat(im1, " ");
4146         }
4147         strcat(im1, "(YAZ Proxy)");
4148         ir->implementationName = im1;
4149
4150         nmem_destroy (nmem);
4151
4152         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
4153             return;
4154     }
4155     if (apdu->which == Z_APDU_searchResponse)
4156     {
4157         Z_SearchResponse *sr = apdu->u.searchResponse;
4158         m_last_resultCount = *sr->resultCount;
4159         int status = *sr->searchStatus;
4160         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
4161         {
4162             m_last_ok = 1;
4163
4164             if (sr->records && sr->records->which == Z_Records_DBOSD)
4165             {
4166                 m_cache.add(odr_decode(),
4167                             sr->records->u.databaseOrSurDiagnostics, 1,
4168                             *sr->resultCount);
4169             }
4170         }
4171     }
4172     if (apdu->which == Z_APDU_presentResponse)
4173     {
4174         Z_PresentResponse *pr = apdu->u.presentResponse;
4175         if (m_sr_transform)
4176         {
4177             m_sr_transform = 0;
4178             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
4179             Z_SearchResponse *sr = new_apdu->u.searchResponse;
4180             sr->referenceId = pr->referenceId;
4181             *sr->resultCount = m_last_resultCount;
4182             sr->records = pr->records;
4183             sr->nextResultSetPosition = pr->nextResultSetPosition;
4184             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
4185             apdu = new_apdu;
4186         }
4187         if (pr->records &&
4188             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
4189         {
4190             m_cache.add(odr_decode(),
4191                         pr->records->u.databaseOrSurDiagnostics,
4192                         m_resultSetStartPoint, -1);
4193             m_resultSetStartPoint = 0;
4194         }
4195     }
4196     if (m_cookie)
4197         set_otherInformationString(apdu, yaz_oid_userinfo_cookie, 1, m_cookie);
4198
4199     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
4200
4201     if (server)
4202         server->send_to_client(apdu);
4203     if (apdu->which == Z_APDU_close)
4204         shutdown();
4205     else if (server)
4206         server->recv_GDU_more(true);
4207 }
4208
4209 void Yaz_Proxy::low_socket_close()
4210 {
4211 #if WIN32
4212 #else
4213     int i;
4214     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
4215         if  (m_lo_fd[i] >= 0)
4216             ::close(m_lo_fd[i]);
4217 #endif
4218 }
4219
4220 void Yaz_Proxy::low_socket_open()
4221 {
4222 #if WIN32
4223 #else
4224     int i;
4225     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
4226         m_lo_fd[i] = open("/dev/null", O_RDONLY);
4227 #endif
4228 }
4229
4230 int Yaz_Proxy::server(const char *addr)
4231 {
4232     int r = Z_Assoc::server(addr);
4233     if (!r)
4234     {
4235         yaz_log(YLOG_LOG, "%sStarted proxy "
4236 #ifdef VERSION
4237             VERSION
4238 #endif
4239             " on %s", m_session_str, addr);
4240         timeout(1);
4241     }
4242     return r;
4243 }
4244
4245 void Yaz_Proxy::base64_decode(const char *base64, char *buf, int buf_len)
4246 {
4247     const char *base64_chars =
4248         "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
4249     int len = strlen(base64);
4250     int buf_pos = 0;
4251     int index = 1;
4252
4253     for (int pos = 0; pos <= len; pos++)
4254     {
4255         if (base64[pos] == '=' || buf_pos + 1 >= buf_len)
4256             break;
4257
4258         const char *ch_ptr = strchr(base64_chars, base64[pos]);
4259         if (!ch_ptr)
4260             break;
4261         char ch = (char) (ch_ptr - base64_chars);
4262         switch (index)
4263         {
4264         case 1:
4265             buf[buf_pos] = ch << 2;
4266             break;
4267         case 2:
4268             buf[buf_pos++] += (ch & 0x30) >> 4;
4269             buf[buf_pos] = (ch & 0x0f) << 4;
4270             break;
4271         case 3:
4272             buf[buf_pos++] += (ch & 0x3c) >> 2;
4273             buf[buf_pos] = (ch & 0x03) << 6;
4274             break;
4275         case 4:
4276             buf[buf_pos++] += ch;
4277         }
4278         if (index < 4)
4279             index++;
4280         else
4281             index = 1;
4282     }
4283     buf[buf_pos] = '\0';
4284 }
4285
4286 /*
4287  * Local variables:
4288  * c-basic-offset: 4
4289  * c-file-style: "Stroustrup"
4290  * indent-tabs-mode: nil
4291  * End:
4292  * vim: shiftwidth=4 tabstop=8 expandtab
4293  */
4294