src/yaz-proxy.cpp

   1 /* This file is part of YAZ proxy
   2    Copyright (C) 1998-2011 Index Data
   3
   4 YAZ proxy is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17 */
  18
  19 #ifdef WIN32
  20 #define HAVE_SYS_STAT_H 1
  21 #define HAVE_SYS_TYPES_H 1
  22 #endif
  23
  24 #if HAVE_UNISTD_H
  25 #include <unistd.h>
  26 #endif
  27 #if HAVE_SYS_TIME_H
  28 #include <sys/time.h>
  29 #endif
  30 #if HAVE_SYS_TYPES_H
  31 #include <sys/types.h>
  32 #endif
  33 #if HAVE_SYS_STAT_H
  34 #include <sys/stat.h>
  35 #endif
  36
  37 #include <assert.h>
  38 #include <stdlib.h>
  39 #include <time.h>
  40 #include <fcntl.h>
  41
  42 #include <yaz/srw.h>
  43 #include <yaz/marcdisp.h>
  44 #include <yaz/yaz-iconv.h>
  45 #include <yaz/log.h>
  46 #include <yaz/diagbib1.h>
  47 #include "proxyp.h"
  48 #include <yaz/pquery.h>
  49 #include <yaz/otherinfo.h>
  50 #include <yaz/charneg.h>
  51 #include <yaz/oid_db.h>
  52 #include "msg-thread.h"
  53
  54 #if YAZ_VERSIONL >= 0x03001D
  55 #define YAZ_HAS_MK_SURROGATE 1
  56 #else
  57 #define YAZ_HAS_MK_SURROGATE 0
  58 #endif
  59
  60 using namespace yazpp_1;
  61
  62 #ifdef WIN32
  63 #define strncasecmp _strnicmp
  64 #endif
  65
  66 class YAZ_EXPORT Auth_Msg : public IMsg_Thread {
  67 public:
  68     int m_ret;
  69     IMsg_Thread *handle();
  70     void result();
  71     Yaz_Proxy *m_proxy;
  72     NMEM m_nmem;
  73     char *m_apdu_buf;
  74     int m_apdu_len;
  75     Auth_Msg();
  76     virtual ~Auth_Msg();
  77 };
  78
  79 Auth_Msg::Auth_Msg()
  80 {
  81     m_nmem = nmem_create();
  82 }
  83
  84 Auth_Msg::~Auth_Msg()
  85 {
  86     nmem_destroy(m_nmem);
  87 }
  88
  89 IMsg_Thread *Auth_Msg::handle()
  90 {
  91     ODR decode = odr_createmem(ODR_DECODE);
  92     Z_APDU *apdu;
  93
  94     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
  95     int r = z_APDU(decode, &apdu, 0, 0);
  96     if (!r)
  97     {
  98         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
  99     }
 100     else
 101     {
 102         m_ret = m_proxy->handle_authentication(apdu);
 103     }
 104     odr_destroy(decode);
 105     return this;
 106 }
 107
 108 void Auth_Msg::result()
 109 {
 110     if (m_proxy->dec_ref())
 111     {
 112         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
 113     }
 114     else
 115     {
 116         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
 117         Z_APDU *apdu = 0;
 118         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
 119         if (!r)
 120             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
 121         m_proxy->result_authentication(apdu, m_ret);
 122     }
 123     delete this;
 124 }
 125
 126 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
 127 {
 128     if (apdu == 0 || ret == 0)
 129     {
 130         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
 131         *apdu_reject->u.initResponse->result = 0;
 132         send_to_client(apdu_reject);
 133         dec_ref();
 134     }
 135     else
 136     {
 137         if (apdu->which == Z_APDU_initRequest)
 138         {
 139             Yaz_ProxyConfig *cfg = check_reconfigure();
 140             if (cfg)
 141                 cfg->target_authentication(m_default_target, odr_encode(),
 142                                            apdu->u.initRequest);
 143         }
 144         handle_incoming_Z_PDU_2(apdu);
 145     }
 146 }
 147
 148 static const char *apdu_name(Z_APDU *apdu)
 149 {
 150     switch (apdu->which)
 151     {
 152     case Z_APDU_initRequest:
 153         return "initRequest";
 154     case Z_APDU_initResponse:
 155         return "initResponse";
 156     case Z_APDU_searchRequest:
 157         return "searchRequest";
 158     case Z_APDU_searchResponse:
 159         return "searchResponse";
 160     case Z_APDU_presentRequest:
 161         return "presentRequest";
 162     case Z_APDU_presentResponse:
 163         return "presentResponse";
 164     case Z_APDU_deleteResultSetRequest:
 165         return "deleteResultSetRequest";
 166     case Z_APDU_deleteResultSetResponse:
 167         return "deleteResultSetResponse";
 168     case Z_APDU_scanRequest:
 169         return "scanRequest";
 170     case Z_APDU_scanResponse:
 171         return "scanResponse";
 172     case Z_APDU_sortRequest:
 173         return "sortRequest";
 174     case Z_APDU_sortResponse:
 175         return "sortResponse";
 176     case Z_APDU_extendedServicesRequest:
 177         return "extendedServicesRequest";
 178     case Z_APDU_extendedServicesResponse:
 179         return "extendedServicesResponse";
 180     case Z_APDU_close:
 181         return "close";
 182     }
 183     return "other";
 184 }
 185
 186 static const char *gdu_name(Z_GDU *gdu)
 187 {
 188     switch(gdu->which)
 189     {
 190     case Z_GDU_Z3950:
 191         return apdu_name(gdu->u.z3950);
 192     case Z_GDU_HTTP_Request:
 193         return "HTTP Request";
 194     case Z_GDU_HTTP_Response:
 195         return "HTTP Response";
 196     }
 197     return "Unknown request/response";
 198 }
 199
 200 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
 201                      ISocketObservable *the_socket_observable,
 202                      Yaz_Proxy *parent)
 203     :
 204     Z_Assoc(the_PDU_Observable),
 205     m_bw_stat(60), m_pdu_stat(60), m_search_stat(60)
 206 {
 207     m_PDU_Observable = the_PDU_Observable;
 208     m_socket_observable = the_socket_observable;
 209     m_client = 0;
 210     m_parent = parent;
 211     m_clientPool = 0;
 212     m_seqno = 1;
 213     m_keepalive_limit_bw = 500000;
 214     m_keepalive_limit_pdu = 1000;
 215     m_proxyTarget = 0;
 216     m_default_target = 0;
 217     m_proxy_negotiation_charset = 0;
 218     m_proxy_negotiation_lang = 0;
 219     m_proxy_negotiation_default_charset = 0;
 220     m_charset_converter = new Yaz_CharsetConverter;
 221     m_max_clients = 150;
 222     m_log_mask = 0;
 223     m_seed = time(0);
 224     m_client_idletime = 600;
 225     m_target_idletime = 600;
 226     m_max_sockets = 1024;
 227     m_optimize = xstrdup("1");
 228     strcpy(m_session_str, "0 ");
 229     m_session_no = 0;
 230     m_bytes_sent = 0;
 231     m_bytes_recv = 0;
 232     m_bw_max = 0;
 233     m_pdu_max = 0;
 234     m_search_max = 0;
 235     m_max_connect = 0;
 236     m_max_connect_period = 0;
 237     m_limit_connect = 0;
 238     m_limit_connect_period = 0;
 239     m_timeout_mode = timeout_normal;
 240     m_timeout_gdu = 0;
 241     m_max_record_retrieve = 0;
 242     m_reconfig_flag = 0;
 243     m_config_fname = 0;
 244     m_request_no = 0;
 245     m_flag_invalid_session = 0;
 246     m_referenceId = 0;
 247     m_referenceId_mem = nmem_create();
 248     m_config = 0;
 249     m_marcxml_mode = none;
 250     m_stylesheet_xsp = 0;
 251     m_stylesheet_nprl = 0;
 252     m_stylesheet_apdu = 0;
 253     m_s2z_stylesheet = 0;
 254     m_s2z_database = 0;
 255     m_schema = 0;
 256     m_backend_type = 0;
 257     m_backend_charset = 0;
 258     m_frontend_type[0] = -1;
 259     m_initRequest_apdu = 0;
 260     m_initRequest_mem = 0;
 261     m_initRequest_preferredMessageSize = 0;
 262     m_initRequest_maximumRecordSize = 0;
 263     m_initRequest_options = 0;
 264     m_initRequest_version = 0;
 265     m_initRequest_oi_negotiation_charsets = 0;
 266     m_initRequest_oi_negotiation_num_charsets = 0;
 267     m_initRequest_oi_negotiation_langs = 0;
 268     m_initRequest_oi_negotiation_num_langs = 0;
 269     m_initRequest_oi_negotiation_selected = 0;
 270     m_apdu_invalid_session = 0;
 271     m_mem_invalid_session = 0;
 272     m_s2z_odr_init = 0;
 273     m_s2z_odr_search = 0;
 274     m_s2z_odr_scan = 0;
 275     m_s2z_init_apdu = 0;
 276     m_s2z_search_apdu = 0;
 277     m_s2z_present_apdu = 0;
 278     m_s2z_scan_apdu = 0;
 279     m_http_keepalive = 0;
 280     m_http_version = 0;
 281     m_soap_ns = 0;
 282     m_s2z_packing = Z_SRW_recordPacking_string;
 283 #if HAVE_GETTIMEOFDAY
 284     m_time_tv = xmalloc(sizeof(struct timeval));
 285     struct timeval *tv = (struct timeval *) m_time_tv;
 286     tv->tv_sec = 0;
 287     tv->tv_usec = 0;
 288 #else
 289     m_time_tv = 0;
 290 #endif
 291     m_usemarcon_ini_stage1 = 0;
 292     m_usemarcon_ini_stage2 = 0;
 293     m_usemarcon = new Yaz_usemarcon();
 294     if (!m_parent)
 295         low_socket_open();
 296     m_backend_elementset = 0;
 297     m_my_thread = 0;
 298     m_ref_count = 1;
 299     m_main_ptr_dec = false;
 300     m_peername = 0;
 301     m_num_msg_threads = 0;
 302 }
 303
 304 void Yaz_Proxy::inc_ref()
 305 {
 306     m_ref_count++;
 307 }
 308
 309 Yaz_Proxy::~Yaz_Proxy()
 310 {
 311     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 312             m_bytes_sent, m_bytes_recv);
 313     nmem_destroy(m_initRequest_mem);
 314     nmem_destroy(m_mem_invalid_session);
 315     nmem_destroy(m_referenceId_mem);
 316
 317     xfree(m_proxyTarget);
 318     xfree(m_default_target);
 319     xfree(m_proxy_negotiation_charset);
 320     xfree(m_proxy_negotiation_lang);
 321     xfree(m_proxy_negotiation_default_charset);
 322     delete m_charset_converter;
 323     xfree(m_optimize);
 324
 325 #if YAZ_HAVE_XSLT
 326     if (m_stylesheet_xsp)
 327         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 328 #endif
 329     xfree(m_time_tv);
 330
 331     xfree(m_peername);
 332     xfree(m_schema);
 333     xfree(m_backend_type);
 334     xfree(m_backend_charset);
 335     xfree(m_usemarcon_ini_stage1);
 336     xfree(m_usemarcon_ini_stage2);
 337     xfree(m_backend_elementset);
 338     delete m_usemarcon;
 339     if (m_s2z_odr_init)
 340         odr_destroy(m_s2z_odr_init);
 341     if (m_s2z_odr_search)
 342         odr_destroy(m_s2z_odr_search);
 343     if (m_s2z_odr_scan)
 344         odr_destroy(m_s2z_odr_scan);
 345     if (!m_parent)
 346         low_socket_close();
 347     if (!m_parent)
 348         delete m_my_thread;
 349     delete m_config;
 350 }
 351
 352 void Yaz_Proxy::set_debug_mode(int mode)
 353 {
 354     m_debug_mode = mode;
 355 }
 356
 357 int Yaz_Proxy::set_config(const char *config)
 358 {
 359     delete m_config;
 360     m_config = new Yaz_ProxyConfig();
 361     xfree(m_config_fname);
 362     m_config_fname = xstrdup(config);
 363     int r = m_config->read_xml(config);
 364     if (!r)
 365     {
 366         int period = 60;
 367         m_config->get_generic_info(&m_log_mask, &m_max_clients,
 368                                    &m_max_connect, &m_limit_connect, &period,
 369                                    &m_num_msg_threads);
 370         m_connect.set_period(period);
 371     }
 372     return r;
 373 }
 374
 375 void Yaz_Proxy::set_default_target(const char *target)
 376 {
 377     xfree(m_default_target);
 378     m_default_target = 0;
 379     if (target)
 380         m_default_target = (char *) xstrdup(target);
 381 }
 382
 383 void Yaz_Proxy::set_proxy_negotiation(const char *charset, const char *lang,
 384                                        const char *default_charset)
 385 {
 386     yaz_log(YLOG_DEBUG, "%sSet the proxy negotiation: charset to '%s', "
 387         "default charset to '%s', language to '%s'", m_session_str,
 388         charset?charset:"none",
 389         default_charset?default_charset:"none",
 390         lang?lang:"none");
 391     xfree(m_proxy_negotiation_charset);
 392     xfree(m_proxy_negotiation_lang);
 393     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 394     if (charset)
 395         m_proxy_negotiation_charset = (char *) xstrdup(charset);
 396     if (lang)
 397         m_proxy_negotiation_lang = (char *) xstrdup(lang);
 398     if (default_charset)
 399         m_proxy_negotiation_default_charset =
 400             (char *) xstrdup(default_charset);
 401 }
 402
 403 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 404 {
 405     if (m_parent)
 406         return m_parent->check_reconfigure();
 407
 408     Yaz_ProxyConfig *cfg = m_config;
 409     if (m_reconfig_flag)
 410     {
 411         yaz_log(YLOG_LOG, "reconfigure");
 412         if (m_config_fname && cfg)
 413         {
 414             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 415             int r = cfg->read_xml(m_config_fname);
 416             if (r)
 417                 yaz_log(YLOG_WARN, "reconfigure failed");
 418             else
 419             {
 420                 m_log_mask = 0;
 421                 int period = 60;
 422                 cfg->get_generic_info(&m_log_mask, &m_max_clients,
 423                                       &m_max_connect, &m_limit_connect,
 424                                       &period, &m_num_msg_threads);
 425                 m_connect.set_period(period);
 426             }
 427         }
 428         else
 429             yaz_log(YLOG_LOG, "reconfigure");
 430         m_reconfig_flag = 0;
 431     }
 432     return cfg;
 433 }
 434
 435 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 436                                         *the_PDU_Observable, int fd)
 437 {
 438     check_reconfigure();
 439
 440     char session_str[200];
 441     const char *peername = the_PDU_Observable->getpeername();
 442     if (!peername)
 443         peername = "nullpeer";
 444
 445     if (m_log_mask & PROXY_LOG_IP_CLIENT)
 446         sprintf(session_str, "%ld:%d %.80s %d ",
 447                 (long) time(0), m_session_no, peername, 0);
 448     else
 449         sprintf(session_str, "%ld:%d %d ",
 450                 (long) time(0), m_session_no, 0);
 451     m_session_no++;
 452
 453     yaz_log(YLOG_LOG, "%sNew session %s", session_str, peername);
 454
 455     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 456                                          m_socket_observable, this);
 457
 458     new_proxy->m_config = 0;
 459     new_proxy->m_config_fname = 0;
 460     new_proxy->timeout(m_client_idletime);
 461     new_proxy->m_target_idletime = m_target_idletime;
 462     new_proxy->set_default_target(m_default_target);
 463     new_proxy->m_max_clients = m_max_clients;
 464     new_proxy->m_log_mask = m_log_mask;
 465     new_proxy->m_session_no = m_session_no;
 466     new_proxy->m_num_msg_threads = m_num_msg_threads;
 467
 468 #if 0
 469     // in case we want to watch a particular client..
 470     if (!strcmp(peername, "tcp:163.121.19.82")) // NIS GROUP
 471         new_proxy->m_log_mask = 255;
 472 #endif
 473
 474     new_proxy->set_APDU_log(get_APDU_log());
 475     if (new_proxy->m_log_mask & PROXY_LOG_APDU_CLIENT)
 476         new_proxy->set_APDU_yazlog(1);
 477     else
 478         new_proxy->set_APDU_yazlog(0);
 479     strcpy(new_proxy->m_session_str, session_str);
 480     new_proxy->m_peername = xstrdup(peername);
 481     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 482         m_proxy_negotiation_lang, m_proxy_negotiation_default_charset);
 483     // create thread object the first time we get an incoming connection
 484     if (!m_my_thread && m_num_msg_threads > 0)
 485     {
 486         yaz_log(YLOG_LOG, "%sStarting message thread management. number=%d",
 487                  session_str, m_num_msg_threads);
 488         m_my_thread = new Msg_Thread(m_socket_observable, m_num_msg_threads);
 489     }
 490     new_proxy->m_my_thread = m_my_thread;
 491     return new_proxy;
 492 }
 493
 494 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 495 {
 496     Z_OtherInformationUnit *oi =
 497         update_otherInformation(otherInfo, 0, yaz_oid_userinfo_cookie, 1, 1);
 498
 499     if (oi && oi->which == Z_OtherInfo_characterInfo)
 500         return oi->information.characterInfo;
 501     return 0;
 502 }
 503
 504 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 505 {
 506     Z_OtherInformationUnit *oi =
 507         update_otherInformation(otherInfo, 0, yaz_oid_userinfo_proxy, 1, 1);
 508
 509     if (oi && oi->which == Z_OtherInfo_characterInfo)
 510         return oi->information.characterInfo;
 511     return 0;
 512 }
 513 const char *Yaz_Proxy::load_balance(const char **url)
 514 {
 515     int zurl_in_use[MAX_ZURL_PLEX];
 516     int zurl_in_spare[MAX_ZURL_PLEX];
 517     Yaz_ProxyClient *c;
 518     int i;
 519
 520     for (i = 0; i<MAX_ZURL_PLEX; i++)
 521     {
 522         zurl_in_use[i] = 0;
 523         zurl_in_spare[i] = 0;
 524     }
 525     for (c = m_parent->m_clientPool; c; c = c->m_next)
 526     {
 527         for (i = 0; url[i]; i++)
 528             if (!strcmp(url[i], c->get_hostname()))
 529             {
 530                 zurl_in_use[i]++;
 531                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 532                     zurl_in_spare[i]++;
 533             }
 534     }
 535     int min_use = 100000;
 536     int spare_for_min = 0;
 537     int max_spare = 0;
 538     const char *ret_min = 0;
 539     const char *ret_spare = 0;
 540     for (i = 0; url[i]; i++)
 541     {
 542         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 543                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 544         if (min_use > zurl_in_use[i])
 545         {
 546             ret_min = url[i];
 547             min_use = zurl_in_use[i];
 548             spare_for_min = zurl_in_spare[i];
 549         }
 550         if (max_spare < zurl_in_spare[i])
 551         {
 552             ret_spare = url[i];
 553             max_spare = zurl_in_spare[i];
 554         }
 555     }
 556     return ret_min;
 557 }
 558
 559 int Yaz_Proxy::get_number_of_connections()
 560 {
 561     int no_connections = 0;
 562     Yaz_ProxyClient *c;
 563
 564     for (c = m_parent->m_clientPool; c; c = c->m_next)
 565     {
 566         assert(c->m_prev);
 567         assert(*c->m_prev == c);
 568         if (!strcmp(m_proxyTarget, c->get_hostname()))
 569         {
 570             no_connections++;
 571         }
 572     }
 573     yaz_log(YLOG_LOG, "%sExisting %s connections: %d", m_session_str, m_proxyTarget,
 574         no_connections);
 575     return no_connections;
 576 }
 577
 578 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 579                                        const char *proxy_host, int *http_code)
 580 {
 581     assert(m_parent);
 582     Yaz_Proxy *parent = m_parent;
 583     Yaz_ProxyClient *c = m_client;
 584
 585     if (!m_proxyTarget)
 586     {
 587         const char *url[MAX_ZURL_PLEX];
 588         Yaz_ProxyConfig *cfg = check_reconfigure();
 589         if (proxy_host)
 590         {
 591             if (parent && parent->m_debug_mode)
 592             {
 593                 // only to be enabled for debugging...
 594                 if (!strcmp(proxy_host, "stop"))
 595                     exit(0);
 596             }
 597             xfree(m_default_target);
 598             m_default_target = xstrdup(proxy_host);
 599         }
 600         proxy_host = m_default_target;
 601         int client_idletime = -1;
 602         const char *cql2rpn_fname = 0;
 603         const char *negotiation_charset = 0;
 604         const char *negotiation_lang = 0;
 605         const char *query_charset = 0;
 606         const char *default_client_query_charset = 0;
 607         url[0] = m_default_target;
 608         url[1] = 0;
 609         if (cfg)
 610         {
 611             int pre_init = 0;
 612             cfg->get_target_info(proxy_host, url, &m_bw_max,
 613                                  &m_pdu_max, &m_max_record_retrieve,
 614                                  &m_search_max,
 615                                  &m_target_idletime, &client_idletime,
 616                                  &m_max_sockets,
 617                                  &parent->m_max_clients,
 618                                  &m_keepalive_limit_bw,
 619                                  &m_keepalive_limit_pdu,
 620                                  &pre_init,
 621                                  &cql2rpn_fname,
 622                                  &negotiation_charset,
 623                                  &negotiation_lang,
 624                                  &query_charset,
 625                                  &default_client_query_charset);
 626         }
 627         if (client_idletime != -1)
 628         {
 629             m_client_idletime = client_idletime;
 630             timeout(m_client_idletime);
 631         }
 632
 633         // get those FILE descriptors available
 634         m_parent->low_socket_close();
 635         if (cql2rpn_fname)
 636             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 637         // reserve them again
 638         m_parent->low_socket_open();
 639
 640         if (negotiation_charset || negotiation_lang || default_client_query_charset)
 641         {
 642             set_proxy_negotiation(negotiation_charset,
 643                 negotiation_lang, default_client_query_charset);
 644         }
 645         m_charset_converter->set_target_query_charset(query_charset);
 646         if (!url[0])
 647         {
 648             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 649             return 0;
 650         }
 651         // we don't handle multiplexing for cookie session, so we just
 652         // pick the first one in this case (anonymous users will be able
 653         // to use any backend)
 654         if (cookie && *cookie)
 655             m_proxyTarget = (char*) xstrdup(url[0]);
 656         else
 657             m_proxyTarget = (char*) xstrdup(load_balance(url));
 658     }
 659     if (cookie && *cookie)
 660     {   // search in sessions with a cookie
 661         for (c = parent->m_clientPool; c; c = c->m_next)
 662         {
 663             assert(c->m_prev);
 664             assert(*c->m_prev == c);
 665             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 666                 !strcmp(m_proxyTarget, c->get_hostname()))
 667             {
 668                 // Found it in cache
 669                 // The following handles "cancel"
 670                 // If connection is busy (waiting for PDU) and
 671                 // we have an initRequest we can safely do re-open
 672                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 673                 {
 674                     yaz_log(YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 675                              c->get_hostname());
 676                     c->close();
 677                     c->m_init_flag = 0;
 678
 679                     c->m_last_ok = 0;
 680                     c->m_cache.clear();
 681                     c->m_last_resultCount = 0;
 682                     c->m_sr_transform = 0;
 683                     c->m_waiting = 0;
 684                     c->m_resultSetStartPoint = 0;
 685                     c->m_target_idletime = m_target_idletime;
 686                     if (c->client(m_proxyTarget))
 687                     {
 688                         delete c;
 689                         return 0;
 690                     }
 691                     c->timeout(30);
 692                 }
 693                 c->m_seqno = parent->m_seqno;
 694                 if (c->m_server && c->m_server != this)
 695                     c->m_server->m_client = 0;
 696                 c->m_server = this;
 697                 (parent->m_seqno)++;
 698                 yaz_log(YLOG_DEBUG, "get_client 1 %p %p", this, c);
 699                 return c;
 700             }
 701         }
 702     }
 703     else if (!c && apdu->which == Z_APDU_initRequest )
 704     {
 705         // anonymous sessions without cookie.
 706         // if authentication is set it is NOT anonymous se we can't share them.
 707         // If charset and lang negotiation is use it is NOT anonymous session too.
 708         for (c = parent->m_clientPool; c; c = c->m_next)
 709         {
 710             assert(c->m_prev);
 711             assert(*c->m_prev == c);
 712             if (c->m_server == 0 && c->m_cookie == 0 && c->m_waiting == 0
 713                 && c->compare_idAuthentication(apdu)
 714                 && c->compare_charset(apdu)
 715                 && !strcmp(m_proxyTarget, c->get_hostname()))
 716             {
 717                 // found it in cache
 718                 yaz_log(YLOG_LOG, "%sREUSE %d %s",
 719                          m_session_str, parent->m_seqno, c->get_hostname());
 720
 721                 c->m_seqno = parent->m_seqno;
 722                 assert(c->m_server == 0);
 723                 c->m_server = this;
 724
 725                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 726                     c->set_APDU_yazlog(1);
 727                 else
 728                     c->set_APDU_yazlog(0);
 729
 730                 (parent->m_seqno)++;
 731
 732                 parent->pre_init();
 733
 734                 return c;
 735             }
 736         }
 737     }
 738     if (!m_client)
 739     {
 740         if (apdu->which != Z_APDU_initRequest)
 741         {
 742             yaz_log(YLOG_LOG, "%sno init request as first PDU", m_session_str);
 743             *http_code = 500;
 744             return 0;
 745         }
 746
 747         int no_in_use = get_number_of_connections();
 748         if (no_in_use >= m_max_sockets)
 749         {
 750             yaz_log(YLOG_LOG, "%smax sockets reached %d", m_session_str,
 751                 m_max_sockets);
 752             *http_code = 500;
 753             return 0;
 754         }
 755         // go through list of clients - and find the lowest/oldest one.
 756         Yaz_ProxyClient *c_min = 0;
 757         int min_seq = -1;
 758         int no_of_clients = 0;
 759         if (parent->m_clientPool)
 760             yaz_log(YLOG_DEBUG, "Existing sessions");
 761         for (c = parent->m_clientPool; c; c = c->m_next)
 762         {
 763             yaz_log(YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 764                                c->m_waiting, c->get_hostname(),
 765                                c->m_cookie ? c->m_cookie : "");
 766             no_of_clients++;
 767             if (min_seq < 0 || c->m_seqno < min_seq)
 768             {
 769                 min_seq = c->m_seqno;
 770                 c_min = c;
 771             }
 772         }
 773         if (no_of_clients >= parent->m_max_clients)
 774         {
 775             c = c_min;
 776             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 777             {
 778                 yaz_log(YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 779                          m_session_str, parent->m_max_clients, c->m_seqno);
 780                 if (c->m_server && c->m_server != this)
 781                     c->m_server->dec_ref();
 782             }
 783             else
 784             {
 785                 yaz_log(YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 786                          m_session_str, parent->m_max_clients,
 787                          c->m_seqno, parent->m_seqno, c->get_hostname());
 788                 xfree(c->m_cookie);
 789                 c->m_cookie = 0;
 790                 if (cookie)
 791                     c->m_cookie = xstrdup(cookie);
 792                 c->m_seqno = parent->m_seqno;
 793                 if (c->m_server && c->m_server != this)
 794                 {
 795                     c->m_server->m_client = 0;
 796                     c->m_server->dec_ref();
 797                 }
 798                 (parent->m_seqno)++;
 799                 c->m_target_idletime = m_target_idletime;
 800                 c->timeout(m_target_idletime);
 801
 802                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 803                     c->set_APDU_yazlog(1);
 804                 else
 805                     c->set_APDU_yazlog(0);
 806
 807                 return c;
 808             }
 809         }
 810         else
 811         {
 812
 813             yaz_log(YLOG_LOG, "%sNEW %d %s",
 814                      m_session_str, parent->m_seqno, m_proxyTarget);
 815             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 816             c->m_next = parent->m_clientPool;
 817             if (c->m_next)
 818                 c->m_next->m_prev = &c->m_next;
 819             parent->m_clientPool = c;
 820             c->m_prev = &parent->m_clientPool;
 821         }
 822
 823         xfree(c->m_cookie);
 824         c->m_cookie = 0;
 825         if (cookie)
 826             c->m_cookie = xstrdup(cookie);
 827
 828         c->m_seqno = parent->m_seqno;
 829         c->m_init_flag = 0;
 830         c->m_last_resultCount = 0;
 831         c->m_last_ok = 0;
 832         c->m_cache.clear();
 833         c->m_sr_transform = 0;
 834         c->m_waiting = 0;
 835         c->m_resultSetStartPoint = 0;
 836         (parent->m_seqno)++;
 837         if (c->client(m_proxyTarget))
 838         {
 839             delete c;
 840             return 0;
 841         }
 842         c->m_target_idletime = m_target_idletime;
 843         c->timeout(30);
 844
 845         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 846             c->set_APDU_yazlog(1);
 847         else
 848             c->set_APDU_yazlog(0);
 849
 850         c->set_idAuthentication(apdu);
 851     }
 852     yaz_log(YLOG_DEBUG, "get_client 3 %p %p", this, c);
 853     return c;
 854 }
 855
 856 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 857 {
 858     int i;
 859     for (i = 0; i<num; i++)
 860     {
 861         Z_DefaultDiagFormat *r;
 862         Z_DiagRec *p = pp[i];
 863         if (p->which != Z_DiagRec_defaultFormat)
 864         {
 865             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 866             return;
 867         }
 868         else
 869             r = p->u.defaultFormat;
 870         switch (r->which)
 871         {
 872         case Z_DefaultDiagFormat_v2Addinfo:
 873             yaz_log(YLOG_LOG, "%sError " ODR_INT_PRINTF " %s:%s",
 874                     m_session_str,
 875                     *r->condition, diagbib1_str(*r->condition),
 876                     r->u.v2Addinfo);
 877             break;
 878         case Z_DefaultDiagFormat_v3Addinfo:
 879             yaz_log(YLOG_LOG, "%sError " ODR_INT_PRINTF " %s:%s",
 880                     m_session_str,
 881                     *r->condition, diagbib1_str(*r->condition),
 882                     r->u.v3Addinfo);
 883             break;
 884         }
 885     }
 886 }
 887
 888 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 889 {
 890     if (!m_stylesheet_xsp || p->num_records <= 0)
 891     {
 892         return 0;  /* no XSLT to be done ... */
 893     }
 894
 895     m_stylesheet_offset = 0;
 896     m_stylesheet_nprl = p;
 897     m_stylesheet_apdu = apdu;
 898     m_timeout_mode = timeout_xsl;
 899
 900     timeout(0);
 901     return 1;
 902 }
 903
 904 void Yaz_Proxy::convert_xsl_delay()
 905 {
 906 #if YAZ_HAVE_XSLT
 907     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 908     if (npr->which == Z_NamePlusRecord_databaseRecord)
 909     {
 910         Z_External *r = npr->u.databaseRecord;
 911         if (r->which == Z_External_octet)
 912         {
 913 #if 0
 914             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 915 #endif
 916             xmlDocPtr res, doc = xmlParseMemory(
 917                 (char*) r->u.octet_aligned->buf,
 918                 r->u.octet_aligned->len);
 919
 920
 921             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 922                     m_session_str, m_stylesheet_offset);
 923             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 924                                       doc, 0);
 925
 926             if (res)
 927             {
 928                 xmlChar *out_buf;
 929                 int out_len;
 930                 xmlDocDumpFormatMemory(res, &out_buf, &out_len, 1);
 931                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 932                     u.databaseRecord =
 933                     z_ext_record_oid(odr_encode(), yaz_oid_recsyn_xml,
 934                                      (char*) out_buf, out_len);
 935                 xmlFree(out_buf);
 936                 xmlFreeDoc(res);
 937             }
 938
 939             xmlFreeDoc(doc);
 940         }
 941     }
 942 #endif
 943     m_stylesheet_offset++;
 944     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 945     {
 946         m_timeout_mode = timeout_normal;
 947         m_stylesheet_nprl = 0;
 948 #if YAZ_HAVE_XSLT
 949         if (m_stylesheet_xsp)
 950             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 951 #endif
 952         m_stylesheet_xsp = 0;
 953         timeout(m_client_idletime);
 954         send_PDU_convert(m_stylesheet_apdu);
 955     }
 956     else
 957         timeout(0);
 958 }
 959
 960 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 961 {
 962     if (m_frontend_type[0] != -1)
 963     {
 964         int i;
 965         for (i = 0; i < p->num_records; i++)
 966         {
 967             Z_NamePlusRecord *npr = p->records[i];
 968             if (npr->which == Z_NamePlusRecord_databaseRecord)
 969             {
 970                 Z_External *r = npr->u.databaseRecord;
 971                 if (r->which == Z_External_octet)
 972                 {
 973 #if !HAVE_USEMARCON
 974                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 975                         yaz_log(YLOG_LOG, "%sError: USEMARCON requested but not available",
 976                             m_session_str);
 977 #endif
 978 #if HAVE_USEMARCON
 979                     yaz_log(YLOG_DEBUG, "%sUSEMARCON stage1=%s stage2=%s",
 980                         m_session_str,
 981                         m_usemarcon_ini_stage1 ? m_usemarcon_ini_stage1 : "(none)",
 982                         m_usemarcon_ini_stage2 ? m_usemarcon_ini_stage2 : "(none)");
 983                     char *converted;
 984                     int convlen;
 985                     if (m_usemarcon->convert(m_usemarcon_ini_stage1, m_usemarcon_ini_stage2,
 986                         (char*) r->u.octet_aligned->buf, r->u.octet_aligned->len,
 987                         &converted, &convlen))
 988                     {
 989                         npr->u.databaseRecord =
 990                             z_ext_record_oid(odr_encode(),
 991                                              m_frontend_type,
 992                                              converted,
 993                                              strlen(converted));
 994                         free(converted);
 995                     }
 996                     else
 997 #endif
 998 /* HAVE_USEMARCON */
 999                     npr->u.databaseRecord =
1000                         z_ext_record_oid(odr_encode(),
1001                                          m_frontend_type,
1002                                          (char*) r->u.octet_aligned->buf,
1003                                          r->u.octet_aligned->len);
1004                 }
1005             }
1006         }
1007     }
1008 }
1009
1010 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
1011                                         const char *backend_charset)
1012 {
1013     int sel =   m_charset_converter->get_client_charset_selected();
1014     const char *client_record_charset =
1015         m_charset_converter->get_client_query_charset();
1016     if (sel && backend_charset && client_record_charset &&
1017         strcmp(backend_charset, client_record_charset))
1018     {
1019         int i;
1020         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
1021                                         backend_charset);
1022         yaz_marc_t mt = yaz_marc_create();
1023         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
1024         yaz_marc_iconv(mt, cd);
1025         for (i = 0; i < p->num_records; i++)
1026         {
1027             Z_NamePlusRecord *npr = p->records[i];
1028             if (npr->which == Z_NamePlusRecord_databaseRecord)
1029             {
1030                 Z_External *r = npr->u.databaseRecord;
1031                 const Odr_oid *oid = r->direct_reference;
1032                 if (!oid)
1033                     continue;
1034
1035                 if (!oid_oidcmp(oid, yaz_oid_recsyn_sutrs))
1036                 {
1037                     WRBUF w = wrbuf_alloc();
1038
1039                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
1040                                       r->u.octet_aligned->len);
1041                     npr->u.databaseRecord =
1042                         z_ext_record_oid(odr_encode(), oid, wrbuf_buf(w),
1043                                          wrbuf_len(w));
1044                     wrbuf_destroy(w);
1045                 }
1046                 else if (!oid_oidcmp(oid, yaz_oid_recsyn_xml))
1047                 {
1048                     ;
1049                 }
1050                 else if (r->which == Z_External_octet)
1051                 {
1052                     size_t rlen;
1053                     const char *result;
1054                     if (yaz_marc_decode_buf(mt,
1055                                             (char*) r->u.octet_aligned->buf,
1056                                             r->u.octet_aligned->len,
1057                                             &result, &rlen))
1058                     {
1059                         npr->u.databaseRecord =
1060                             z_ext_record_oid(odr_encode(), oid, result, rlen);
1061                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1062                                 m_session_str);
1063                     }
1064                 }
1065             }
1066         }
1067         if (cd)
1068             yaz_iconv_close(cd);
1069         yaz_marc_destroy(mt);
1070     }
1071 }
1072
1073 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1074                                    const char *backend_charset)
1075 {
1076     int i;
1077     if (!backend_charset)
1078         backend_charset = "MARC-8";
1079     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1080     yaz_marc_t mt = yaz_marc_create();
1081     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1082     yaz_marc_iconv(mt, cd);
1083     for (i = 0; i < p->num_records; i++)
1084     {
1085         Z_NamePlusRecord *npr = p->records[i];
1086         if (npr->which == Z_NamePlusRecord_databaseRecord)
1087         {
1088             Z_External *r = npr->u.databaseRecord;
1089             if (r->which == Z_External_OPAC)
1090             {
1091                 WRBUF w = wrbuf_alloc();
1092
1093                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
1094                 npr->u.databaseRecord = z_ext_record_oid(
1095                     odr_encode(), yaz_oid_recsyn_xml,
1096                     wrbuf_buf(w), wrbuf_len(w));
1097                 wrbuf_destroy(w);
1098             }
1099             else if (r->which == Z_External_octet)
1100             {
1101                 size_t rlen;
1102                 const char *result;
1103                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1104                                         r->u.octet_aligned->len,
1105                                         &result, &rlen))
1106                 {
1107                     npr->u.databaseRecord =
1108                         z_ext_record_oid(odr_encode(), yaz_oid_recsyn_xml,
1109                                          result, rlen);
1110                 }
1111             }
1112         }
1113     }
1114     if (cd)
1115         yaz_iconv_close(cd);
1116     yaz_marc_destroy(mt);
1117 }
1118
1119 void Yaz_Proxy::logtime()
1120 {
1121 #if HAVE_GETTIMEOFDAY
1122     struct timeval *tv = (struct timeval*) m_time_tv;
1123     if (tv->tv_sec)
1124     {
1125         struct timeval tv1;
1126         gettimeofday(&tv1, 0);
1127         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1128             (tv1.tv_usec - tv->tv_usec);
1129         if (diff >= 0)
1130             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1131                     diff/1000000, (diff/1000)%1000);
1132     }
1133     tv->tv_sec = 0;
1134     tv->tv_usec = 0;
1135 #endif
1136 }
1137
1138 int Yaz_Proxy::send_http_response(int code)
1139 {
1140     ODR o = odr_encode();
1141     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1142     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1143     if (m_http_version)
1144         hres->version = odr_strdup(o, m_http_version);
1145     if (m_http_keepalive)
1146         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1147     else
1148         timeout(0);
1149     if (code == 401)
1150         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate",
1151                           "Basic realm=\"YAZ Proxy\"");
1152
1153
1154     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1155     {
1156         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1157                  gdu_name(gdu));
1158     }
1159     int len;
1160     int r = send_GDU(gdu, &len);
1161     m_bytes_sent += len;
1162     m_bw_stat.add_bytes(len);
1163     logtime();
1164
1165     recv_GDU_more(true);
1166
1167     return r;
1168 }
1169
1170 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */)
1171 {
1172     ODR o = odr_encode();
1173     const char *ctype = "text/xml";
1174     Z_GDU *gdu = z_get_HTTP_Response(o, http_code);
1175     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1176     if (m_http_version)
1177         hres->version = odr_strdup(o, m_http_version);
1178     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1179     if (m_http_keepalive)
1180         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1181     else
1182         timeout(0);
1183     if (http_code == 401)
1184         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\"");
1185
1186     static Z_SOAP_Handler soap_handlers[2] = {
1187 #if YAZ_HAVE_XSLT
1188         { (char*) "http://www.loc.gov/zing/srw/", 0,
1189          (Z_SOAP_fun) yaz_srw_codec},
1190 #endif
1191         {0, 0, 0}
1192     };
1193
1194     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1195     soap_package->which = Z_SOAP_generic;
1196     soap_package->u.generic =
1197         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1198     soap_package->u.generic->no = 0;
1199     soap_package->u.generic->ns = soap_handlers[0].ns;
1200     soap_package->u.generic->p = (void *) srw_pdu;
1201     soap_package->ns = m_soap_ns;
1202     z_soap_codec_enc_xsl(o, &soap_package,
1203                          &hres->content_buf, &hres->content_len,
1204                          soap_handlers, 0, m_s2z_stylesheet);
1205     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1206     {
1207         yaz_log(YLOG_LOG, "%sSending %s to client", m_session_str,
1208                 gdu_name(gdu));
1209     }
1210     int len;
1211     int r = send_GDU(gdu, &len);
1212     m_bytes_sent += len;
1213     m_bw_stat.add_bytes(len);
1214     logtime();
1215
1216     recv_GDU_more(true);
1217
1218     return r;
1219 }
1220
1221 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1222 {
1223     ODR o = odr_encode();
1224     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1225         odr_malloc(o, sizeof(*diagnostic));
1226     int num_diagnostic = 1;
1227     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1228     return send_srw_search_response(diagnostic, num_diagnostic, srw_error == 3 ? 401 : 200);
1229 }
1230
1231 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1232                              Z_DefaultDiagFormat *ddf)
1233 {
1234     int bib1_code = *ddf->condition;
1235     if (bib1_code == 109)
1236         return 404;
1237     srw_res->num_diagnostics = 1;
1238     srw_res->diagnostics = (Z_SRW_diagnostic *)
1239         odr_malloc(o, sizeof(*srw_res->diagnostics));
1240     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1241                           yaz_diag_bib1_to_srw(*ddf->condition),
1242                           ddf->u.v2Addinfo);
1243     return 0;
1244 }
1245
1246 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_scanResponse *srw_res,
1247                              Z_DiagRec *dr)
1248 {
1249     if (dr->which == Z_DiagRec_defaultFormat)
1250     {
1251         int bib1_code = *dr->u.defaultFormat->condition;
1252         if (bib1_code == 109)
1253             return 404;
1254         srw_res->num_diagnostics = 1;
1255         srw_res->diagnostics = (Z_SRW_diagnostic *)
1256             odr_malloc(o, sizeof(*srw_res->diagnostics));
1257         yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1258                           yaz_diag_bib1_to_srw(*dr->u.defaultFormat->condition),
1259                           dr->u.defaultFormat->u.v2Addinfo);
1260     }
1261     return 0;
1262 }
1263
1264 #if YAZ_HAS_MK_SURROGATE
1265 #else
1266 static void yazproxy_mk_sru_surrogate(ODR o, Z_SRW_record *record, int pos,
1267                           int code, const char *details)
1268 {
1269     const char *message = yaz_diag_srw_str(code);
1270     int len = 200;
1271     if (message)
1272         len += strlen(message);
1273     if (details)
1274         len += strlen(details);
1275
1276     record->recordData_buf = (char *) odr_malloc(o, len);
1277
1278     sprintf(record->recordData_buf, "<diagnostic "
1279             "xmlns=\"http://www.loc.gov/zing/srw/diagnostic/\">\n"
1280             " <uri>info:srw/diagnostic/1/%d</uri>\n", code);
1281     if (details)
1282         sprintf(record->recordData_buf + strlen(record->recordData_buf),
1283                 " <details>%s</details>\n", details);
1284     if (message)
1285         sprintf(record->recordData_buf + strlen(record->recordData_buf),
1286                 " <message>%s</message>\n", message);
1287     sprintf(record->recordData_buf + strlen(record->recordData_buf),
1288             "</diagnostic>\n");
1289     record->recordData_len = strlen(record->recordData_buf);
1290     record->recordPosition = odr_intdup(o, pos);
1291     record->recordSchema = odr_strdup(o, "info:srw/schema/1/diagnostics-v1.1");
1292 }
1293 #endif
1294
1295 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1296 {
1297     ODR o = odr_encode();
1298     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1299     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1300
1301     srw_res->numberOfRecords = odr_intdup (o, hits);
1302     if (records && records->which == Z_Records_DBOSD)
1303     {
1304         srw_res->num_records =
1305             records->u.databaseOrSurDiagnostics->num_records;
1306         int i;
1307         srw_res->records = (Z_SRW_record *)
1308             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1309         for (i = 0; i < srw_res->num_records; i++)
1310         {
1311             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1312             if (npr->which != Z_NamePlusRecord_databaseRecord)
1313             {
1314 #if YAZ_HAS_MK_SURROGATE
1315                 yaz_mk_sru_surrogate(
1316                     o, srw_res->records + i, i+start,
1317                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1318 #else
1319                 yazproxy_mk_sru_surrogate(
1320                     o, srw_res->records + i, i+start,
1321                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1322 #endif
1323                 continue;
1324             }
1325             Z_External *r = npr->u.databaseRecord;
1326
1327             if (r->which == Z_External_octet
1328                 && !oid_oidcmp(r->direct_reference, yaz_oid_recsyn_xml))
1329             {
1330                 srw_res->records[i].recordSchema = m_schema;
1331                 srw_res->records[i].recordPacking = m_s2z_packing;
1332                 srw_res->records[i].recordData_buf = (char*)
1333                     r->u.octet_aligned->buf;
1334                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1335                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1336             }
1337             else
1338             {
1339 #if YAZ_HAS_MK_SURROGATE
1340                 yaz_mk_sru_surrogate(
1341                     o, srw_res->records + i, i+start,
1342                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1343 #else
1344                 yazproxy_mk_sru_surrogate(
1345                     o, srw_res->records + i, i+start,
1346                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1347 #endif
1348             }
1349         }
1350     }
1351     if (records && records->which == Z_Records_NSD)
1352     {
1353         int http_code;
1354         http_code = z_to_srw_diag(odr_encode(), srw_res,
1355                                    records->u.nonSurrogateDiagnostic);
1356         if (http_code)
1357             return send_http_response(http_code);
1358     }
1359     return send_srw_response(srw_pdu);
1360
1361 }
1362
1363 int Yaz_Proxy::send_to_srw_client_ok(Z_ListEntries *entries)
1364 {
1365     ODR o = odr_encode();
1366     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_scan_response);
1367     Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1368
1369     if (entries && entries->num_entries > 0)
1370     {
1371         srw_res->num_terms = entries->num_entries;
1372         int i;
1373         srw_res->terms = (Z_SRW_scanTerm *)
1374             odr_malloc(o, srw_res->num_terms * sizeof(Z_SRW_scanTerm));
1375         for (i = 0; i < srw_res->num_terms; i++)
1376         {
1377             if (entries->entries[i]->which == Z_Entry_termInfo)
1378             {
1379                 switch(entries->entries[i]->u.termInfo->term->which)
1380                 {
1381                 case Z_Term_general:
1382                     srw_res->terms[i].value = odr_strdupn(o,
1383                         (char *) entries->entries[i]->u.termInfo->term->u.general->buf,
1384                         entries->entries[i]->u.termInfo->term->u.general->len);
1385                     break;
1386                 default:
1387                     srw_res->terms[i].value = NULL;
1388                 }
1389                 if (entries->entries[i]->u.termInfo->globalOccurrences != NULL)
1390                     srw_res->terms[i].numberOfRecords = odr_intdup(o,
1391                         *entries->entries[i]->u.termInfo->globalOccurrences);
1392                 else
1393                     srw_res->terms[i].numberOfRecords = NULL;
1394                 if (entries->entries[i]->u.termInfo->displayTerm != NULL)
1395                     srw_res->terms[i].displayTerm = odr_strdup(o,
1396                         entries->entries[i]->u.termInfo->displayTerm);
1397                 else
1398                     srw_res->terms[i].displayTerm = NULL;
1399                 srw_res->terms[i].whereInList = NULL;
1400             }
1401         }
1402     }
1403     if (entries && entries->num_nonsurrogateDiagnostics > 0)
1404     {
1405         int http_code;
1406         http_code = z_to_srw_diag(odr_encode(), srw_res,
1407                                    entries->nonsurrogateDiagnostics[0]);
1408         if (http_code)
1409             return send_http_response(http_code);
1410     }
1411     return send_srw_response(srw_pdu);
1412
1413 }
1414
1415 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1416                                         int num_diagnostics, int http_code /* = 200 */)
1417 {
1418     ODR o = odr_encode();
1419     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1420     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1421
1422     srw_res->num_diagnostics = num_diagnostics;
1423     srw_res->diagnostics = diagnostics;
1424     return send_srw_response(srw_pdu, http_code);
1425 }
1426
1427 int Yaz_Proxy::send_srw_scan_response(Z_SRW_diagnostic *diagnostics,
1428                                         int num_diagnostics, int http_code /* = 200 */)
1429 {
1430     ODR o = odr_encode();
1431     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_scan_response);
1432     Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1433
1434     srw_res->num_diagnostics = num_diagnostics;
1435     srw_res->diagnostics = diagnostics;
1436     return send_srw_response(srw_pdu, http_code);
1437 }
1438
1439 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1440                                         int num_diagnostics)
1441 {
1442     int http_status = 404;
1443     Yaz_ProxyConfig *cfg = check_reconfigure();
1444     if (cfg)
1445     {
1446         int len;
1447         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1448                                        m_s2z_database, &len, &http_status);
1449         if (b)
1450         {
1451             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1452             Z_SRW_explainResponse *er = res->u.explain_response;
1453
1454             er->record.recordData_buf = b;
1455             er->record.recordData_len = len;
1456             er->record.recordPacking = m_s2z_packing;
1457             er->record.recordSchema = odr_strdup(odr_encode(),
1458                                                  "http://explain.z3950.org/dtd/2.0/");
1459
1460             er->diagnostics = diagnostics;
1461             er->num_diagnostics = num_diagnostics;
1462             return send_srw_response(res);
1463         }
1464     }
1465     return send_http_response(http_status);
1466 }
1467
1468 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1469 {
1470     if (m_http_version)
1471     {
1472         if (apdu->which == Z_APDU_initResponse)
1473         {
1474             Z_InitResponse *res = apdu->u.initResponse;
1475             if (*res->result == 0)
1476             {
1477                 send_to_srw_client_error(3, 0);
1478             }
1479             else if (m_s2z_search_apdu)
1480             {
1481                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1482             }
1483             else if (m_s2z_scan_apdu)
1484             {
1485                 handle_incoming_Z_PDU(m_s2z_scan_apdu);
1486             }
1487             else
1488             {
1489                 send_srw_explain_response(0, 0);
1490             }
1491         }
1492         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1493         {
1494             m_s2z_search_apdu = 0;
1495             Z_SearchResponse *res = apdu->u.searchResponse;
1496             m_s2z_hit_count = *res->resultCount;
1497             if (res->records && res->records->which == Z_Records_NSD)
1498             {
1499                 send_to_srw_client_ok(0, res->records, 1);
1500             }
1501             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1502             {
1503                 // adjust
1504                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1505
1506                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1507                 {
1508                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1509                         > m_s2z_hit_count)
1510                         *pr->numberOfRecordsRequested =
1511                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1512                 }
1513                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1514             }
1515             else
1516             {
1517                 m_s2z_present_apdu = 0;
1518                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1519             }
1520         }
1521         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1522         {
1523             int start =
1524                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1525
1526             m_s2z_present_apdu = 0;
1527             Z_PresentResponse *res = apdu->u.presentResponse;
1528             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1529         }
1530         else if (m_s2z_scan_apdu && apdu->which == Z_APDU_scanResponse)
1531         {
1532             Z_ScanResponse *res = apdu->u.scanResponse;
1533             send_to_srw_client_ok(res->entries);
1534         }
1535     }
1536     else
1537     {
1538         int len = 0;
1539         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1540             yaz_log(YLOG_LOG, "%sSending %s to client", m_session_str,
1541                      apdu_name(apdu));
1542         int r = send_Z_PDU(apdu, &len);
1543         m_bytes_sent += len;
1544         m_bw_stat.add_bytes(len);
1545         logtime();
1546         return r;
1547     }
1548     return 0;
1549 }
1550
1551 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1552 {
1553     int kill_session = 0;
1554     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1555
1556     if (new_id)
1557         *new_id = m_referenceId;
1558
1559     if (apdu->which == Z_APDU_searchResponse)
1560     {
1561         Z_SearchResponse *sr = apdu->u.searchResponse;
1562         Z_Records *p = sr->records;
1563         if (p && p->which == Z_Records_NSD)
1564         {
1565             Z_DiagRec dr, *dr_p = &dr;
1566             dr.which = Z_DiagRec_defaultFormat;
1567             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1568
1569             *sr->searchStatus = 0;
1570             display_diagrecs(&dr_p, 1);
1571         }
1572         else
1573         {
1574             if (p && p->which == Z_Records_DBOSD)
1575             {
1576                 if (m_backend_type
1577 #if HAVE_USEMARCON
1578                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1579 #endif
1580                     )
1581                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1582                 if (m_marcxml_mode == marcxml)
1583                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1584                                        m_backend_charset);
1585                 else
1586                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1587                                             m_backend_charset);
1588                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1589                     return 0;
1590
1591             }
1592             if (sr->resultCount)
1593             {
1594                 yaz_log(YLOG_LOG, "%s" ODR_INT_PRINTF " hits", m_session_str,
1595                         *sr->resultCount);
1596                 if (*sr->resultCount < 0)
1597                 {
1598                     m_flag_invalid_session = 1;
1599                     kill_session = 1;
1600
1601                     *sr->searchStatus = 0;
1602                     sr->records =
1603                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1604                     *sr->resultCount = 0;
1605                 }
1606             }
1607         }
1608     }
1609     else if (apdu->which == Z_APDU_presentResponse)
1610     {
1611         Z_PresentResponse *sr = apdu->u.presentResponse;
1612         Z_Records *p = sr->records;
1613         if (p && p->which == Z_Records_NSD)
1614         {
1615             Z_DiagRec dr, *dr_p = &dr;
1616             dr.which = Z_DiagRec_defaultFormat;
1617             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1618             if (*sr->presentStatus == Z_PresentStatus_success)
1619                 *sr->presentStatus = Z_PresentStatus_failure;
1620             display_diagrecs(&dr_p, 1);
1621         }
1622         if (p && p->which == Z_Records_DBOSD)
1623         {
1624             if (m_backend_type
1625 #if HAVE_USEMARCON
1626                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1627 #endif
1628                 )
1629                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1630             if (m_marcxml_mode == marcxml)
1631                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1632                                    m_backend_charset);
1633             else
1634                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1635                                         m_backend_charset);
1636             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1637                 return 0;
1638         }
1639     }
1640     else if (apdu->which == Z_APDU_initResponse)
1641     {
1642         //Get and check negotiation record
1643         //from init response.
1644         handle_charset_lang_negotiation(apdu);
1645
1646         if (m_initRequest_options)
1647         {
1648             Z_Options *nopt =
1649                 (Odr_bitmask *)odr_malloc(odr_encode(),
1650                                           sizeof(Odr_bitmask));
1651             ODR_MASK_ZERO(nopt);
1652
1653             int i;
1654             for (i = 0; i<24; i++)
1655                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1656                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1657                     ODR_MASK_SET(nopt, i);
1658             apdu->u.initResponse->options = nopt;
1659         }
1660         if (m_initRequest_version)
1661         {
1662             Z_ProtocolVersion *nopt =
1663                 (Odr_bitmask *)odr_malloc(odr_encode(),
1664                                           sizeof(Odr_bitmask));
1665             ODR_MASK_ZERO(nopt);
1666
1667             int i;
1668             for (i = 0; i<8; i++)
1669                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1670                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1671                     ODR_MASK_SET(nopt, i);
1672             apdu->u.initResponse->protocolVersion = nopt;
1673         }
1674         apdu->u.initResponse->preferredMessageSize =
1675             odr_intdup(odr_encode(),
1676                        m_client->m_initResponse_preferredMessageSize >
1677                        m_initRequest_preferredMessageSize ?
1678                        m_initRequest_preferredMessageSize :
1679                        m_client->m_initResponse_preferredMessageSize);
1680         apdu->u.initResponse->maximumRecordSize =
1681             odr_intdup(odr_encode(),
1682                        m_client->m_initResponse_maximumRecordSize >
1683                        m_initRequest_maximumRecordSize ?
1684                        m_initRequest_maximumRecordSize :
1685                        m_client->m_initResponse_maximumRecordSize);
1686     }
1687
1688     int r = send_PDU_convert(apdu);
1689     if (r)
1690         return r;
1691     if (kill_session)
1692     {
1693         delete m_client;
1694         m_client = 0;
1695         m_parent->pre_init();
1696     }
1697     return r;
1698 }
1699
1700 void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu)
1701 {
1702     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1703
1704     odr_reset(m_idAuthentication_odr);
1705     z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0);
1706     m_idAuthentication_ber_buf =
1707         odr_getbuf(m_idAuthentication_odr,
1708                    &m_idAuthentication_ber_size, 0);
1709 }
1710
1711 bool Yaz_ProxyClient::compare_charset(Z_APDU *apdu)
1712 {
1713     return true;
1714 }
1715
1716 bool Yaz_ProxyClient::compare_idAuthentication(Z_APDU *apdu)
1717 {
1718     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1719     ODR odr = odr_createmem(ODR_ENCODE);
1720
1721     z_IdAuthentication(odr, &t, 1, 0);
1722     int sz;
1723     char *buf = odr_getbuf(odr, &sz, 0);
1724     if (buf && m_idAuthentication_ber_buf
1725         && sz == m_idAuthentication_ber_size
1726         && !memcmp(m_idAuthentication_ber_buf, buf, sz))
1727     {
1728         odr_destroy(odr);
1729         return true;
1730     }
1731     odr_destroy(odr);
1732     if (!buf && !m_idAuthentication_ber_buf)
1733         return true;
1734     return false;
1735 }
1736
1737 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1738 {
1739     int len = 0;
1740     const char *apdu_name_tmp = apdu_name(apdu);
1741     int r = send_Z_PDU(apdu, &len);
1742     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1743         yaz_log(YLOG_LOG, "%sSending %s to %s %d bytes",
1744                  get_session_str(),
1745                  apdu_name_tmp, get_hostname(), len);
1746     m_bytes_sent += len;
1747     return r;
1748 }
1749
1750 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1751 {
1752     if (apdu->which == Z_APDU_presentRequest)
1753     {
1754         Z_PresentRequest *pr = apdu->u.presentRequest;
1755         int toget = *pr->numberOfRecordsRequested;
1756         int start = *pr->resultSetStartPoint;
1757
1758         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1759                 pr->resultSetId, start, toget);
1760
1761         if (*m_parent->m_optimize == '0')
1762             return apdu;
1763
1764         if (!m_client->m_last_resultSetId)
1765         {
1766             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1767             new_apdu->u.presentResponse->records =
1768                 create_nonSurrogateDiagnostics(
1769                     odr_encode(),
1770                     YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
1771                     pr->resultSetId);
1772             send_to_client(new_apdu);
1773             return 0;
1774         }
1775         if (start < 1 || toget < 0)
1776         {
1777             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1778             new_apdu->u.presentResponse->records =
1779                 create_nonSurrogateDiagnostics(
1780                     odr_encode(),
1781                     YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
1782                     0);
1783             send_to_client(new_apdu);
1784             return 0;
1785         }
1786         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1787         {
1788             if (start+toget-1 > m_client->m_last_resultCount)
1789             {
1790                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1791                 new_apdu->u.presentResponse->records =
1792                     create_nonSurrogateDiagnostics(
1793                         odr_encode(),
1794                         YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
1795                         0);
1796                 send_to_client(new_apdu);
1797                 return 0;
1798             }
1799             Z_NamePlusRecordList *npr;
1800 #if 0
1801             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1802                     m_session_str, start, toget, yaz_z3950oid_to_str(
1803                         pr->preferredRecordSyntax, &oclass));
1804 #endif
1805             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1806                                           pr->preferredRecordSyntax,
1807                                           pr->recordComposition))
1808             {
1809                 yaz_log(YLOG_LOG, "%sReturned cached records for present request",
1810                          m_session_str);
1811                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1812                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1813
1814                 new_apdu->u.presentResponse->numberOfRecordsReturned
1815                     = odr_intdup(odr_encode(), toget);
1816
1817                 new_apdu->u.presentResponse->records = (Z_Records*)
1818                     odr_malloc(odr_encode(), sizeof(Z_Records));
1819                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1820                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1821                 new_apdu->u.presentResponse->nextResultSetPosition =
1822                     odr_intdup(odr_encode(), start+toget);
1823
1824                 send_to_client(new_apdu);
1825                 return 0;
1826             }
1827         }
1828     }
1829
1830     if (apdu->which != Z_APDU_searchRequest)
1831         return apdu;
1832     Z_SearchRequest *sr = apdu->u.searchRequest;
1833     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1834     Yaz_Z_Databases this_databases;
1835
1836     this_databases.set(sr->num_databaseNames, (const char **)
1837                        sr->databaseNames);
1838
1839     this_query->set_Z_Query(sr->query);
1840
1841     // Check for non-negative piggyback params.
1842     if (*sr->smallSetUpperBound < 0
1843         || *sr->largeSetLowerBound < 0
1844         || *sr->mediumSetPresentNumber < 0)
1845     {
1846         Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1847         // Not a present request.. But can't find better diagnostic
1848         new_apdu->u.searchResponse->records =
1849             create_nonSurrogateDiagnostics(
1850                 odr_encode(),
1851                 YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, 0);
1852         send_to_client(new_apdu);
1853         return 0;
1854     }
1855
1856     char query_str[4096];
1857     this_query->print(query_str, sizeof(query_str)-1);
1858     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1859
1860     if (*m_parent->m_optimize != '0' &&
1861         m_client->m_last_ok && m_client->m_last_query &&
1862         m_client->m_last_query->match(this_query) &&
1863         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1864         m_client->m_last_databases.match(this_databases))
1865     {
1866         delete this_query;
1867         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1868             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1869         {
1870             Z_NamePlusRecordList *npr;
1871             int toget = *sr->mediumSetPresentNumber;
1872             Z_RecordComposition *comp = 0;
1873
1874             if (toget > m_client->m_last_resultCount)
1875                 toget = m_client->m_last_resultCount;
1876
1877             if (sr->mediumSetElementSetNames)
1878             {
1879                 comp = (Z_RecordComposition *)
1880                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1881                 comp->which = Z_RecordComp_simple;
1882                 comp->u.simple = sr->mediumSetElementSetNames;
1883             }
1884
1885             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1886                                           sr->preferredRecordSyntax, comp))
1887             {
1888                 yaz_log(YLOG_LOG, "%sReturned cached records for medium set",
1889                          m_session_str);
1890                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1891                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1892                 new_apdu->u.searchResponse->resultCount =
1893                     &m_client->m_last_resultCount;
1894
1895                 new_apdu->u.searchResponse->numberOfRecordsReturned
1896                     = odr_intdup(odr_encode(), toget);
1897
1898                 new_apdu->u.searchResponse->presentStatus =
1899                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1900                 new_apdu->u.searchResponse->records = (Z_Records*)
1901                     odr_malloc(odr_encode(), sizeof(Z_Records));
1902                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1903                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1904                 new_apdu->u.searchResponse->nextResultSetPosition =
1905                     odr_intdup(odr_encode(), toget+1);
1906                 send_to_client(new_apdu);
1907                 return 0;
1908             }
1909             else
1910             {
1911                 // medium Set
1912                 // send present request (medium size)
1913                 yaz_log(YLOG_LOG, "%sOptimizing search for medium set",
1914                          m_session_str);
1915
1916                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1917                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1918                 pr->referenceId = sr->referenceId;
1919                 pr->resultSetId = sr->resultSetName;
1920                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1921                 *pr->numberOfRecordsRequested = toget;
1922                 pr->recordComposition = comp;
1923                 m_client->m_sr_transform = 1;
1924                 return new_apdu;
1925             }
1926         }
1927         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1928             m_client->m_last_resultCount <= 0)
1929         {
1930             // large set. Return pseudo-search response immediately
1931             yaz_log(YLOG_LOG, "%sOptimizing search for large set",
1932                      m_session_str);
1933             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1934             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1935             new_apdu->u.searchResponse->resultCount =
1936                 &m_client->m_last_resultCount;
1937             send_to_client(new_apdu);
1938             return 0;
1939         }
1940         else
1941         {
1942             Z_NamePlusRecordList *npr;
1943             int toget = m_client->m_last_resultCount;
1944             Z_RecordComposition *comp = 0;
1945             // small set
1946             // send a present request (small set)
1947
1948             if (sr->smallSetElementSetNames)
1949             {
1950                 comp = (Z_RecordComposition *)
1951                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1952                 comp->which = Z_RecordComp_simple;
1953                 comp->u.simple = sr->smallSetElementSetNames;
1954             }
1955
1956             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1957                                           sr->preferredRecordSyntax, comp))
1958             {
1959                 yaz_log(YLOG_LOG, "%sReturned cached records for small set",
1960                          m_session_str);
1961                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1962                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1963                 new_apdu->u.searchResponse->resultCount =
1964                     &m_client->m_last_resultCount;
1965
1966                 new_apdu->u.searchResponse->numberOfRecordsReturned
1967                     = odr_intdup(odr_encode(), toget);
1968
1969                 new_apdu->u.searchResponse->presentStatus =
1970                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1971                 new_apdu->u.searchResponse->records = (Z_Records*)
1972                     odr_malloc(odr_encode(), sizeof(Z_Records));
1973                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1974                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1975                 new_apdu->u.searchResponse->nextResultSetPosition =
1976                     odr_intdup(odr_encode(), toget+1);
1977                 send_to_client(new_apdu);
1978                 return 0;
1979             }
1980             else
1981             {
1982                 yaz_log(YLOG_LOG, "%sOptimizing search for small set",
1983                          m_session_str);
1984                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1985                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1986                 pr->referenceId = sr->referenceId;
1987                 pr->resultSetId = sr->resultSetName;
1988                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1989                 *pr->numberOfRecordsRequested = toget;
1990                 pr->recordComposition = comp;
1991                 m_client->m_sr_transform = 1;
1992                 return new_apdu;
1993             }
1994         }
1995     }
1996     else  // query doesn't match
1997     {
1998         delete m_client->m_last_query;
1999         m_client->m_last_query = this_query;
2000         m_client->m_last_ok = 0;
2001         m_client->m_cache.clear();
2002         m_client->m_resultSetStartPoint = 0;
2003
2004         xfree(m_client->m_last_resultSetId);
2005         m_client->m_last_resultSetId = xstrdup(sr->resultSetName);
2006
2007         m_client->m_last_databases.set(sr->num_databaseNames,
2008                                        (const char **) sr->databaseNames);
2009     }
2010     return apdu;
2011 }
2012
2013
2014 void Yaz_Proxy::inc_request_no()
2015 {
2016     m_request_no++;
2017     char *cp = m_session_str + strlen(m_session_str)-1;
2018     if (*cp == ' ')
2019         cp--;
2020     while (*cp && *cp != ' ')
2021         cp--;
2022     if (*cp)
2023         sprintf(cp+1, "%d ", m_request_no);
2024 }
2025
2026 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
2027 {
2028     inc_request_no();
2029
2030     m_bytes_recv += len;
2031
2032     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2033         yaz_log(YLOG_LOG, "%sReceiving %s from client %d bytes",
2034                  m_session_str, gdu_name(apdu), len);
2035
2036 #if 0
2037     // try to make a _bad_ attribute set ID .. Don't enable this in prod.
2038     if (apdu->which == Z_GDU_Z3950
2039         && apdu->u.z3950->which == Z_APDU_searchRequest)
2040     {
2041         Z_SearchRequest *req = apdu->u.z3950->u.searchRequest;
2042         if (req->query && req->query->which == Z_Query_type_1)
2043         {
2044             Z_RPNQuery *rpnquery = req->query->u.type_1;
2045             if (rpnquery->attributeSetId)
2046             {
2047                 rpnquery->attributeSetId[0] = -2;
2048                 rpnquery->attributeSetId[1] = -1;
2049                 yaz_log(YLOG_WARN, "%sBAD FIXUP TEST", m_session_str);
2050             }
2051         }
2052     }
2053 #endif
2054
2055 #if HAVE_GETTIMEOFDAY
2056     gettimeofday((struct timeval *) m_time_tv, 0);
2057 #endif
2058     m_bw_stat.add_bytes(len);
2059     m_pdu_stat.add_bytes(1);
2060
2061     GDU *gdu = new GDU(apdu);
2062
2063     if (gdu->get() == 0)
2064     {
2065         delete gdu;
2066         yaz_log(YLOG_LOG, "%sUnable to encode package", m_session_str);
2067         m_in_queue.clear();
2068         dec_ref();
2069         return;
2070     }
2071     m_in_queue.enqueue(gdu);
2072     recv_GDU_more(false);
2073 }
2074
2075 void Yaz_Proxy::HTTP_Forwarded(Z_GDU *z_gdu)
2076 {
2077     if (z_gdu->which == Z_GDU_HTTP_Request)
2078     {
2079         Z_HTTP_Request *hreq = z_gdu->u.HTTP_Request;
2080         const char *x_forwarded_for =
2081             z_HTTP_header_lookup(hreq->headers, "X-Forwarded-For");
2082         if (x_forwarded_for)
2083         {
2084             xfree(m_peername);
2085             m_peername = (char*) xmalloc(strlen(x_forwarded_for)+5);
2086             sprintf(m_peername, "tcp:%s", x_forwarded_for);
2087
2088             yaz_log(YLOG_LOG, "%sHTTP Forwarded from %s", m_session_str,
2089                     m_peername);
2090             if (m_log_mask & PROXY_LOG_IP_CLIENT)
2091                 sprintf(m_session_str, "%ld:%d %.80s %d ",
2092                         (long) time(0), m_session_no, m_peername, m_request_no);
2093             else
2094                 sprintf(m_session_str, "%ld:%d %d ",
2095                         (long) time(0), m_session_no, m_request_no);
2096         }
2097     }
2098 }
2099
2100 void Yaz_Proxy::connect_stat(bool &block, int &reduce)
2101 {
2102
2103     m_parent->m_connect.cleanup(false);
2104     m_parent->m_connect.add_connect(m_peername);
2105
2106     int connect_total = m_parent->m_connect.get_total(m_peername);
2107     int max_connect = m_parent->m_max_connect;
2108
2109     if (max_connect && connect_total > max_connect)
2110     {
2111         yaz_log(YLOG_LOG, "%sconnect not accepted total=%d max=%d",
2112                 m_session_str, connect_total, max_connect);
2113         block = true;
2114     }
2115     else
2116         block = false;
2117     yaz_log(YLOG_LOG, "%sconnect accepted total=%d", m_session_str,
2118             connect_total);
2119
2120     int limit_connect = m_parent->m_limit_connect;
2121     if (limit_connect)
2122         reduce = connect_total / limit_connect;
2123     else
2124         reduce = 0;
2125 }
2126
2127 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
2128 {
2129     HTTP_Forwarded(gdu->get());
2130
2131     int reduce = 0;
2132
2133     if (m_request_no == 1)
2134     {
2135         bool block = false;
2136
2137         connect_stat(block, reduce);
2138
2139         if (block)
2140         {
2141             m_timeout_mode = timeout_busy;
2142             timeout(0);
2143             return;
2144         }
2145     }
2146
2147     int bw_total = m_bw_stat.get_total();
2148     int pdu_total = m_pdu_stat.get_total();
2149     int search_total = m_search_stat.get_total();
2150
2151     assert(m_timeout_mode == timeout_busy);
2152     assert(m_timeout_gdu == 0);
2153
2154     if (m_search_max)
2155         reduce += search_total / m_search_max;
2156     if (m_bw_max)
2157         reduce += (bw_total/m_bw_max);
2158     if (m_pdu_max)
2159     {
2160         if (pdu_total > m_pdu_max)
2161         {
2162             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
2163             reduce = (reduce > nreduce) ? reduce : nreduce;
2164         }
2165     }
2166     m_http_version = 0;
2167
2168 #if 0
2169     /* uncomment to force a big reduce */
2170     m_timeout_mode = timeout_reduce;
2171     m_timeout_gdu = gdu;
2172     timeout(3);       // call us reduce seconds later
2173     return;
2174 #endif
2175     if (reduce)
2176     {
2177         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d search=%d limit-bw=%d limit-pdu=%d limit-search=%d",
2178                 m_session_str, reduce, bw_total, pdu_total, search_total,
2179                 m_bw_max, m_pdu_max, m_search_max);
2180
2181         m_timeout_mode = timeout_reduce;
2182         m_timeout_gdu = gdu;
2183         timeout(reduce);       // call us reduce seconds later
2184     }
2185     else
2186         recv_GDU_normal(gdu);
2187 }
2188
2189 void Yaz_Proxy::recv_GDU_more(bool normal)
2190 {
2191     GDU *g;
2192     if (normal && m_timeout_mode == timeout_busy)
2193         m_timeout_mode = timeout_normal;
2194     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
2195     {
2196         m_timeout_mode = timeout_busy;
2197         inc_ref();
2198         recv_GDU_reduce(g);
2199         if (dec_ref())
2200             break;
2201     }
2202 }
2203
2204 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
2205 {
2206     Z_GDU *apdu = 0;
2207     gdu->move_away_gdu(odr_decode(), &apdu);
2208     delete gdu;
2209
2210     if (apdu->which == Z_GDU_Z3950)
2211         handle_incoming_Z_PDU(apdu->u.z3950);
2212     else if (apdu->which == Z_GDU_HTTP_Request)
2213         handle_incoming_HTTP(apdu->u.HTTP_Request);
2214 }
2215
2216 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
2217 {
2218     if (m_max_record_retrieve)
2219     {
2220         if (apdu->which == Z_APDU_presentRequest)
2221         {
2222             Z_PresentRequest *pr = apdu->u.presentRequest;
2223             if (pr->numberOfRecordsRequested &&
2224                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
2225                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
2226         }
2227     }
2228 }
2229
2230 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
2231 {
2232     if (apdu->which == Z_APDU_initRequest)
2233     {
2234         if (m_initRequest_options &&
2235             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
2236             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
2237         {
2238             // There is no negotiation proposal from
2239             // client's side. OK. The proxy negotiation
2240             // in use, only.
2241             Z_InitRequest *initRequest = apdu->u.initRequest;
2242             Z_OtherInformation **otherInfo;
2243             Z_OtherInformationUnit *oi;
2244             get_otherInfoAPDU(apdu, &otherInfo);
2245             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2246             if (oi)
2247             {
2248                 ODR_MASK_SET(initRequest->options,
2249                     Z_Options_negotiationModel);
2250                 oi->which = Z_OtherInfo_externallyDefinedInfo;
2251                 oi->information.externallyDefinedInfo =
2252                 yaz_set_proposal_charneg(odr_encode(),
2253                     (const char**)&m_proxy_negotiation_charset,
2254                     m_proxy_negotiation_charset ? 1:0,
2255                     (const char**)&m_proxy_negotiation_lang,
2256                     m_proxy_negotiation_lang ? 1:0,
2257                     1);
2258             }
2259         }
2260         else if (m_initRequest_options &&
2261                  ODR_MASK_GET(m_initRequest_options,
2262                               Z_Options_negotiationModel) &&
2263                  m_charset_converter->get_target_query_charset())
2264         {
2265             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
2266                     m_session_str,
2267                     m_charset_converter->get_target_query_charset());
2268             Z_InitRequest *initRequest = apdu->u.initRequest;
2269             Z_CharSetandLanguageNegotiation *negotiation =
2270                 yaz_get_charneg_record (initRequest->otherInfo);
2271             if (negotiation &&
2272                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
2273             {
2274                 NMEM nmem = nmem_create();
2275                 char **charsets = 0;
2276                 int num_charsets = 0;
2277                 char **langs = 0;
2278                 int num_langs = 0;
2279                 int selected = 0;
2280                 yaz_get_proposal_charneg (nmem, negotiation,
2281                                           &charsets, &num_charsets,
2282                                           &langs, &num_langs, &selected);
2283                 int i;
2284                 for (i = 0; i<num_charsets; i++)
2285                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
2286                             charsets[i]);
2287                 for (i = 0; i<num_langs; i++)
2288                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
2289                             langs[i]);
2290
2291                 const char *t_charset =
2292                     m_charset_converter->get_target_query_charset();
2293                 // sweep through charsets and pick the first supported
2294                 // conversion
2295                 for (i = 0; i<num_charsets; i++)
2296                 {
2297                     const char *c_charset = charsets[i];
2298                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
2299                         break;
2300                 }
2301                 if (i != num_charsets)
2302                 {
2303                     // got one .. set up ODR for reverse direction
2304                     const char *c_charset = charsets[i];
2305                     odr_set_charset(odr_encode(), c_charset, t_charset);
2306                     m_charset_converter->set_client_query_charset(c_charset);
2307                     m_charset_converter->set_client_charset_selected(selected);
2308                 }
2309                 nmem_destroy(nmem);
2310                 ODR_MASK_CLEAR(m_initRequest_options,
2311                                Z_Options_negotiationModel);
2312                 yaz_del_charneg_record(&initRequest->otherInfo);
2313             }
2314             else
2315             {
2316                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
2317                         m_session_str);
2318             }
2319         }
2320         else if (m_charset_converter->get_target_query_charset() &&
2321             m_proxy_negotiation_default_charset)
2322         {
2323             m_charset_converter->
2324                 set_client_query_charset(m_proxy_negotiation_default_charset);
2325         }
2326     }
2327     else if (apdu->which == Z_APDU_initResponse)
2328     {
2329         Z_InitResponse *initResponse = apdu->u.initResponse;
2330         Z_OtherInformation **otherInfo;
2331         get_otherInfoAPDU(apdu, &otherInfo);
2332
2333         Z_CharSetandLanguageNegotiation *charneg = 0;
2334
2335         if (otherInfo && *otherInfo &&
2336             ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel)
2337             && (charneg = yaz_get_charneg_record(*otherInfo)))
2338         {
2339             char *charset = 0;
2340             char *lang = 0;
2341             int selected = 0;
2342
2343             yaz_get_response_charneg(m_referenceId_mem, charneg,
2344                 &charset, &lang, &selected);
2345
2346             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
2347                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
2348
2349             if (m_initRequest_options &&
2350                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
2351             {
2352                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
2353                     m_session_str);
2354             }
2355             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2356             {
2357                 // negotiation-charset, negotiation-lang
2358                 // elements of config file in use.
2359
2360                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2361                     m_session_str);
2362
2363                 // clear negotiation option.
2364                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2365
2366                 // Delete negotiation (charneg-3) entry.
2367                 yaz_del_charneg_record(otherInfo);
2368             }
2369         }
2370         else
2371         {
2372             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2373             {
2374                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2375                         m_session_str);
2376             }
2377             else if (m_charset_converter->get_client_query_charset())
2378             {
2379                 Z_OtherInformation **otherInfo;
2380                 Z_OtherInformationUnit *oi;
2381                 get_otherInfoAPDU(apdu, &otherInfo);
2382                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2383                 if (oi)
2384                 {
2385                     ODR_MASK_SET(initResponse->options,
2386                                  Z_Options_negotiationModel);
2387                     if (m_initRequest_options)
2388                         ODR_MASK_SET(m_initRequest_options,
2389                                      Z_Options_negotiationModel);
2390
2391                     oi->which = Z_OtherInfo_externallyDefinedInfo;
2392                     oi->information.externallyDefinedInfo =
2393                         yaz_set_response_charneg(
2394                             odr_encode(),
2395                             m_charset_converter->get_client_query_charset(),
2396                             0 /* no lang */,
2397                             m_charset_converter->get_client_charset_selected());
2398                 }
2399             }
2400         }
2401     }
2402 }
2403
2404 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2405                                                      int error,
2406                                                      const char *addinfo)
2407 {
2408     Z_Records *rec = (Z_Records *)
2409         odr_malloc (odr, sizeof(*rec));
2410     Odr_int *err = (Odr_int *)
2411         odr_malloc (odr, sizeof(*err));
2412     Z_DiagRec *drec = (Z_DiagRec *)
2413         odr_malloc (odr, sizeof(*drec));
2414     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2415         odr_malloc (odr, sizeof(*dr));
2416     *err = error;
2417     rec->which = Z_Records_NSD;
2418     rec->u.nonSurrogateDiagnostic = dr;
2419     dr->diagnosticSetId = odr_oiddup(odr, yaz_oid_diagset_bib_1);
2420     dr->condition = err;
2421     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2422     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2423     return rec;
2424 }
2425
2426 Z_ListEntries *Yaz_Proxy::create_nonSurrogateDiagnostics2(ODR odr,
2427                                                      int error,
2428                                                      const char *addinfo)
2429 {
2430     Z_ListEntries *rec = (Z_ListEntries *)
2431         odr_malloc (odr, sizeof(*rec));
2432     Odr_int *err = (Odr_int *)
2433         odr_malloc (odr, sizeof(*err));
2434     Z_DiagRec *drec = (Z_DiagRec *)
2435         odr_malloc (odr, sizeof(*drec));
2436     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2437         odr_malloc (odr, sizeof(*dr));
2438     *err = error;
2439     drec->which = Z_DiagRec_defaultFormat;
2440     drec->u.defaultFormat = dr;
2441     rec->num_entries = 0;
2442     rec->entries = NULL;
2443     rec->num_nonsurrogateDiagnostics = 1;
2444     rec->nonsurrogateDiagnostics =
2445       (Z_DiagRec **)odr_malloc(odr, sizeof(Z_DiagRec *));
2446     rec->nonsurrogateDiagnostics[0] = drec;
2447     dr->diagnosticSetId = odr_oiddup(odr, yaz_oid_diagset_bib_1);
2448     dr->condition = err;
2449     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2450     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2451     return rec;
2452 }
2453
2454 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2455 {
2456     if (apdu->which == Z_APDU_searchRequest &&
2457         apdu->u.searchRequest->query &&
2458         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2459         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2460     {
2461         Z_RPNQuery *rpnquery = 0;
2462         Z_SearchRequest *sr = apdu->u.searchRequest;
2463         char *addinfo = 0;
2464
2465         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2466                 sr->query->u.type_104->u.cql);
2467
2468         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2469                                           &rpnquery, odr_encode(),
2470                                           &addinfo);
2471         if (r == -3)
2472             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2473         else if (r)
2474         {
2475             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2476             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2477
2478             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2479             new_apdu->u.searchResponse->records =
2480                 create_nonSurrogateDiagnostics(odr_encode(),
2481                                                yaz_diag_srw_to_bib1(r),
2482                                                addinfo);
2483             *new_apdu->u.searchResponse->searchStatus = 0;
2484
2485             send_to_client(new_apdu);
2486
2487             return 0;
2488         }
2489         else
2490         {
2491             sr->query->which = Z_Query_type_1;
2492             sr->query->u.type_1 = rpnquery;
2493         }
2494         return apdu;
2495     }
2496     else if (apdu->which == Z_APDU_scanRequest)
2497     {
2498         Z_RPNQuery *rpnquery = 0;
2499         Z_ScanRequest *sr = apdu->u.scanRequest;
2500         char *addinfo = 0;
2501
2502         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2503                 sr->termListAndStartPoint->term->u.characterString);
2504
2505         int r = m_cql2rpn.query_transform(sr->termListAndStartPoint->term->u.characterString,
2506                                           &rpnquery, odr_encode(),
2507                                           &addinfo);
2508         if (r == -3)
2509             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2510         else if (r)
2511         {
2512             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2513             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_scanResponse);
2514
2515             new_apdu->u.scanResponse->referenceId = sr->referenceId;
2516             new_apdu->u.scanResponse->entries =
2517                 create_nonSurrogateDiagnostics2(odr_encode(),
2518                                                yaz_diag_srw_to_bib1(r),
2519                                                addinfo);
2520             *new_apdu->u.scanResponse->scanStatus = Z_Scan_failure;
2521
2522             send_to_client(new_apdu);
2523
2524             return 0;
2525         }
2526         else
2527         {
2528             sr->attributeSet = rpnquery->attributeSetId;
2529             if (rpnquery->RPNStructure->which == Z_RPNStructure_simple)
2530                 sr->termListAndStartPoint = rpnquery->RPNStructure->u.simple->u.attributesPlusTerm;
2531         }
2532         return apdu;
2533     }
2534     return apdu;
2535 }
2536
2537 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2538 {
2539     if (apdu->which == Z_APDU_searchRequest &&
2540         apdu->u.searchRequest->query)
2541     {
2542         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2543             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2544         {
2545             if (m_http_version)
2546                 m_charset_converter->set_client_query_charset("UTF-8");
2547             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2548             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2549         }
2550     }
2551     else if (apdu->which == Z_APDU_scanRequest &&
2552         apdu->u.scanRequest->termListAndStartPoint)
2553     {
2554         if (apdu->u.scanRequest->termListAndStartPoint->term)
2555             if (m_http_version)
2556                 m_charset_converter->set_client_query_charset("UTF-8");
2557             Z_Term *term = apdu->u.scanRequest->termListAndStartPoint->term;
2558             m_charset_converter->convert_term(term, odr_encode());
2559     }
2560     return apdu;
2561 }
2562
2563
2564 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2565 {
2566     if (apdu->which == Z_APDU_searchRequest)
2567     {
2568         Z_SearchRequest *sr = apdu->u.searchRequest;
2569         int err = 0;
2570         char *addinfo = 0;
2571
2572         Yaz_ProxyConfig *cfg = check_reconfigure();
2573         if (cfg)
2574             err = cfg->check_query(odr_encode(), m_default_target,
2575                                    sr->query, &addinfo);
2576         if (err)
2577         {
2578             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2579
2580             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2581             new_apdu->u.searchResponse->records =
2582                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2583             *new_apdu->u.searchResponse->searchStatus = 0;
2584
2585             send_to_client(new_apdu);
2586
2587             return 0;
2588         }
2589     }
2590     else if (apdu->which == Z_APDU_scanRequest)
2591     {
2592         Z_ScanRequest *sr = apdu->u.scanRequest;
2593         int err = 0;
2594         char *addinfo = 0;
2595
2596         Yaz_ProxyConfig *cfg = check_reconfigure();
2597 // Something like this needs to be implemented later:
2598 /*
2599         if (cfg)
2600             err = cfg->check_type_1_attributes(odr_encode(), m_default_target,
2601                                    sr->termListAndStartPoint->attributes, &addinfo);
2602 */
2603         if (err)
2604         {
2605             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_scanResponse);
2606
2607             new_apdu->u.scanResponse->referenceId = sr->referenceId;
2608             new_apdu->u.scanResponse->entries =
2609                 create_nonSurrogateDiagnostics2(odr_encode(), err, addinfo);
2610             *new_apdu->u.scanResponse->scanStatus = Z_Scan_failure;
2611
2612             send_to_client(new_apdu);
2613
2614             return 0;
2615         }
2616     }
2617
2618     return apdu;
2619 }
2620
2621 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2622 {
2623     if (apdu->which != Z_APDU_initRequest)
2624         return 1;  // pass if no init request
2625     Z_InitRequest *req = apdu->u.initRequest;
2626
2627     Yaz_ProxyConfig *cfg = check_reconfigure();
2628     if (!cfg)
2629         return 1;  // pass if no config
2630
2631     int ret;
2632     if (req->idAuthentication == 0)
2633     {
2634         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2635                                          m_peername);
2636     }
2637     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2638     {
2639         ret = cfg->client_authentication(
2640             m_default_target,
2641             req->idAuthentication->u.idPass->userId,
2642             req->idAuthentication->u.idPass->groupId,
2643             req->idAuthentication->u.idPass->password,
2644             m_peername);
2645     }
2646     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2647     {
2648         char user[64], pass[64];
2649         *user = '\0';
2650         *pass = '\0';
2651         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2652         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2653                                          m_peername);
2654     }
2655     else
2656         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2657                                          m_peername);
2658     return ret;
2659 }
2660
2661 int Yaz_Proxy::handle_global_authentication(Z_APDU *apdu)
2662 {
2663     if (apdu->which != Z_APDU_initRequest)
2664         return 1;  // pass if no init request
2665     Z_InitRequest *req = apdu->u.initRequest;
2666
2667     Yaz_ProxyConfig *cfg = check_reconfigure();
2668     if (!cfg)
2669         return 1;  // pass if no config
2670
2671     int ret;
2672     if (req->idAuthentication == 0)
2673     {
2674         ret = cfg->global_client_authentication(0, 0, 0,
2675                                                 m_peername);
2676     }
2677     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2678     {
2679         ret = cfg->global_client_authentication(
2680             req->idAuthentication->u.idPass->userId,
2681             req->idAuthentication->u.idPass->groupId,
2682             req->idAuthentication->u.idPass->password,
2683             m_peername);
2684     }
2685     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2686     {
2687         char user[64], pass[64];
2688         *user = '\0';
2689         *pass = '\0';
2690         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2691         ret = cfg->global_client_authentication(user, 0, pass,
2692                                                 m_peername);
2693     }
2694     else
2695         ret = cfg->global_client_authentication(0, 0, 0, m_peername);
2696     return ret;
2697 }
2698
2699 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2700 {
2701     m_marcxml_mode = none;
2702     if (apdu->which == Z_APDU_searchRequest)
2703     {
2704         Z_SearchRequest *sr = apdu->u.searchRequest;
2705         int err = 0;
2706         char *addinfo = 0;
2707         Yaz_ProxyConfig *cfg = check_reconfigure();
2708
2709         Z_RecordComposition rc_temp, *rc = 0;
2710         if (sr->smallSetElementSetNames)
2711         {
2712             rc_temp.which = Z_RecordComp_simple;
2713             rc_temp.u.simple = sr->smallSetElementSetNames;
2714             rc = &rc_temp;
2715         }
2716
2717         if (sr->preferredRecordSyntax)
2718             oid_oidcpy(m_frontend_type, sr->preferredRecordSyntax);
2719         else
2720             m_frontend_type[0] = -1;
2721
2722         char *stylesheet_name = 0;
2723         if (cfg)
2724             err = cfg->check_syntax(odr_encode(),
2725                                     m_default_target,
2726                                     sr->preferredRecordSyntax, rc,
2727                                     &addinfo, &stylesheet_name, &m_schema,
2728                                     &m_backend_type, &m_backend_charset,
2729                                     &m_usemarcon_ini_stage1,
2730                                     &m_usemarcon_ini_stage2,
2731                                     &m_backend_elementset);
2732         if (stylesheet_name)
2733         {
2734             m_parent->low_socket_close();
2735
2736 #if YAZ_HAVE_XSLT
2737             if (m_stylesheet_xsp)
2738                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2739             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2740                                                        stylesheet_name);
2741 #endif
2742             m_stylesheet_offset = 0;
2743             xfree(stylesheet_name);
2744
2745             m_parent->low_socket_open();
2746         }
2747         if (err == -1)
2748         {
2749             sr->smallSetElementSetNames = 0;
2750             sr->mediumSetElementSetNames = 0;
2751             m_marcxml_mode = marcxml;
2752             sr->preferredRecordSyntax =
2753                 yaz_string_to_oid_odr(
2754                     yaz_oid_std(), CLASS_RECSYN,
2755                     m_backend_type ? m_backend_type : "usmarc",
2756                     odr_encode());
2757         }
2758         else if (err)
2759         {
2760             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2761
2762             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2763             new_apdu->u.searchResponse->records =
2764                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2765             *new_apdu->u.searchResponse->searchStatus = 0;
2766
2767             send_to_client(new_apdu);
2768
2769             return 0;
2770         }
2771         else if (m_backend_type)
2772         {
2773             sr->preferredRecordSyntax =
2774                 yaz_string_to_oid_odr(yaz_oid_std(), CLASS_RECSYN,
2775                                       m_backend_type, odr_encode());
2776         }
2777         if (m_backend_elementset)
2778         {
2779             Z_ElementSetNames *esn =
2780                 mk_esn_from_schema(
2781                     odr_encode(),
2782                     *m_backend_elementset ? m_backend_elementset : 0);
2783             sr->smallSetElementSetNames = esn;
2784             sr->mediumSetElementSetNames = esn;
2785         }
2786     }
2787     else if (apdu->which == Z_APDU_presentRequest)
2788     {
2789         Z_PresentRequest *pr = apdu->u.presentRequest;
2790         int err = 0;
2791         char *addinfo = 0;
2792         Yaz_ProxyConfig *cfg = check_reconfigure();
2793
2794         if (pr->preferredRecordSyntax)
2795             oid_oidcpy(m_frontend_type, pr->preferredRecordSyntax);
2796         else
2797             m_frontend_type[0] = -1;
2798
2799         char *stylesheet_name = 0;
2800         if (cfg)
2801             err = cfg->check_syntax(odr_encode(), m_default_target,
2802                                     pr->preferredRecordSyntax,
2803                                     pr->recordComposition,
2804                                     &addinfo, &stylesheet_name, &m_schema,
2805                                     &m_backend_type, &m_backend_charset,
2806                                     &m_usemarcon_ini_stage1,
2807                                     &m_usemarcon_ini_stage2,
2808                                     &m_backend_elementset);
2809         if (stylesheet_name)
2810         {
2811             m_parent->low_socket_close();
2812
2813 #if YAZ_HAVE_XSLT
2814             if (m_stylesheet_xsp)
2815                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2816             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2817                                                        stylesheet_name);
2818 #endif
2819             m_stylesheet_offset = 0;
2820             xfree(stylesheet_name);
2821
2822             m_parent->low_socket_open();
2823         }
2824         if (err == -1)
2825         {
2826             pr->recordComposition = 0;
2827             m_marcxml_mode = marcxml;
2828
2829             pr->preferredRecordSyntax =
2830                 yaz_string_to_oid_odr(
2831                     yaz_oid_std(), CLASS_RECSYN,
2832                     m_backend_type ? m_backend_type : "usmarc",
2833                     odr_encode());
2834         }
2835         else if (err)
2836         {
2837             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2838
2839             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2840             new_apdu->u.presentResponse->records =
2841                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2842             *new_apdu->u.presentResponse->presentStatus =
2843                 Z_PresentStatus_failure;
2844
2845             send_to_client(new_apdu);
2846
2847             return 0;
2848         }
2849         else if (m_backend_type)
2850         {
2851             pr->preferredRecordSyntax =
2852                 yaz_string_to_oid_odr(yaz_oid_std(),
2853                                       CLASS_RECSYN, m_backend_type,
2854                                       odr_encode());
2855         }
2856         if (m_backend_elementset)
2857         {
2858             Z_ElementSetNames *esn =
2859                 mk_esn_from_schema(
2860                     odr_encode(),
2861                     *m_backend_elementset ? m_backend_elementset : 0);
2862             Z_RecordComposition *comp = (Z_RecordComposition *)
2863                 odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
2864             comp->which = Z_RecordComp_simple;
2865             comp->u.simple = esn;
2866             pr->recordComposition = comp;
2867         }
2868     }
2869     return apdu;
2870 }
2871
2872 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2873 {
2874     if (!schema)
2875         return 0;
2876     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2877         odr_malloc(o, sizeof(Z_ElementSetNames));
2878     esn->which = Z_ElementSetNames_generic;
2879     esn->u.generic = odr_strdup(o, schema);
2880     return esn;
2881 }
2882
2883 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2884 {
2885     const char *t = 0;
2886     Yaz_ProxyConfig *cfg = check_reconfigure();
2887     if (cfg)
2888         t = cfg->get_explain_name(db, backend_db);
2889
2890     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2891     {
2892         releaseClient();
2893     }
2894
2895     if (t)
2896     {
2897         xfree(m_default_target);
2898         m_default_target = xstrdup(t);
2899     }
2900 }
2901
2902 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2903 {
2904     struct stat sbuf;
2905     if (strcmp(hreq->method, "GET"))
2906         return 0;
2907     if (hreq->path[0] != '/')
2908         return 0;
2909     const char *cp = hreq->path;
2910     while (*cp)
2911     {
2912         if (*cp == '/' && strchr("/.", cp[1]))
2913             return 0;
2914         cp++;
2915     }
2916
2917     Yaz_ProxyConfig *cfg = check_reconfigure();
2918
2919     if (!cfg->get_file_access_info(hreq->path+1))
2920         return 0;
2921
2922     const char *fname = hreq->path+1;
2923     if (stat(fname, &sbuf))
2924     {
2925         yaz_log(YLOG_LOG|YLOG_ERRNO, "%sstat failed for %s", m_session_str,
2926                 fname);
2927         return 0;
2928     }
2929     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2930     {
2931         yaz_log(YLOG_LOG, "%sNot a regular file %s", m_session_str, fname);
2932         return 0;
2933     }
2934     if (sbuf.st_size > (off_t) 1000000)
2935     {
2936         yaz_log(YLOG_WARN, "%sFile %s too large for transfer", m_session_str,
2937                 fname);
2938         return 0;
2939     }
2940
2941     ODR o = odr_encode();
2942
2943     const char *ctype = cfg->check_mime_type(fname);
2944     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2945     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2946     if (m_http_version)
2947         hres->version = odr_strdup(o, m_http_version);
2948     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2949     if (m_http_keepalive)
2950         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2951     else
2952         timeout(0);
2953
2954     hres->content_len = sbuf.st_size;
2955     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2956     FILE *f = fopen(fname, "rb");
2957     if (f)
2958     {
2959         fread(hres->content_buf, 1, hres->content_len, f);
2960         fclose(f);
2961     }
2962     else
2963     {
2964         return 0;
2965     }
2966     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2967     {
2968         yaz_log(YLOG_LOG, "%sSending file %s to client", m_session_str,
2969                  fname);
2970     }
2971     int len;
2972     send_GDU(gdu, &len);
2973     recv_GDU_more(true);
2974     return 1;
2975 }
2976
2977 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2978 {
2979     if (m_s2z_odr_init)
2980     {
2981         odr_destroy(m_s2z_odr_init);
2982         m_s2z_odr_init = 0;
2983     }
2984     if (m_s2z_odr_search)
2985     {
2986         odr_destroy(m_s2z_odr_search);
2987         m_s2z_odr_search = 0;
2988     }
2989     if (m_s2z_odr_scan)
2990     {
2991         odr_destroy(m_s2z_odr_scan);
2992         m_s2z_odr_scan = 0;
2993     }
2994
2995     m_http_keepalive = 0;
2996     m_http_version = 0;
2997     if (!strcmp(hreq->version, "1.0"))
2998     {
2999         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
3000         if (v && !strcmp(v, "Keep-Alive"))
3001             m_http_keepalive = 1;
3002         else
3003             m_http_keepalive = 0;
3004         m_http_version = "1.0";
3005     }
3006     else
3007     {
3008         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
3009         if (v && !strcmp(v, "close"))
3010             m_http_keepalive = 0;
3011         else
3012             m_http_keepalive = 1;
3013         m_http_version = "1.1";
3014     }
3015
3016     const char *a = z_HTTP_header_lookup(hreq->headers, "Authorization");
3017     char authorization_str[255];
3018     *authorization_str = '\0';
3019     if (a && strncasecmp(a, "Basic ", 6) == 0)
3020         base64_decode(a + 6, authorization_str, 254);
3021
3022     Z_SRW_PDU *srw_pdu = 0;
3023     Z_SOAP *soap_package = 0;
3024     char *charset = 0;
3025     Z_SRW_diagnostic *diagnostic = 0;
3026     int num_diagnostic = 0;
3027
3028     yaz_log(YLOG_LOG, "%s%s %s", m_session_str, hreq->method, hreq->path);
3029
3030     if (file_access(hreq))
3031     {
3032         return;
3033     }
3034     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
3035                             &charset) == 0
3036              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
3037                                &charset, &diagnostic, &num_diagnostic) == 0)
3038     {
3039         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
3040         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
3041         m_s2z_odr_scan = odr_createmem(ODR_ENCODE);
3042         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
3043         m_s2z_init_apdu = 0;
3044         m_s2z_search_apdu = 0;
3045         m_s2z_present_apdu = 0;
3046         m_s2z_scan_apdu = 0;
3047
3048         m_s2z_stylesheet = 0;
3049
3050         Z_IdAuthentication *auth = NULL;
3051         if (srw_pdu->username && !srw_pdu->password)
3052         {
3053             yaz_log(YLOG_LOG, "username: %s\n", srw_pdu->username);
3054             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
3055             auth->which = Z_IdAuthentication_open;
3056             auth->u.open = odr_strdup(m_s2z_odr_init, srw_pdu->username);
3057         }
3058         else if (srw_pdu->username && srw_pdu->password)
3059         {
3060             yaz_log(YLOG_LOG, "username/password: %s/%s\n",
3061                     srw_pdu->username, srw_pdu->password);
3062             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
3063             auth->which = Z_IdAuthentication_idPass;
3064             auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
3065             auth->u.idPass->groupId = NULL;
3066             auth->u.idPass->password = odr_strdup(m_s2z_odr_init, srw_pdu->password);
3067             auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, srw_pdu->username);
3068         }
3069         else
3070         {
3071             if (*authorization_str)
3072             {
3073                 yaz_log(YLOG_LOG, "authorization_str present: %s\n", authorization_str);
3074                 auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
3075                 auth->which = Z_IdAuthentication_idPass;
3076                 auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
3077                 auth->u.idPass->groupId = NULL;
3078                 char *p = strchr(authorization_str, ':');
3079                 if (p)
3080                 {
3081                     *p = '\0';
3082                     p++;
3083                     auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
3084                 }
3085                 auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
3086             }
3087         }
3088
3089         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
3090         {
3091
3092             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
3093
3094             const char *backend_db = srw_req->database;
3095             srw_get_client(srw_req->database, &backend_db);
3096
3097             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
3098             // recordXPath unsupported.
3099             if (srw_req->recordXPath)
3100             {
3101                 yaz_add_srw_diagnostic(odr_decode(),
3102                                        &diagnostic, &num_diagnostic,
3103                                        72, 0);
3104             }
3105             // sort unsupported
3106             if (srw_req->sort_type != Z_SRW_sort_type_none)
3107             {
3108                 yaz_add_srw_diagnostic(odr_decode(),
3109                                        &diagnostic, &num_diagnostic,
3110                                        80, 0);
3111             }
3112             // save stylesheet
3113             if (srw_req->stylesheet)
3114                 m_s2z_stylesheet =
3115                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
3116
3117             // set packing for response records ..
3118             if (srw_req->recordPacking &&
3119                 !strcmp(srw_req->recordPacking, "xml"))
3120                 m_s2z_packing = Z_SRW_recordPacking_XML;
3121             else
3122                 m_s2z_packing = Z_SRW_recordPacking_string;
3123
3124             if (num_diagnostic)
3125             {
3126                 Z_SRW_PDU *srw_pdu =
3127                     yaz_srw_get(odr_encode(),
3128                                 Z_SRW_searchRetrieve_response);
3129                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
3130
3131                 srw_res->diagnostics = diagnostic;
3132                 srw_res->num_diagnostics = num_diagnostic;
3133                 send_srw_response(srw_pdu);
3134                 return;
3135             }
3136
3137             // prepare search PDU
3138             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
3139                                           Z_APDU_searchRequest);
3140             Z_SearchRequest *z_searchRequest =
3141                 m_s2z_search_apdu->u.searchRequest;
3142
3143             z_searchRequest->num_databaseNames = 1;
3144             z_searchRequest->databaseNames = (char**)
3145                 odr_malloc(m_s2z_odr_search, sizeof(char *));
3146             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
3147                                                            backend_db);
3148
3149             // query transformation
3150             Z_Query *query = (Z_Query *)
3151                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
3152             z_searchRequest->query = query;
3153
3154             if (srw_req->query_type == Z_SRW_query_type_cql)
3155             {
3156                 Z_External *ext = (Z_External *)
3157                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
3158                 ext->direct_reference =
3159                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
3160                 ext->indirect_reference = 0;
3161                 ext->descriptor = 0;
3162                 ext->which = Z_External_CQL;
3163                 ext->u.cql = srw_req->query.cql;
3164
3165                 query->which = Z_Query_type_104;
3166                 query->u.type_104 =  ext;
3167             }
3168             else if (srw_req->query_type == Z_SRW_query_type_pqf)
3169             {
3170                 Z_RPNQuery *RPNquery;
3171                 YAZ_PQF_Parser pqf_parser;
3172
3173                 pqf_parser = yaz_pqf_create ();
3174
3175                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
3176                                           srw_req->query.pqf);
3177                 if (!RPNquery)
3178                 {
3179                     const char *pqf_msg;
3180                     size_t off;
3181                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
3182                     int ioff = off;
3183                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
3184                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
3185
3186                     send_to_srw_client_error(10, 0);
3187                     return;
3188                 }
3189                 query->which = Z_Query_type_1;
3190                 query->u.type_1 =  RPNquery;
3191
3192                 yaz_pqf_destroy (pqf_parser);
3193             }
3194             else
3195             {
3196                 send_to_srw_client_error(7, "query");
3197                 return;
3198             }
3199
3200             // present
3201             m_s2z_present_apdu = 0;
3202             int max = 0;
3203             if (srw_req->maximumRecords)
3204                 max = *srw_req->maximumRecords;
3205             int start = 1;
3206             if (srw_req->startRecord)
3207                 start = *srw_req->startRecord;
3208             if (max > 0)
3209             {
3210                 // Some backend, such as Voyager doesn't honor piggyback
3211                 // So we use present always (0 &&).
3212                 if (0 && start <= 1)  // Z39.50 piggyback
3213                 {
3214                     *z_searchRequest->smallSetUpperBound = max;
3215                     *z_searchRequest->mediumSetPresentNumber = max;
3216                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
3217
3218                     z_searchRequest->preferredRecordSyntax =
3219                         odr_oiddup(m_s2z_odr_search, yaz_oid_recsyn_xml);
3220
3221                     if (srw_req->recordSchema)
3222                     {
3223                         z_searchRequest->smallSetElementSetNames =
3224                             z_searchRequest->mediumSetElementSetNames =
3225                             mk_esn_from_schema(m_s2z_odr_search,
3226                                                srw_req->recordSchema);
3227                     }
3228                 }
3229                 else   // Z39.50 present
3230                 {
3231                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
3232                                                    Z_APDU_presentRequest);
3233                     Z_PresentRequest *z_presentRequest =
3234                         m_s2z_present_apdu->u.presentRequest;
3235                     *z_presentRequest->resultSetStartPoint = start;
3236                     *z_presentRequest->numberOfRecordsRequested = max;
3237
3238                     z_presentRequest->preferredRecordSyntax =
3239                         odr_oiddup(m_s2z_odr_search, yaz_oid_recsyn_xml);
3240                     if (srw_req->recordSchema)
3241                     {
3242                         z_presentRequest->recordComposition =
3243                             (Z_RecordComposition *)
3244                             odr_malloc(m_s2z_odr_search,
3245                                        sizeof(Z_RecordComposition));
3246                         z_presentRequest->recordComposition->which =
3247                             Z_RecordComp_simple;
3248                         z_presentRequest->recordComposition->u.simple =
3249                             mk_esn_from_schema(m_s2z_odr_search,
3250                                                srw_req->recordSchema);
3251                     }
3252                 }
3253             }
3254             if (!m_client)
3255             {
3256                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
3257                                             Z_APDU_initRequest);
3258
3259                 ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan);
3260                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
3261
3262                 // prevent m_initRequest_apdu memory from being grabbed
3263                 // in Yaz_Proxy::handle_incoming_Z_PDU
3264                 m_initRequest_apdu = m_s2z_init_apdu;
3265                 handle_incoming_Z_PDU(m_s2z_init_apdu);
3266                 return;
3267             }
3268             else
3269             {
3270                 handle_incoming_Z_PDU(m_s2z_search_apdu);
3271                 return;
3272             }
3273         }
3274         else if (srw_pdu->which == Z_SRW_explain_request)
3275         {
3276             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
3277
3278             const char *backend_db = srw_req->database;
3279             srw_get_client(srw_req->database, &backend_db);
3280
3281             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
3282
3283             // save stylesheet
3284             if (srw_req->stylesheet)
3285                 m_s2z_stylesheet =
3286                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
3287
3288             if (srw_req->recordPacking &&
3289                 !strcmp(srw_req->recordPacking, "xml"))
3290                 m_s2z_packing = Z_SRW_recordPacking_XML;
3291             else
3292                 m_s2z_packing = Z_SRW_recordPacking_string;
3293
3294             if (num_diagnostic)
3295             {
3296                 send_srw_explain_response(diagnostic, num_diagnostic);
3297                 return;
3298             }
3299
3300             if (!m_client)
3301             {
3302                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
3303                                             Z_APDU_initRequest);
3304
3305                 ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan);
3306                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
3307
3308                 // prevent m_initRequest_apdu memory from being grabbed
3309                 // in Yaz_Proxy::handle_incoming_Z_PDU
3310                 m_initRequest_apdu = m_s2z_init_apdu;
3311                 handle_incoming_Z_PDU(m_s2z_init_apdu);
3312             }
3313             else
3314                 send_srw_explain_response(0, 0);
3315             return;
3316         }
3317         else if (srw_pdu->which == Z_SRW_scan_request)
3318         {
3319             Z_SRW_scanRequest *srw_req = srw_pdu->u.scan_request;
3320
3321             const char *backend_db = srw_req->database;
3322             srw_get_client(srw_req->database, &backend_db);
3323
3324             m_s2z_database = odr_strdup(m_s2z_odr_init,
3325                                         srw_req->database);
3326             // save stylesheet
3327             if (srw_req->stylesheet)
3328                 m_s2z_stylesheet =
3329                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
3330
3331             // prepare scan PDU
3332             m_s2z_scan_apdu = zget_APDU(m_s2z_odr_scan,
3333                                           Z_APDU_scanRequest);
3334             Z_ScanRequest *z_scanRequest =
3335                 m_s2z_scan_apdu->u.scanRequest;
3336
3337             z_scanRequest->num_databaseNames = 1;
3338             z_scanRequest->databaseNames = (char**)
3339                 odr_malloc(m_s2z_odr_scan, sizeof(char *));
3340             z_scanRequest->databaseNames[0] = odr_strdup(m_s2z_odr_scan,
3341                                                            backend_db);
3342
3343              // query transformation
3344             if (srw_req->query_type == Z_SRW_query_type_cql)
3345             {
3346                 z_scanRequest->termListAndStartPoint =
3347                     (Z_AttributesPlusTerm *)odr_malloc(m_s2z_odr_scan, sizeof(Z_AttributesPlusTerm));
3348                 z_scanRequest->termListAndStartPoint->attributes = NULL;
3349                 z_scanRequest->termListAndStartPoint->term =
3350                    (Z_Term *)odr_malloc(m_s2z_odr_scan, sizeof(Z_Term));
3351                 z_scanRequest->termListAndStartPoint->term->which =
3352                   Z_Term_characterString;
3353                 z_scanRequest->termListAndStartPoint->term->u.characterString =
3354                   odr_strdup(m_s2z_odr_scan, srw_req->scanClause.cql);
3355             }
3356
3357             if (srw_req->responsePosition)
3358                 z_scanRequest->preferredPositionInResponse =
3359                     odr_intdup(m_s2z_odr_scan, *srw_req->responsePosition);
3360             if (srw_req->maximumTerms)
3361                 *z_scanRequest->numberOfTermsRequested = *srw_req->maximumTerms;
3362
3363             if (!m_client)
3364             {
3365                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
3366                                             Z_APDU_initRequest);
3367
3368                 ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan);
3369                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
3370
3371                 // prevent m_initRequest_apdu memory from being grabbed
3372                 // in Yaz_Proxy::handle_incoming_Z_PDU
3373                 m_initRequest_apdu = m_s2z_init_apdu;
3374                 handle_incoming_Z_PDU(m_s2z_init_apdu);
3375                 return;
3376             }
3377             else
3378             {
3379                 handle_incoming_Z_PDU(m_s2z_scan_apdu);
3380                 return;
3381             }
3382         }
3383         else
3384         {
3385             m_s2z_database = 0;
3386
3387             send_to_srw_client_error(4, 0);
3388         }
3389     }
3390     send_http_response(400);
3391 }
3392
3393 void Yaz_Proxy::handle_init(Z_APDU *apdu)
3394 {
3395
3396     Z_OtherInformation **oi;
3397     get_otherInfoAPDU(apdu, &oi);
3398
3399     if (apdu->u.initRequest->implementationId)
3400         yaz_log(YLOG_LOG, "%simplementationId: %s",
3401                 m_session_str, apdu->u.initRequest->implementationId);
3402     if (apdu->u.initRequest->implementationName)
3403         yaz_log(YLOG_LOG, "%simplementationName: %s",
3404                 m_session_str, apdu->u.initRequest->implementationName);
3405     if (apdu->u.initRequest->implementationVersion)
3406         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
3407                 m_session_str, apdu->u.initRequest->implementationVersion);
3408     if (m_initRequest_apdu == 0)
3409     {
3410         if (m_initRequest_mem)
3411             nmem_destroy(m_initRequest_mem);
3412
3413         m_initRequest_apdu = apdu;
3414         m_initRequest_mem = odr_extract_mem(odr_decode());
3415
3416         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
3417             preferredMessageSize;
3418         *apdu->u.initRequest->preferredMessageSize = 64*1024*1024;
3419         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
3420             maximumRecordSize;
3421         *apdu->u.initRequest->maximumRecordSize = 64*1024*1024;
3422
3423         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
3424             yaz_get_charneg_record(*oi);
3425
3426         // Save proposal charsets and langs.
3427         if (ODR_MASK_GET(apdu->u.initRequest->options,
3428                          Z_Options_negotiationModel)
3429             && charSetandLangRecord)
3430         {
3431
3432             yaz_get_proposal_charneg(m_referenceId_mem,
3433                                      charSetandLangRecord,
3434                                      &m_initRequest_oi_negotiation_charsets,
3435                                      &m_initRequest_oi_negotiation_num_charsets,
3436                                      &m_initRequest_oi_negotiation_langs,
3437                                      &m_initRequest_oi_negotiation_num_langs,
3438                                      &m_initRequest_oi_negotiation_selected);
3439
3440             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
3441             {
3442                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
3443                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
3444                         m_initRequest_oi_negotiation_charsets[i]:"none");
3445             }
3446             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
3447             {
3448                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
3449                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
3450                         m_initRequest_oi_negotiation_langs[i]:"none");
3451             }
3452             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
3453                     m_session_str, m_initRequest_oi_negotiation_selected);
3454         }
3455         // save init options for the response..
3456         m_initRequest_options = apdu->u.initRequest->options;
3457
3458         apdu->u.initRequest->options =
3459             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
3460                                        sizeof(Odr_bitmask));
3461         ODR_MASK_ZERO(apdu->u.initRequest->options);
3462         int i;
3463         for (i = 0; i<= 24; i++)
3464             ODR_MASK_SET(apdu->u.initRequest->options, i);
3465         // check negotiation option
3466         if (!ODR_MASK_GET(m_initRequest_options,
3467                           Z_Options_negotiationModel))
3468         {
3469             ODR_MASK_CLEAR(apdu->u.initRequest->options,
3470                            Z_Options_negotiationModel);
3471         }
3472         ODR_MASK_CLEAR(apdu->u.initRequest->options,
3473                        Z_Options_concurrentOperations);
3474         // make new version
3475         m_initRequest_version = apdu->u.initRequest->protocolVersion;
3476         apdu->u.initRequest->protocolVersion =
3477             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
3478                                        sizeof(Odr_bitmask));
3479         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
3480
3481         for (i = 0; i<= 8; i++)
3482             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
3483     }
3484     handle_charset_lang_negotiation(apdu);
3485     if (m_client->m_init_flag)
3486     {
3487         if (handle_init_response_for_invalid_session(apdu))
3488             return;
3489         if (m_client->m_initResponse)
3490         {
3491             Z_APDU *apdu2 = m_client->m_initResponse;
3492             apdu2->u.initResponse->otherInfo = 0;
3493             if (m_client->m_cookie && *m_client->m_cookie)
3494                 set_otherInformationString(apdu2, yaz_oid_userinfo_cookie,
3495                                            1, m_client->m_cookie);
3496             apdu2->u.initResponse->referenceId =
3497                 apdu->u.initRequest->referenceId;
3498             apdu2->u.initResponse->options = m_client->m_initResponse_options;
3499             apdu2->u.initResponse->protocolVersion =
3500                 m_client->m_initResponse_version;
3501
3502             handle_charset_lang_negotiation(apdu2);
3503
3504             if (m_timeout_mode == timeout_busy)
3505                 m_timeout_mode = timeout_normal;
3506             send_to_client(apdu2);
3507             return;
3508         }
3509     }
3510     m_client->m_init_flag = 1;
3511
3512     if (m_num_msg_threads && m_my_thread)
3513     {
3514         Auth_Msg *m = new Auth_Msg;
3515         m->m_proxy = this;
3516         z_APDU(odr_encode(), &apdu, 0, "encode");
3517         char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
3518         m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
3519         memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
3520         odr_reset(odr_encode());
3521
3522         inc_ref();
3523         m_my_thread->put(m);
3524     }
3525     else
3526     {
3527         int ret = handle_authentication(apdu);
3528         result_authentication(apdu, ret);
3529     }
3530 }
3531
3532 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
3533 {
3534     Z_ReferenceId **refid = get_referenceIdP(apdu);
3535     nmem_reset(m_referenceId_mem);
3536     if (refid && *refid)
3537     {
3538         m_referenceId = (Z_ReferenceId *)
3539             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
3540         m_referenceId->len = m_referenceId->size = (*refid)->len;
3541         m_referenceId->buf = (unsigned char *)
3542             nmem_malloc(m_referenceId_mem, (*refid)->len);
3543         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
3544     }
3545     else
3546         m_referenceId = 0;
3547
3548     if (!m_client && m_flag_invalid_session)
3549     {
3550         // Got request for a session that is invalid..
3551         m_apdu_invalid_session = apdu; // save package
3552         m_mem_invalid_session = odr_extract_mem(odr_decode());
3553         apdu = m_initRequest_apdu;     // but throw an init to the target
3554     }
3555
3556     if (apdu->which == Z_APDU_searchRequest)
3557         m_search_stat.add_bytes(1);
3558
3559     // Handle global authentication
3560     if (!handle_global_authentication(apdu))
3561     {
3562         if (m_http_version)
3563         {   // HTTP. Send unauthorized
3564             send_http_response(401);
3565             return;
3566         }
3567         else
3568         {
3569             // Z39.50 just shutdown
3570             timeout(0);
3571             return;
3572         }
3573         return;
3574     }
3575
3576     // Determine our client.
3577     Z_OtherInformation **oi;
3578     get_otherInfoAPDU(apdu, &oi);
3579     int http_code = 404;
3580     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi), &http_code);
3581     if (!m_client)
3582     {
3583         if (m_http_version)
3584         {   // HTTP. Send not found
3585             send_http_response(http_code);
3586             return;
3587         }
3588         else
3589         {
3590             // Z39.50 just shutdown
3591             timeout(0);
3592             return;
3593         }
3594     }
3595
3596     m_client->m_server = this;
3597
3598     if (apdu->which == Z_APDU_initRequest)
3599         handle_init(apdu);
3600     else
3601         handle_incoming_Z_PDU_2(apdu);
3602 }
3603
3604 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
3605 {
3606     handle_max_record_retrieve(apdu);
3607
3608     if (apdu)
3609         apdu = handle_syntax_validation(apdu);
3610
3611     if (apdu)
3612         apdu = handle_query_transformation(apdu);
3613
3614     if (apdu)
3615         apdu = handle_target_charset_conversion(apdu);
3616
3617     if (apdu)
3618         apdu = handle_query_validation(apdu);
3619
3620     if (apdu)
3621         apdu = result_set_optimize(apdu);
3622
3623     if (!apdu)
3624     {
3625         m_client->timeout(m_target_idletime);  // mark it active even
3626         recv_GDU_more(true);
3627         // though we didn't use it
3628         return;
3629     }
3630
3631     // delete other info construct completely if 0 elements
3632     Z_OtherInformation **oi;
3633     get_otherInfoAPDU(apdu, &oi);
3634     if (oi && *oi && (*oi)->num_elements == 0)
3635         *oi = 0;
3636
3637     if (apdu->which == Z_APDU_presentRequest &&
3638         m_client->m_resultSetStartPoint == 0)
3639     {
3640         Z_PresentRequest *pr = apdu->u.presentRequest;
3641         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3642         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3643     } else {
3644         m_client->m_resultSetStartPoint = 0;
3645     }
3646     if (m_client->send_to_target(apdu) < 0)
3647     {
3648         m_client->shutdown();
3649     }
3650     else
3651         m_client->m_waiting = 1;
3652 }
3653
3654 void Yaz_Proxy::connectNotify()
3655 {
3656 }
3657
3658 void Yaz_Proxy::releaseClient()
3659 {
3660     xfree(m_proxyTarget);
3661     m_proxyTarget = 0;
3662     m_flag_invalid_session = 0;
3663     // only keep if keep_alive flag is set...
3664     if (m_client &&
3665         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3666         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3667         m_client->m_waiting == 0)
3668     {
3669         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3670                  m_session_str,
3671                  m_client->get_hostname());
3672         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3673                 m_session_str, m_client->m_pdu_recv,
3674                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3675                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3676         assert (m_client->m_waiting != 2);
3677         // Tell client (if any) that no server connection is there..
3678         m_client->m_server = 0;
3679         m_client = 0;
3680     }
3681     else if (m_client)
3682     {
3683         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) close %s",
3684                  m_session_str,
3685                  m_client->get_hostname());
3686         assert (m_client->m_waiting != 2);
3687         delete m_client;
3688         m_client = 0;
3689     }
3690     else if (!m_parent)
3691     {
3692         yaz_log(YLOG_LOG, "%sshutdown (client to proxy) bad state",
3693                  m_session_str);
3694         assert (m_parent);
3695     }
3696     else
3697     {
3698         yaz_log(YLOG_LOG, "%sShutdown (client to proxy)",
3699                  m_session_str);
3700     }
3701     if (m_parent)
3702         m_parent->pre_init();
3703 }
3704
3705 bool Yaz_Proxy::dec_ref()
3706 {
3707     m_http_keepalive = 0;
3708
3709     --m_ref_count;
3710     if (m_ref_count > 0)
3711         return false;
3712
3713     releaseClient();
3714
3715     delete this;
3716     return true;
3717 }
3718
3719 const char *Yaz_ProxyClient::get_session_str()
3720 {
3721     if (!m_server)
3722         return "0 ";
3723     return m_server->get_session_str();
3724 }
3725
3726 void Yaz_ProxyClient::shutdown()
3727 {
3728     yaz_log(YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3729              get_hostname());
3730
3731     if (m_server)
3732     {
3733         m_waiting = 1;   // ensure it's released from Yaz_Proxy::releaseClient
3734         m_server->dec_ref();
3735     }
3736     else
3737         delete this;
3738 }
3739
3740 void Yaz_Proxy::failNotify()
3741 {
3742     inc_request_no();
3743     yaz_log(YLOG_LOG, "%sConnection closed by client", get_session_str());
3744     dec_ref();
3745 }
3746
3747 void Yaz_Proxy::send_response_fail_client(const char *addr)
3748 {
3749     if (m_http_version)
3750     {
3751         Z_SRW_diagnostic *diagnostic = 0;
3752         int num_diagnostic = 0;
3753
3754         yaz_add_srw_diagnostic(odr_encode(),
3755                                &diagnostic, &num_diagnostic,
3756                                YAZ_SRW_SYSTEM_TEMPORARILY_UNAVAILABLE, addr);
3757         if (m_s2z_search_apdu)
3758             send_srw_search_response(diagnostic, num_diagnostic);
3759         else if (m_s2z_scan_apdu)
3760             send_srw_scan_response(diagnostic, num_diagnostic);
3761         else
3762             send_srw_explain_response(diagnostic, num_diagnostic);
3763     }
3764 }
3765
3766 void Yaz_ProxyClient::failNotify()
3767 {
3768     if (m_server)
3769         m_server->inc_request_no();
3770     yaz_log(YLOG_LOG, "%sConnection closed by target %s",
3771              get_session_str(), get_hostname());
3772
3773     if (m_server)
3774         m_server->send_response_fail_client(get_hostname());
3775     shutdown();
3776 }
3777
3778 void Yaz_ProxyClient::connectNotify()
3779 {
3780     const char *s = get_session_str();
3781     const char *h = get_hostname();
3782     yaz_log(YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3783              m_target_idletime);
3784     timeout(m_target_idletime);
3785     if (!m_server)
3786         pre_init_client();
3787 }
3788
3789 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3790                                               *the_PDU_Observable, int fd)
3791 {
3792     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3793 }
3794
3795 Yaz_ProxyClient::~Yaz_ProxyClient()
3796 {
3797     if (m_prev)
3798         *m_prev = m_next;
3799     if (m_next)
3800         m_next->m_prev = m_prev;
3801     m_waiting = 2;     // for debugging purposes only.
3802     odr_destroy(m_init_odr);
3803     odr_destroy(m_idAuthentication_odr);
3804     delete m_last_query;
3805     xfree(m_last_resultSetId);
3806     xfree(m_cookie);
3807 }
3808
3809 void Yaz_ProxyClient::pre_init_client()
3810 {
3811     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3812     Z_InitRequest *req = apdu->u.initRequest;
3813
3814     int i;
3815     for (i = 0; i<= 24; i++)
3816         ODR_MASK_SET(req->options, i);
3817     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3818                    Z_Options_negotiationModel);
3819     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3820                    Z_Options_concurrentOperations);
3821     for (i = 0; i<= 10; i++)
3822         ODR_MASK_SET(req->protocolVersion, i);
3823
3824     if (send_to_target(apdu) < 0)
3825     {
3826         delete this;
3827     }
3828     else
3829     {
3830         m_waiting = 1;
3831         m_init_flag = 1;
3832     }
3833 }
3834
3835 void Yaz_Proxy::pre_init()
3836 {
3837     int i;
3838     const char *name = 0;
3839     const char *zurl_in_use[MAX_ZURL_PLEX];
3840     int limit_bw, limit_pdu, limit_req, limit_search;
3841     int target_idletime, client_idletime;
3842     int max_sockets = m_max_sockets;
3843     int max_clients;
3844     int keepalive_limit_bw, keepalive_limit_pdu;
3845     int pre_init;
3846     const char *cql2rpn = 0;
3847     const char *authentication = 0;
3848     const char *negotiation_charset = 0;
3849     const char *negotiation_lang = 0;
3850
3851     Yaz_ProxyConfig *cfg = check_reconfigure();
3852
3853     zurl_in_use[0] = 0;
3854
3855     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3856         set_APDU_yazlog(1);
3857     else
3858         set_APDU_yazlog(0);
3859
3860     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3861                                           &limit_bw, &limit_pdu, &limit_req,
3862                                           &limit_search,
3863                                           &target_idletime, &client_idletime,
3864                                           &max_sockets,
3865                                           &max_clients,
3866                                           &keepalive_limit_bw,
3867                                           &keepalive_limit_pdu,
3868                                           &pre_init,
3869                                           &cql2rpn,
3870                                           &authentication,
3871                                           &negotiation_charset,
3872                                           &negotiation_lang,
3873                                           0,
3874                                           0) ; i++)
3875     {
3876         if (pre_init)
3877         {
3878             int j;
3879             for (j = 0; zurl_in_use[j]; j++)
3880             {
3881                 Yaz_ProxyClient *c;
3882                 int spare = 0;
3883                 int spare_waiting = 0;
3884                 int in_use = 0;
3885                 int other = 0;
3886                 for (c = m_clientPool; c; c = c->m_next)
3887                 {
3888                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3889                     {
3890                         if (c->m_cookie == 0)
3891                         {
3892                             if (c->m_server == 0)
3893                                 if (c->m_waiting)
3894                                     spare_waiting++;
3895                                 else
3896                                     spare++;
3897                             else
3898                                 in_use++;
3899                         }
3900                         else
3901                             other++;
3902                     }
3903                 }
3904                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3905                         "sparew=%d preinit=%d",m_session_str,
3906                         name, zurl_in_use[j], in_use, other,
3907                         spare, spare_waiting, pre_init);
3908                 if (spare + spare_waiting < pre_init
3909                     && in_use + spare + spare_waiting + other < max_sockets)
3910                 {
3911                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3912                     c->m_next = m_clientPool;
3913                     if (c->m_next)
3914                         c->m_next->m_prev = &c->m_next;
3915                     m_clientPool = c;
3916                     c->m_prev = &m_clientPool;
3917
3918                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3919                         c->set_APDU_yazlog(1);
3920                     else
3921                         c->set_APDU_yazlog(0);
3922
3923                     if (c->client(zurl_in_use[j]))
3924                     {
3925                         timeout(60);
3926                         delete c;
3927                         return;
3928                     }
3929                     c->timeout(30);
3930                     c->m_waiting = 1;
3931                     c->m_target_idletime = target_idletime;
3932                     c->m_seqno = m_seqno++;
3933                 }
3934             }
3935         }
3936     }
3937 }
3938
3939 void Yaz_Proxy::timeoutNotify()
3940 {
3941     if (m_parent)
3942     {
3943         GDU *gdu;
3944         switch(m_timeout_mode)
3945         {
3946         case timeout_normal:
3947         case timeout_busy:
3948             inc_request_no();
3949             m_in_queue.clear();
3950             yaz_log(YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3951             dec_ref();
3952             break;
3953         case timeout_reduce:
3954             timeout(m_client_idletime);
3955             m_timeout_mode = timeout_busy;
3956             gdu = m_timeout_gdu;
3957             m_timeout_gdu = 0;
3958             recv_GDU_normal(gdu);
3959             break;
3960         case timeout_xsl:
3961             assert(m_stylesheet_nprl);
3962             convert_xsl_delay();
3963             recv_GDU_more(true);
3964         }
3965     }
3966     else
3967     {
3968         timeout(600);
3969         pre_init();
3970     }
3971 }
3972
3973 void Yaz_Proxy::markInvalid()
3974 {
3975     m_client = 0;
3976     m_flag_invalid_session = 1;
3977 }
3978
3979 void Yaz_ProxyClient::timeoutNotify()
3980 {
3981     if (m_server)
3982         m_server->inc_request_no();
3983
3984     yaz_log(YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3985              get_hostname());
3986
3987     if (m_server)
3988         m_server->send_response_fail_client(get_hostname());
3989
3990     Yaz_Proxy *proxy_root = m_root;
3991
3992     shutdown();
3993
3994     proxy_root->pre_init();
3995 }
3996
3997 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
3998                                  Yaz_Proxy *parent) :
3999     Z_Assoc (the_PDU_Observable)
4000 {
4001     m_cookie = 0;
4002     m_next = 0;
4003     m_prev = 0;
4004     m_init_flag = 0;
4005     m_last_query = 0;
4006     m_last_resultSetId = 0;
4007     m_last_resultCount = 0;
4008     m_last_ok = 0;
4009     m_sr_transform = 0;
4010     m_waiting = 0;
4011     m_init_odr = odr_createmem (ODR_DECODE);
4012     m_initResponse = 0;
4013     m_initResponse_options = 0;
4014     m_initResponse_version = 0;
4015     m_initResponse_preferredMessageSize = 0;
4016     m_initResponse_maximumRecordSize = 0;
4017     m_resultSetStartPoint = 0;
4018     m_bytes_sent = m_bytes_recv = 0;
4019     m_pdu_recv = 0;
4020     m_server = 0;
4021     m_seqno = 0;
4022     m_target_idletime = 600;
4023     m_root = parent;
4024     m_idAuthentication_odr = odr_createmem(ODR_ENCODE);
4025     m_idAuthentication_ber_buf = 0;
4026     m_idAuthentication_ber_size = 0;
4027 }
4028
4029 const char *Yaz_Proxy::option(const char *name, const char *value)
4030 {
4031     if (!strcmp (name, "optimize")) {
4032         if (value) {
4033             xfree(m_optimize);
4034             m_optimize = xstrdup(value);
4035         }
4036         return m_optimize;
4037     }
4038     return 0;
4039 }
4040
4041 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
4042 {
4043
4044 }
4045
4046 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
4047 {
4048     if (apdu->which == Z_GDU_Z3950)
4049         recv_Z_PDU(apdu->u.z3950, len);
4050     else if (apdu->which == Z_GDU_HTTP_Response)
4051         recv_HTTP_response(apdu->u.HTTP_Response, len);
4052     else
4053         shutdown();
4054 }
4055
4056 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
4057 {
4058     if (!m_flag_invalid_session)
4059         return 0;
4060     m_flag_invalid_session = 0;
4061     handle_incoming_Z_PDU(m_apdu_invalid_session);
4062     assert (m_mem_invalid_session);
4063     nmem_destroy(m_mem_invalid_session);
4064     m_mem_invalid_session = 0;
4065     return 1;
4066 }
4067
4068 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
4069 {
4070     m_bytes_recv += len;
4071
4072     m_pdu_recv++;
4073     m_waiting = 0;
4074     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
4075         yaz_log(YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
4076                  apdu_name(apdu), get_hostname(), len);
4077     if (apdu->which == Z_APDU_initResponse)
4078     {
4079         if (!m_server)  // if this is a pre init session , check for more
4080             m_root->pre_init();
4081         NMEM nmem = odr_extract_mem (odr_decode());
4082         odr_reset (m_init_odr);
4083         nmem_transfer (m_init_odr->mem, nmem);
4084         m_initResponse = apdu;
4085         m_initResponse_options = apdu->u.initResponse->options;
4086         m_initResponse_version = apdu->u.initResponse->protocolVersion;
4087         m_initResponse_preferredMessageSize =
4088             *apdu->u.initResponse->preferredMessageSize;
4089         m_initResponse_maximumRecordSize =
4090             *apdu->u.initResponse->maximumRecordSize;
4091
4092         Z_InitResponse *ir = apdu->u.initResponse;
4093
4094         // apply YAZ Proxy version
4095         char *imv0 = ir->implementationVersion;
4096         char *imv1 = (char*)
4097             odr_malloc(m_init_odr, 20 + (imv0 ? strlen(imv0) : 0));
4098         *imv1 = '\0';
4099         if (imv0)
4100             strcat(imv1, imv0);
4101 #ifdef VERSION
4102         strcat(imv1, "/" VERSION);
4103 #endif
4104         ir->implementationVersion = imv1;
4105
4106         // apply YAZ Proxy implementation name
4107         char *im0 = ir->implementationName;
4108         char *im1 = (char*)
4109             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
4110         *im1 = '\0';
4111         if (im0)
4112         {
4113             strcat(im1, im0);
4114             strcat(im1, " ");
4115         }
4116         strcat(im1, "(YAZ Proxy)");
4117         ir->implementationName = im1;
4118
4119         nmem_destroy (nmem);
4120
4121         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
4122             return;
4123     }
4124     if (apdu->which == Z_APDU_searchResponse)
4125     {
4126         Z_SearchResponse *sr = apdu->u.searchResponse;
4127         m_last_resultCount = *sr->resultCount;
4128         int status = *sr->searchStatus;
4129         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
4130         {
4131             m_last_ok = 1;
4132
4133             if (sr->records && sr->records->which == Z_Records_DBOSD)
4134             {
4135                 m_cache.add(odr_decode(),
4136                             sr->records->u.databaseOrSurDiagnostics, 1,
4137                             *sr->resultCount);
4138             }
4139         }
4140     }
4141     if (apdu->which == Z_APDU_presentResponse)
4142     {
4143         Z_PresentResponse *pr = apdu->u.presentResponse;
4144         if (m_sr_transform)
4145         {
4146             m_sr_transform = 0;
4147             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
4148             Z_SearchResponse *sr = new_apdu->u.searchResponse;
4149             sr->referenceId = pr->referenceId;
4150             *sr->resultCount = m_last_resultCount;
4151             sr->records = pr->records;
4152             sr->nextResultSetPosition = pr->nextResultSetPosition;
4153             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
4154             apdu = new_apdu;
4155         }
4156         if (pr->records &&
4157             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
4158         {
4159             m_cache.add(odr_decode(),
4160                         pr->records->u.databaseOrSurDiagnostics,
4161                         m_resultSetStartPoint, -1);
4162             m_resultSetStartPoint = 0;
4163         }
4164     }
4165     if (m_cookie)
4166         set_otherInformationString(apdu, yaz_oid_userinfo_cookie, 1, m_cookie);
4167
4168     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
4169
4170     if (server)
4171         server->send_to_client(apdu);
4172     if (apdu->which == Z_APDU_close)
4173         shutdown();
4174     else if (server)
4175         server->recv_GDU_more(true);
4176 }
4177
4178 void Yaz_Proxy::low_socket_close()
4179 {
4180 #if WIN32
4181 #else
4182     int i;
4183     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
4184         if  (m_lo_fd[i] >= 0)
4185             ::close(m_lo_fd[i]);
4186 #endif
4187 }
4188
4189 void Yaz_Proxy::low_socket_open()
4190 {
4191 #if WIN32
4192 #else
4193     int i;
4194     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
4195         m_lo_fd[i] = open("/dev/null", O_RDONLY);
4196 #endif
4197 }
4198
4199 int Yaz_Proxy::server(const char *addr)
4200 {
4201     int r = Z_Assoc::server(addr);
4202     if (!r)
4203     {
4204         yaz_log(YLOG_LOG, "%sStarted proxy "
4205 #ifdef VERSION
4206             VERSION
4207 #endif
4208             " on %s", m_session_str, addr);
4209         timeout(1);
4210     }
4211     return r;
4212 }
4213
4214 void Yaz_Proxy::base64_decode(const char *base64, char *buf, int buf_len)
4215 {
4216     const char *base64_chars =
4217         "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
4218     int len = strlen(base64);
4219     int buf_pos = 0;
4220     int index = 1;
4221
4222     for (int pos = 0; pos <= len; pos++)
4223     {
4224         if (base64[pos] == '=' || buf_pos + 1 >= buf_len)
4225             break;
4226
4227         const char *ch_ptr = strchr(base64_chars, base64[pos]);
4228         if (!ch_ptr)
4229             break;
4230         char ch = (char) (ch_ptr - base64_chars);
4231         switch (index)
4232         {
4233         case 1:
4234             buf[buf_pos] = ch << 2;
4235             break;
4236         case 2:
4237             buf[buf_pos++] += (ch & 0x30) >> 4;
4238             buf[buf_pos] = (ch & 0x0f) << 4;
4239             break;
4240         case 3:
4241             buf[buf_pos++] += (ch & 0x3c) >> 2;
4242             buf[buf_pos] = (ch & 0x03) << 6;
4243             break;
4244         case 4:
4245             buf[buf_pos++] += ch;
4246         }
4247         if (index < 4)
4248             index++;
4249         else
4250             index = 1;
4251     }
4252     buf[buf_pos] = '\0';
4253 }
4254
4255 /*
4256  * Local variables:
4257  * c-basic-offset: 4
4258  * c-file-style: "Stroustrup"
4259  * indent-tabs-mode: nil
4260  * End:
4261  * vim: shiftwidth=4 tabstop=8 expandtab
4262  */
4263