src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.30 2005-06-08 13:29:03 adam Exp $
   2    Copyright (c) 1998-2005, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <stdlib.h>
  42 #include <time.h>
  43 #include <fcntl.h>
  44
  45 #include <yaz/srw.h>
  46 #include <yaz/marcdisp.h>
  47 #include <yaz/yaz-iconv.h>
  48 #include <yaz/log.h>
  49 #include <yaz/diagbib1.h>
  50 #include "proxyp.h"
  51 #include <yaz/pquery.h>
  52 #include <yaz/otherinfo.h>
  53 #include <yaz/charneg.h>
  54 #include "msg-thread.h"
  55
  56 using namespace yazpp_1;
  57
  58 class Auth_Msg : public IMsg_Thread {
  59 public:
  60     int m_ret;
  61     void destroy();
  62     IMsg_Thread *handle();
  63     void result();
  64     Yaz_Proxy *m_proxy;
  65     Z_APDU *m_apdu;
  66 };
  67
  68 IMsg_Thread *Auth_Msg::handle()
  69 {
  70     yaz_log(YLOG_LOG, "Auth_Msg:handle begin");
  71     m_ret = m_proxy->handle_authentication(m_apdu);
  72     yaz_log(YLOG_LOG, "Auth_Msg:handle end");
  73     return this;
  74 }
  75
  76 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
  77 {
  78     if (ret == 0)
  79     {
  80         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
  81         *apdu_reject->u.initResponse->result = 0;
  82         send_to_client(apdu_reject);
  83         shutdown();
  84     }
  85     else
  86         handle_incoming_Z_PDU_2(apdu);
  87 }
  88
  89 void Auth_Msg::result()
  90 {
  91     yaz_log(YLOG_LOG, "Auth_Msg:result");
  92     m_proxy->result_authentication(m_apdu, m_ret);
  93     delete this;
  94 }
  95
  96 static const char *apdu_name(Z_APDU *apdu)
  97 {
  98     switch (apdu->which)
  99     {
 100     case Z_APDU_initRequest:
 101         return "initRequest";
 102     case Z_APDU_initResponse:
 103         return "initResponse";
 104     case Z_APDU_searchRequest:
 105         return "searchRequest";
 106     case Z_APDU_searchResponse:
 107         return "searchResponse";
 108     case Z_APDU_presentRequest:
 109         return "presentRequest";
 110     case Z_APDU_presentResponse:
 111         return "presentResponse";
 112     case Z_APDU_deleteResultSetRequest:
 113         return "deleteResultSetRequest";
 114     case Z_APDU_deleteResultSetResponse:
 115         return "deleteResultSetResponse";
 116     case Z_APDU_scanRequest:
 117         return "scanRequest";
 118     case Z_APDU_scanResponse:
 119         return "scanResponse";
 120     case Z_APDU_sortRequest:
 121         return "sortRequest";
 122     case Z_APDU_sortResponse:
 123         return "sortResponse";
 124     case Z_APDU_extendedServicesRequest:
 125         return "extendedServicesRequest";
 126     case Z_APDU_extendedServicesResponse:
 127         return "extendedServicesResponse";
 128     case Z_APDU_close:
 129         return "close";
 130     }
 131     return "other";
 132 }
 133
 134 static const char *gdu_name(Z_GDU *gdu)
 135 {
 136     switch(gdu->which)
 137     {
 138     case Z_GDU_Z3950:
 139         return apdu_name(gdu->u.z3950);
 140     case Z_GDU_HTTP_Request:
 141         return "HTTP Request";
 142     case Z_GDU_HTTP_Response:
 143         return "HTTP Response";
 144     }
 145     return "Unknown request/response";
 146 }
 147
 148 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
 149                      ISocketObservable *the_socket_observable,
 150                      Yaz_Proxy *parent)
 151     :
 152     Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 153 {
 154     m_PDU_Observable = the_PDU_Observable;
 155     m_socket_observable = the_socket_observable;
 156     m_client = 0;
 157     m_parent = parent;
 158     m_clientPool = 0;
 159     m_seqno = 1;
 160     m_keepalive_limit_bw = 500000;
 161     m_keepalive_limit_pdu = 1000;
 162     m_proxyTarget = 0;
 163     m_default_target = 0;
 164     m_proxy_negotiation_charset = 0;
 165     m_proxy_negotiation_lang = 0;
 166     m_charset_converter = new Yaz_CharsetConverter;
 167     m_max_clients = 150;
 168     m_log_mask = 0;
 169     m_seed = time(0);
 170     m_client_idletime = 600;
 171     m_target_idletime = 600;
 172     m_optimize = xstrdup ("1");
 173     strcpy(m_session_str, "0 ");
 174     m_session_no = 0;
 175     m_bytes_sent = 0;
 176     m_bytes_recv = 0;
 177     m_bw_hold_PDU = 0;
 178     m_bw_max = 0;
 179     m_pdu_max = 0;
 180     m_max_record_retrieve = 0;
 181     m_reconfig_flag = 0;
 182     m_config_fname = 0;
 183     m_request_no = 0;
 184     m_invalid_session = 0;
 185     m_referenceId = 0;
 186     m_referenceId_mem = nmem_create();
 187     m_config = 0;
 188     m_marcxml_mode = none;
 189     m_stylesheet_xsp = 0;
 190     m_stylesheet_nprl = 0;
 191     m_s2z_stylesheet = 0;
 192     m_s2z_database = 0;
 193     m_schema = 0;
 194     m_backend_type = 0;
 195     m_backend_charset = 0;
 196     m_frontend_type = 0;
 197     m_initRequest_apdu = 0;
 198     m_initRequest_mem = 0;
 199     m_initRequest_preferredMessageSize = 0;
 200     m_initRequest_maximumRecordSize = 0;
 201     m_initRequest_options = 0;
 202     m_initRequest_version = 0;
 203     m_initRequest_oi_negotiation_charsets = 0;
 204     m_initRequest_oi_negotiation_num_charsets = 0;
 205     m_initRequest_oi_negotiation_langs = 0;
 206     m_initRequest_oi_negotiation_num_langs = 0;
 207     m_initRequest_oi_negotiation_selected = 0;
 208     m_apdu_invalid_session = 0;
 209     m_mem_invalid_session = 0;
 210     m_s2z_odr_init = 0;
 211     m_s2z_odr_search = 0;
 212     m_s2z_init_apdu = 0;
 213     m_s2z_search_apdu = 0;
 214     m_s2z_present_apdu = 0;
 215     m_http_keepalive = 0;
 216     m_http_version = 0;
 217     m_soap_ns = 0;
 218     m_s2z_packing = Z_SRW_recordPacking_string;
 219 #if HAVE_GETTIMEOFDAY
 220     m_time_tv = xmalloc(sizeof(struct timeval));
 221     struct timeval *tv = (struct timeval *) m_time_tv;
 222     tv->tv_sec = 0;
 223     tv->tv_usec = 0;
 224 #else
 225     m_time_tv = 0;
 226 #endif
 227     m_usemarcon_ini_stage1 = 0;
 228     m_usemarcon_ini_stage2 = 0;
 229     m_usemarcon = new Yaz_usemarcon();
 230     if (!m_parent)
 231         low_socket_open();
 232     m_my_thread = 0;
 233 }
 234
 235 Yaz_Proxy::~Yaz_Proxy()
 236 {
 237     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 238             m_bytes_sent, m_bytes_recv);
 239     nmem_destroy(m_initRequest_mem);
 240     nmem_destroy(m_mem_invalid_session);
 241     nmem_destroy(m_referenceId_mem);
 242
 243     xfree(m_proxyTarget);
 244     xfree(m_default_target);
 245     xfree(m_proxy_negotiation_charset);
 246     xfree(m_proxy_negotiation_lang);
 247     delete m_charset_converter;
 248     xfree(m_optimize);
 249
 250 #if HAVE_XSLT
 251     if (m_stylesheet_xsp)
 252         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 253 #endif
 254     xfree (m_time_tv);
 255
 256     xfree (m_schema);
 257     xfree (m_backend_type);
 258     xfree (m_backend_charset);
 259     xfree (m_usemarcon_ini_stage1);
 260     xfree (m_usemarcon_ini_stage2);
 261     delete m_usemarcon;
 262     if (m_s2z_odr_init)
 263         odr_destroy(m_s2z_odr_init);
 264     if (m_s2z_odr_search)
 265         odr_destroy(m_s2z_odr_search);
 266     if (!m_parent)
 267         low_socket_close();
 268     if (!m_parent)
 269         delete m_my_thread;
 270     delete m_config;
 271 }
 272
 273 void Yaz_Proxy::set_debug_mode(int mode)
 274 {
 275     m_debug_mode = mode;
 276 }
 277
 278 int Yaz_Proxy::set_config(const char *config)
 279 {
 280     delete m_config;
 281     m_config = new Yaz_ProxyConfig();
 282     xfree(m_config_fname);
 283     m_config_fname = xstrdup(config);
 284     int r = m_config->read_xml(config);
 285     if (!r)
 286         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 287     return r;
 288 }
 289
 290 void Yaz_Proxy::set_default_target(const char *target)
 291 {
 292     xfree (m_default_target);
 293     m_default_target = 0;
 294     if (target)
 295         m_default_target = (char *) xstrdup (target);
 296 }
 297
 298 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang)
 299 {
 300     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
 301         "language to '%s'", m_session_str, charset?charset:"none",
 302         lang?lang:"none");
 303     xfree (m_proxy_negotiation_charset);
 304     xfree (m_proxy_negotiation_lang);
 305     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 306     if (charset)
 307         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 308     if (lang)
 309         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 310 }
 311
 312 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 313 {
 314     if (m_parent)
 315         return m_parent->check_reconfigure();
 316
 317     Yaz_ProxyConfig *cfg = m_config;
 318     if (m_reconfig_flag)
 319     {
 320         yaz_log(YLOG_LOG, "reconfigure");
 321         yaz_log_reopen();
 322         if (m_config_fname && cfg)
 323         {
 324             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 325             int r = cfg->read_xml(m_config_fname);
 326             if (r)
 327                 yaz_log(YLOG_WARN, "reconfigure failed");
 328             else
 329             {
 330                 m_log_mask = 0;
 331                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 332             }
 333         }
 334         else
 335             yaz_log(YLOG_LOG, "reconfigure");
 336         m_reconfig_flag = 0;
 337     }
 338     return cfg;
 339 }
 340
 341 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 342                                         *the_PDU_Observable, int fd)
 343 {
 344     check_reconfigure();
 345     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 346                                          m_socket_observable, this);
 347     new_proxy->m_config = 0;
 348     new_proxy->m_config_fname = 0;
 349     new_proxy->timeout(m_client_idletime);
 350     new_proxy->m_target_idletime = m_target_idletime;
 351     new_proxy->set_default_target(m_default_target);
 352     new_proxy->m_max_clients = m_max_clients;
 353     new_proxy->m_log_mask = m_log_mask;
 354     new_proxy->set_APDU_log(get_APDU_log());
 355     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 356         new_proxy->set_APDU_yazlog(1);
 357     else
 358         new_proxy->set_APDU_yazlog(0);
 359     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 360     m_session_no++;
 361     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 362              the_PDU_Observable->getpeername());
 363     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 364         m_proxy_negotiation_lang);
 365     // create thread object the first time we get an incoming connection
 366     if (!m_my_thread)
 367         m_my_thread = new Msg_Thread(m_socket_observable);
 368     new_proxy->m_my_thread = m_my_thread;
 369     return new_proxy;
 370 }
 371
 372 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 373 {
 374     int oid[OID_SIZE];
 375     Z_OtherInformationUnit *oi;
 376     struct oident ent;
 377     ent.proto = PROTO_Z3950;
 378     ent.oclass = CLASS_USERINFO;
 379     ent.value = (oid_value) VAL_COOKIE;
 380     assert (oid_ent_to_oid (&ent, oid));
 381
 382     if (oid_ent_to_oid (&ent, oid) &&
 383         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 384         oi->which == Z_OtherInfo_characterInfo)
 385         return oi->information.characterInfo;
 386     return 0;
 387 }
 388 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 389 {
 390     int oid[OID_SIZE];
 391     Z_OtherInformationUnit *oi;
 392     struct oident ent;
 393     ent.proto = PROTO_Z3950;
 394     ent.oclass = CLASS_USERINFO;
 395     ent.value = (oid_value) VAL_PROXY;
 396     if (oid_ent_to_oid (&ent, oid) &&
 397         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 398         oi->which == Z_OtherInfo_characterInfo)
 399         return oi->information.characterInfo;
 400     return 0;
 401 }
 402 const char *Yaz_Proxy::load_balance(const char **url)
 403 {
 404     int zurl_in_use[MAX_ZURL_PLEX];
 405     int zurl_in_spare[MAX_ZURL_PLEX];
 406     Yaz_ProxyClient *c;
 407     int i;
 408
 409     for (i = 0; i<MAX_ZURL_PLEX; i++)
 410     {
 411         zurl_in_use[i] = 0;
 412         zurl_in_spare[i] = 0;
 413     }
 414     for (c = m_parent->m_clientPool; c; c = c->m_next)
 415     {
 416         for (i = 0; url[i]; i++)
 417             if (!strcmp(url[i], c->get_hostname()))
 418             {
 419                 zurl_in_use[i]++;
 420                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 421                     zurl_in_spare[i]++;
 422             }
 423     }
 424     int min_use = 100000;
 425     int spare_for_min = 0;
 426     int max_spare = 0;
 427     const char *ret_min = 0;
 428     const char *ret_spare = 0;
 429     for (i = 0; url[i]; i++)
 430     {
 431         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 432                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 433         if (min_use > zurl_in_use[i])
 434         {
 435             ret_min = url[i];
 436             min_use = zurl_in_use[i];
 437             spare_for_min = zurl_in_spare[i];
 438         }
 439         if (max_spare < zurl_in_spare[i])
 440         {
 441             ret_spare = url[i];
 442             max_spare = zurl_in_spare[i];
 443         }
 444     }
 445     return ret_min;
 446 }
 447
 448 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 449                                        const char *proxy_host)
 450 {
 451     assert (m_parent);
 452     Yaz_Proxy *parent = m_parent;
 453     Yaz_ProxyClient *c = m_client;
 454
 455     if (!m_proxyTarget)
 456     {
 457         const char *url[MAX_ZURL_PLEX];
 458         Yaz_ProxyConfig *cfg = check_reconfigure();
 459         if (proxy_host)
 460         {
 461             if (parent && parent->m_debug_mode)
 462             {
 463                 // only to be enabled for debugging...
 464                 if (!strcmp(proxy_host, "stop"))
 465                     exit(0);
 466             }
 467             xfree(m_default_target);
 468             m_default_target = xstrdup(proxy_host);
 469         }
 470         proxy_host = m_default_target;
 471         int client_idletime = -1;
 472         const char *cql2rpn_fname = 0;
 473         const char *negotiation_charset = 0;
 474         const char *negotiation_lang = 0;
 475         const char *query_charset = 0;
 476         url[0] = m_default_target;
 477         url[1] = 0;
 478         if (cfg)
 479         {
 480             int pre_init = 0;
 481             cfg->get_target_info(proxy_host, url, &m_bw_max,
 482                                  &m_pdu_max, &m_max_record_retrieve,
 483                                  &m_target_idletime, &client_idletime,
 484                                  &parent->m_max_clients,
 485                                  &m_keepalive_limit_bw,
 486                                  &m_keepalive_limit_pdu,
 487                                  &pre_init,
 488                                  &cql2rpn_fname,
 489                                  &negotiation_charset,
 490                                  &negotiation_lang,
 491                                  &query_charset);
 492         }
 493         if (client_idletime != -1)
 494         {
 495             m_client_idletime = client_idletime;
 496             timeout(m_client_idletime);
 497         }
 498         if (cql2rpn_fname)
 499             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 500         if (negotiation_charset || negotiation_lang)
 501         {
 502             set_proxy_negotiation(negotiation_charset,
 503                 negotiation_lang);
 504         }
 505         m_charset_converter->set_target_query_charset(query_charset);
 506         if (!url[0])
 507         {
 508             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 509             return 0;
 510         }
 511         // we don't handle multiplexing for cookie session, so we just
 512         // pick the first one in this case (anonymous users will be able
 513         // to use any backend)
 514         if (cookie && *cookie)
 515             m_proxyTarget = (char*) xstrdup(url[0]);
 516         else
 517             m_proxyTarget = (char*) xstrdup(load_balance(url));
 518     }
 519     if (cookie && *cookie)
 520     {   // search in sessions with a cookie
 521         for (c = parent->m_clientPool; c; c = c->m_next)
 522         {
 523             assert (c->m_prev);
 524             assert (*c->m_prev == c);
 525             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 526                 !strcmp(m_proxyTarget, c->get_hostname()))
 527             {
 528                 // Found it in cache
 529                 // The following handles "cancel"
 530                 // If connection is busy (waiting for PDU) and
 531                 // we have an initRequest we can safely do re-open
 532                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 533                 {
 534                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 535                              c->get_hostname());
 536                     c->close();
 537                     c->m_init_flag = 0;
 538
 539                     c->m_last_ok = 0;
 540                     c->m_cache.clear();
 541                     c->m_last_resultCount = 0;
 542                     c->m_sr_transform = 0;
 543                     c->m_waiting = 0;
 544                     c->m_resultSetStartPoint = 0;
 545                     c->m_target_idletime = m_target_idletime;
 546                     if (c->client(m_proxyTarget))
 547                     {
 548                         delete c;
 549                         return 0;
 550                     }
 551                     c->timeout(30);
 552                 }
 553                 c->m_seqno = parent->m_seqno;
 554                 if (c->m_server && c->m_server != this)
 555                     c->m_server->m_client = 0;
 556                 c->m_server = this;
 557                 (parent->m_seqno)++;
 558                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 559                 return c;
 560             }
 561         }
 562     }
 563     else if (!c &&
 564             apdu->which == Z_APDU_initRequest &&
 565             apdu->u.initRequest->idAuthentication == 0 &&
 566             !ODR_MASK_GET(apdu->u.initRequest->options, Z_Options_negotiationModel))
 567     {
 568         // anonymous sessions without cookie.
 569         // if authentication is set it is NOT anonymous se we can't share them.
 570         // If charset and lang negotiation is use it is NOT anonymous session too.
 571         for (c = parent->m_clientPool; c; c = c->m_next)
 572         {
 573             assert(c->m_prev);
 574             assert(*c->m_prev == c);
 575             if (c->m_server == 0 && c->m_cookie == 0 &&
 576                 c->m_waiting == 0 &&
 577                 !strcmp(m_proxyTarget, c->get_hostname()))
 578             {
 579                 // found it in cache
 580                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 581                          m_session_str, parent->m_seqno, c->get_hostname());
 582
 583                 c->m_seqno = parent->m_seqno;
 584                 assert(c->m_server == 0);
 585                 c->m_server = this;
 586
 587                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 588                     c->set_APDU_yazlog(1);
 589                 else
 590                     c->set_APDU_yazlog(0);
 591
 592                 (parent->m_seqno)++;
 593
 594                 parent->pre_init();
 595
 596                 return c;
 597             }
 598         }
 599     }
 600     if (!m_client)
 601     {
 602         if (apdu->which != Z_APDU_initRequest)
 603         {
 604             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 605             return 0;
 606         }
 607         Z_InitRequest *initRequest = apdu->u.initRequest;
 608
 609         if (initRequest->idAuthentication)
 610         {
 611             // the client uses authentication. We set the keepalive PDU
 612             // to 0 so we don't cache it in releaseClient
 613             m_keepalive_limit_pdu = 0;
 614         }
 615         // go through list of clients - and find the lowest/oldest one.
 616         Yaz_ProxyClient *c_min = 0;
 617         int min_seq = -1;
 618         int no_of_clients = 0;
 619         if (parent->m_clientPool)
 620             yaz_log (YLOG_DEBUG, "Existing sessions");
 621         for (c = parent->m_clientPool; c; c = c->m_next)
 622         {
 623             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 624                                c->m_waiting, c->get_hostname(),
 625                                c->m_cookie ? c->m_cookie : "");
 626             no_of_clients++;
 627             if (min_seq < 0 || c->m_seqno < min_seq)
 628             {
 629                 min_seq = c->m_seqno;
 630                 c_min = c;
 631             }
 632         }
 633         if (no_of_clients >= parent->m_max_clients)
 634         {
 635             c = c_min;
 636             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 637             {
 638                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 639                          m_session_str, parent->m_max_clients, c->m_seqno);
 640                 if (c->m_server && c->m_server != this)
 641                     delete c->m_server;
 642                 c->m_server = 0;
 643             }
 644             else
 645             {
 646                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 647                          m_session_str, parent->m_max_clients,
 648                          c->m_seqno, parent->m_seqno, c->get_hostname());
 649                 xfree (c->m_cookie);
 650                 c->m_cookie = 0;
 651                 if (cookie)
 652                     c->m_cookie = xstrdup(cookie);
 653                 c->m_seqno = parent->m_seqno;
 654                 if (c->m_server && c->m_server != this)
 655                 {
 656                     c->m_server->m_client = 0;
 657                     delete c->m_server;
 658                 }
 659                 (parent->m_seqno)++;
 660                 c->m_target_idletime = m_target_idletime;
 661                 c->timeout(m_target_idletime);
 662
 663                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 664                     c->set_APDU_yazlog(1);
 665                 else
 666                     c->set_APDU_yazlog(0);
 667
 668                 return c;
 669             }
 670         }
 671         else
 672         {
 673             yaz_log (YLOG_LOG, "%sNEW %d %s",
 674                      m_session_str, parent->m_seqno, m_proxyTarget);
 675             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 676             c->m_next = parent->m_clientPool;
 677             if (c->m_next)
 678                 c->m_next->m_prev = &c->m_next;
 679             parent->m_clientPool = c;
 680             c->m_prev = &parent->m_clientPool;
 681         }
 682
 683         xfree (c->m_cookie);
 684         c->m_cookie = 0;
 685         if (cookie)
 686             c->m_cookie = xstrdup(cookie);
 687
 688         c->m_seqno = parent->m_seqno;
 689         c->m_init_flag = 0;
 690         c->m_last_resultCount = 0;
 691         c->m_last_ok = 0;
 692         c->m_cache.clear();
 693         c->m_sr_transform = 0;
 694         c->m_waiting = 0;
 695         c->m_resultSetStartPoint = 0;
 696         (parent->m_seqno)++;
 697         if (c->client(m_proxyTarget))
 698         {
 699             delete c;
 700             return 0;
 701         }
 702         c->m_target_idletime = m_target_idletime;
 703         c->timeout(30);
 704
 705         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 706             c->set_APDU_yazlog(1);
 707         else
 708             c->set_APDU_yazlog(0);
 709     }
 710     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 711     return c;
 712 }
 713
 714 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 715 {
 716     int i;
 717     for (i = 0; i<num; i++)
 718     {
 719         oident *ent;
 720         Z_DefaultDiagFormat *r;
 721         Z_DiagRec *p = pp[i];
 722         if (p->which != Z_DiagRec_defaultFormat)
 723         {
 724             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 725             return;
 726         }
 727         else
 728             r = p->u.defaultFormat;
 729         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 730             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 731             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 732         switch (r->which)
 733         {
 734         case Z_DefaultDiagFormat_v2Addinfo:
 735             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 736                     m_session_str,
 737                     *r->condition, diagbib1_str(*r->condition),
 738                     r->u.v2Addinfo);
 739             break;
 740         case Z_DefaultDiagFormat_v3Addinfo:
 741             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 742                     m_session_str,
 743                     *r->condition, diagbib1_str(*r->condition),
 744                     r->u.v3Addinfo);
 745             break;
 746         }
 747     }
 748 }
 749
 750 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 751 {
 752     if (!m_stylesheet_xsp || p->num_records <= 0)
 753         return 0;  /* no XSLT to be done ... */
 754
 755     m_stylesheet_offset = 0;
 756     m_stylesheet_nprl = p;
 757     m_stylesheet_apdu = apdu;
 758     timeout(0);
 759     return 1;
 760 }
 761
 762 void Yaz_Proxy::convert_xsl_delay()
 763 {
 764 #if HAVE_XSLT
 765     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 766     if (npr->which == Z_NamePlusRecord_databaseRecord)
 767     {
 768         Z_External *r = npr->u.databaseRecord;
 769         if (r->which == Z_External_octet)
 770         {
 771 #if 0
 772             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 773 #endif
 774             xmlDocPtr res, doc = xmlParseMemory(
 775                 (char*) r->u.octet_aligned->buf,
 776                 r->u.octet_aligned->len);
 777
 778
 779             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 780                     m_session_str, m_stylesheet_offset);
 781             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 782                                       doc, 0);
 783
 784             if (res)
 785             {
 786                 xmlChar *out_buf;
 787                 int out_len;
 788                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 789
 790                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 791                     u.databaseRecord =
 792                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 793                                  (char*) out_buf, out_len);
 794                 xmlFree(out_buf);
 795                 xmlFreeDoc(res);
 796             }
 797
 798             xmlFreeDoc(doc);
 799         }
 800     }
 801 #endif
 802     m_stylesheet_offset++;
 803     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 804     {
 805         m_stylesheet_nprl = 0;
 806 #if HAVE_XSLT
 807         if (m_stylesheet_xsp)
 808             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 809 #endif
 810         m_stylesheet_xsp = 0;
 811         timeout(m_client_idletime);
 812         send_PDU_convert(m_stylesheet_apdu);
 813     }
 814     else
 815         timeout(0);
 816 }
 817
 818 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 819 {
 820     if (m_frontend_type != VAL_NONE)
 821     {
 822         int i;
 823         for (i = 0; i < p->num_records; i++)
 824         {
 825             Z_NamePlusRecord *npr = p->records[i];
 826             if (npr->which == Z_NamePlusRecord_databaseRecord)
 827             {
 828                 Z_External *r = npr->u.databaseRecord;
 829                 if (r->which == Z_External_octet)
 830                 {
 831 #if HAVE_USEMARCON
 832                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 833                     {
 834                         if (!m_usemarcon->m_stage1)
 835                         {
 836                             m_usemarcon->m_stage1 = new CDetails();
 837                         }
 838                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 839                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 840                         int res = m_usemarcon->m_stage1->Start();
 841                         if (res == 0)
 842                         {
 843                             char *converted;
 844                             int convlen;
 845                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 846                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 847                             {
 848                                 if (!m_usemarcon->m_stage2)
 849                                 {
 850                                     m_usemarcon->m_stage2 = new CDetails();
 851                                 }
 852                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 853                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 854                                 res = m_usemarcon->m_stage2->Start();
 855                                 if (res == 0)
 856                                 {
 857                                     free(converted);
 858                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 859                                 }
 860                                 else
 861                                 {
 862                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 863                                 }
 864                             }
 865                             npr->u.databaseRecord =
 866                                 z_ext_record(odr_encode(),
 867                                              m_frontend_type,
 868                                              converted,
 869                                              strlen(converted));
 870                             free(converted);
 871                         }
 872                         else
 873                         {
 874                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 875                         }
 876                         continue;
 877                     }
 878 #endif
 879 /* HAVE_USEMARCON */
 880                     npr->u.databaseRecord =
 881                         z_ext_record(odr_encode(),
 882                                      m_frontend_type,
 883                                      (char*) r->u.octet_aligned->buf,
 884                                      r->u.octet_aligned->len);
 885                 }
 886             }
 887         }
 888     }
 889 }
 890
 891 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
 892                                         const char *backend_charset)
 893 {
 894     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
 895     int sel =   m_charset_converter->get_client_charset_selected();
 896     const char *client_record_charset =
 897         m_charset_converter->get_client_query_charset();
 898     if (sel && backend_charset && client_record_charset &&
 899         strcmp(backend_charset, client_record_charset))
 900     {
 901         int i;
 902         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
 903                                         backend_charset);
 904         yaz_marc_t mt = yaz_marc_create();
 905         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
 906         yaz_marc_iconv(mt, cd);
 907         for (i = 0; i < p->num_records; i++)
 908         {
 909             Z_NamePlusRecord *npr = p->records[i];
 910             if (npr->which == Z_NamePlusRecord_databaseRecord)
 911             {
 912                 Z_External *r = npr->u.databaseRecord;
 913                 oident *ent = oid_getentbyoid(r->direct_reference);
 914                 if (!ent || ent->value == VAL_NONE)
 915                     continue;
 916
 917                 if (ent->value == VAL_SUTRS)
 918                 {
 919                     WRBUF w = wrbuf_alloc();
 920
 921                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
 922                                       r->u.octet_aligned->len);
 923                     npr->u.databaseRecord =
 924                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
 925                                      wrbuf_len(w));
 926                     wrbuf_free(w, 1);
 927                 }
 928                 else if (ent->value == VAL_TEXT_XML)
 929                 {
 930                     ;
 931                 }
 932                 else if (r->which == Z_External_octet)
 933                 {
 934                     int rlen;
 935                     char *result;
 936                     if (yaz_marc_decode_buf(mt,
 937                                             (char*) r->u.octet_aligned->buf,
 938                                             r->u.octet_aligned->len,
 939                                             &result, &rlen))
 940                     {
 941                         npr->u.databaseRecord =
 942                             z_ext_record(odr_encode(), ent->value, result, rlen);
 943                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
 944                                 m_session_str);
 945                     }
 946                 }
 947             }
 948         }
 949         if (cd)
 950             yaz_iconv_close(cd);
 951         yaz_marc_destroy(mt);
 952     }
 953     else
 954     {
 955         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
 956     }
 957 }
 958
 959 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
 960                                    const char *backend_charset)
 961 {
 962     int i;
 963     if (!backend_charset)
 964         backend_charset = "MARC-8";
 965     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
 966     yaz_marc_t mt = yaz_marc_create();
 967     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 968     yaz_marc_iconv(mt, cd);
 969     for (i = 0; i < p->num_records; i++)
 970     {
 971         Z_NamePlusRecord *npr = p->records[i];
 972         if (npr->which == Z_NamePlusRecord_databaseRecord)
 973         {
 974             Z_External *r = npr->u.databaseRecord;
 975             if (r->which == Z_External_OPAC)
 976             {
 977                 WRBUF w = wrbuf_alloc();
 978
 979                 yaz_display_OPAC(w, r->u.opac, 0);
 980                 npr->u.databaseRecord = z_ext_record(
 981                     odr_encode(), VAL_TEXT_XML,
 982                     wrbuf_buf(w), wrbuf_len(w)
 983                     );
 984                 wrbuf_free(w, 1);
 985             }
 986             else if (r->which == Z_External_octet)
 987             {
 988                 int rlen;
 989                 char *result;
 990                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 991                                         r->u.octet_aligned->len,
 992                                         &result, &rlen))
 993                 {
 994                     npr->u.databaseRecord =
 995                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
 996                 }
 997             }
 998         }
 999     }
1000     if (cd)
1001         yaz_iconv_close(cd);
1002     yaz_marc_destroy(mt);
1003 }
1004
1005 void Yaz_Proxy::logtime()
1006 {
1007 #if HAVE_GETTIMEOFDAY
1008     struct timeval *tv = (struct timeval*) m_time_tv;
1009     if (tv->tv_sec)
1010     {
1011         struct timeval tv1;
1012         gettimeofday(&tv1, 0);
1013         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1014             (tv1.tv_usec - tv->tv_usec);
1015         if (diff >= 0)
1016             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1017                     diff/1000000, (diff/1000)%1000);
1018     }
1019     tv->tv_sec = 0;
1020     tv->tv_usec = 0;
1021 #endif
1022 }
1023
1024 int Yaz_Proxy::send_http_response(int code)
1025 {
1026     ODR o = odr_encode();
1027     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1028     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1029     if (m_http_version)
1030         hres->version = odr_strdup(o, m_http_version);
1031     if (m_http_keepalive)
1032         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1033     else
1034         timeout(0);
1035
1036     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1037     {
1038         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1039                  gdu_name(gdu));
1040     }
1041     int len;
1042     int r = send_GDU(gdu, &len);
1043     m_bytes_sent += len;
1044     m_bw_stat.add_bytes(len);
1045     logtime();
1046     return r;
1047 }
1048
1049 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
1050 {
1051     ODR o = odr_encode();
1052     const char *ctype = "text/xml";
1053     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
1054     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1055     if (m_http_version)
1056         hres->version = odr_strdup(o, m_http_version);
1057     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1058     if (m_http_keepalive)
1059         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1060     else
1061         timeout(0);
1062
1063     static Z_SOAP_Handler soap_handlers[2] = {
1064 #if HAVE_XSLT
1065         {"http://www.loc.gov/zing/srw/", 0,
1066          (Z_SOAP_fun) yaz_srw_codec},
1067 #endif
1068         {0, 0, 0}
1069     };
1070
1071     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1072     soap_package->which = Z_SOAP_generic;
1073     soap_package->u.generic =
1074         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1075     soap_package->u.generic->no = 0;
1076     soap_package->u.generic->ns = soap_handlers[0].ns;
1077     soap_package->u.generic->p = (void *) srw_pdu;
1078     soap_package->ns = m_soap_ns;
1079     z_soap_codec_enc_xsl(o, &soap_package,
1080                          &hres->content_buf, &hres->content_len,
1081                          soap_handlers, 0, m_s2z_stylesheet);
1082     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1083     {
1084         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1085                  gdu_name(gdu));
1086     }
1087     int len;
1088     int r = send_GDU(gdu, &len);
1089     m_bytes_sent += len;
1090     m_bw_stat.add_bytes(len);
1091     logtime();
1092     return r;
1093 }
1094
1095 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1096 {
1097     ODR o = odr_encode();
1098     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1099     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1100
1101     srw_res->num_diagnostics = 1;
1102     srw_res->diagnostics = (Z_SRW_diagnostic *)
1103         odr_malloc(o, sizeof(*srw_res->diagnostics));
1104     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
1105     return send_srw_response(srw_pdu);
1106 }
1107
1108 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1109                              Z_DefaultDiagFormat *ddf)
1110 {
1111     int bib1_code = *ddf->condition;
1112     if (bib1_code == 109)
1113         return 404;
1114     srw_res->num_diagnostics = 1;
1115     srw_res->diagnostics = (Z_SRW_diagnostic *)
1116         odr_malloc(o, sizeof(*srw_res->diagnostics));
1117     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1118                           yaz_diag_bib1_to_srw(*ddf->condition),
1119                           ddf->u.v2Addinfo);
1120     return 0;
1121 }
1122
1123 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1124 {
1125     ODR o = odr_encode();
1126     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1127     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1128
1129     srw_res->numberOfRecords = odr_intdup (o, hits);
1130     if (records && records->which == Z_Records_DBOSD)
1131     {
1132         srw_res->num_records =
1133             records->u.databaseOrSurDiagnostics->num_records;
1134         int i;
1135         srw_res->records = (Z_SRW_record *)
1136             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1137         for (i = 0; i < srw_res->num_records; i++)
1138         {
1139             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1140             if (npr->which != Z_NamePlusRecord_databaseRecord)
1141             {
1142                 srw_res->records[i].recordSchema = "diagnostic";
1143                 srw_res->records[i].recordPacking = m_s2z_packing;
1144                 srw_res->records[i].recordData_buf = "67";
1145                 srw_res->records[i].recordData_len = 2;
1146                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1147                 continue;
1148             }
1149             Z_External *r = npr->u.databaseRecord;
1150             oident *ent = oid_getentbyoid(r->direct_reference);
1151             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1152             {
1153                 srw_res->records[i].recordSchema = m_schema;
1154                 srw_res->records[i].recordPacking = m_s2z_packing;
1155                 srw_res->records[i].recordData_buf = (char*)
1156                     r->u.octet_aligned->buf;
1157                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1158                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1159             }
1160             else
1161             {
1162                 srw_res->records[i].recordSchema = "diagnostic";
1163                 srw_res->records[i].recordPacking = m_s2z_packing;
1164                 srw_res->records[i].recordData_buf = "67";
1165                 srw_res->records[i].recordData_len = 2;
1166                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1167             }
1168         }
1169     }
1170     if (records && records->which == Z_Records_NSD)
1171     {
1172         int http_code;
1173         http_code = z_to_srw_diag(odr_encode(), srw_res,
1174                                    records->u.nonSurrogateDiagnostic);
1175         if (http_code)
1176             return send_http_response(http_code);
1177     }
1178     return send_srw_response(srw_pdu);
1179
1180 }
1181
1182 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1183                                         int num_diagnostics)
1184 {
1185     Yaz_ProxyConfig *cfg = check_reconfigure();
1186     if (cfg)
1187     {
1188         int len;
1189         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1190                                        m_s2z_database, &len);
1191         if (b)
1192         {
1193             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1194             Z_SRW_explainResponse *er = res->u.explain_response;
1195
1196             er->record.recordData_buf = b;
1197             er->record.recordData_len = len;
1198             er->record.recordPacking = m_s2z_packing;
1199             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1200
1201             er->diagnostics = diagnostics;
1202             er->num_diagnostics = num_diagnostics;
1203             return send_srw_response(res);
1204         }
1205     }
1206     return send_http_response(404);
1207 }
1208
1209 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1210 {
1211     if (m_http_version)
1212     {
1213         if (apdu->which == Z_APDU_initResponse)
1214         {
1215             Z_InitResponse *res = apdu->u.initResponse;
1216             if (*res->result == 0)
1217             {
1218                 send_to_srw_client_error(3, 0);
1219             }
1220             else if (!m_s2z_search_apdu)
1221             {
1222                 send_srw_explain_response(0, 0);
1223             }
1224             else
1225             {
1226                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1227             }
1228         }
1229         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1230         {
1231             m_s2z_search_apdu = 0;
1232             Z_SearchResponse *res = apdu->u.searchResponse;
1233             m_s2z_hit_count = *res->resultCount;
1234             if (res->records && res->records->which == Z_Records_NSD)
1235             {
1236                 send_to_srw_client_ok(0, res->records, 1);
1237             }
1238             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1239             {
1240                 // adjust
1241                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1242
1243                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1244                 {
1245                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1246                         > m_s2z_hit_count)
1247                         *pr->numberOfRecordsRequested =
1248                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1249                 }
1250                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1251             }
1252             else
1253             {
1254                 m_s2z_present_apdu = 0;
1255                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1256             }
1257         }
1258         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1259         {
1260             int start =
1261                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1262
1263             m_s2z_present_apdu = 0;
1264             Z_PresentResponse *res = apdu->u.presentResponse;
1265             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1266         }
1267     }
1268     else
1269     {
1270         int len = 0;
1271         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1272             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1273                      apdu_name(apdu));
1274         int r = send_Z_PDU(apdu, &len);
1275         m_bytes_sent += len;
1276         m_bw_stat.add_bytes(len);
1277         logtime();
1278         return r;
1279     }
1280     return 0;
1281 }
1282
1283 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1284 {
1285     int kill_session = 0;
1286     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1287
1288     if (new_id)
1289         *new_id = m_referenceId;
1290
1291     if (apdu->which == Z_APDU_searchResponse)
1292     {
1293         Z_SearchResponse *sr = apdu->u.searchResponse;
1294         Z_Records *p = sr->records;
1295         if (p && p->which == Z_Records_NSD)
1296         {
1297             Z_DiagRec dr, *dr_p = &dr;
1298             dr.which = Z_DiagRec_defaultFormat;
1299             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1300
1301             *sr->searchStatus = 0;
1302             display_diagrecs(&dr_p, 1);
1303         }
1304         else
1305         {
1306             if (p && p->which == Z_Records_DBOSD)
1307             {
1308                 if (m_backend_type
1309 #if HAVE_USEMARCON
1310                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1311 #endif
1312                     )
1313                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1314                 if (m_marcxml_mode == marcxml)
1315                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1316                                        m_backend_charset);
1317                 else
1318                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1319                                             m_backend_charset);
1320                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1321                     return 0;
1322
1323             }
1324             if (sr->resultCount)
1325             {
1326                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1327                         *sr->resultCount);
1328                 if (*sr->resultCount < 0)
1329                 {
1330                     m_invalid_session = 1;
1331                     kill_session = 1;
1332
1333                     *sr->searchStatus = 0;
1334                     sr->records =
1335                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1336                     *sr->resultCount = 0;
1337                 }
1338             }
1339         }
1340     }
1341     else if (apdu->which == Z_APDU_presentResponse)
1342     {
1343         Z_PresentResponse *sr = apdu->u.presentResponse;
1344         Z_Records *p = sr->records;
1345         if (p && p->which == Z_Records_NSD)
1346         {
1347             Z_DiagRec dr, *dr_p = &dr;
1348             dr.which = Z_DiagRec_defaultFormat;
1349             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1350             if (*sr->presentStatus == Z_PresentStatus_success)
1351                 *sr->presentStatus = Z_PresentStatus_failure;
1352             display_diagrecs(&dr_p, 1);
1353         }
1354         if (p && p->which == Z_Records_DBOSD)
1355         {
1356             if (m_backend_type
1357 #if HAVE_USEMARCON
1358                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1359 #endif
1360                 )
1361                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1362             if (m_marcxml_mode == marcxml)
1363                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1364                                    m_backend_charset);
1365             else
1366                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1367                                         m_backend_charset);
1368             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1369                 return 0;
1370         }
1371     }
1372     else if (apdu->which == Z_APDU_initResponse)
1373     {
1374         //Get and check negotiation record
1375         //from init response.
1376         handle_charset_lang_negotiation(apdu);
1377
1378         if (m_initRequest_options)
1379         {
1380             Z_Options *nopt =
1381                 (Odr_bitmask *)odr_malloc(odr_encode(),
1382                                           sizeof(Odr_bitmask));
1383             ODR_MASK_ZERO(nopt);
1384
1385             int i;
1386             for (i = 0; i<24; i++)
1387                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1388                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1389                     ODR_MASK_SET(nopt, i);
1390             apdu->u.initResponse->options = nopt;
1391         }
1392         if (m_initRequest_version)
1393         {
1394             Z_ProtocolVersion *nopt =
1395                 (Odr_bitmask *)odr_malloc(odr_encode(),
1396                                           sizeof(Odr_bitmask));
1397             ODR_MASK_ZERO(nopt);
1398
1399             int i;
1400             for (i = 0; i<8; i++)
1401                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1402                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1403                     ODR_MASK_SET(nopt, i);
1404             apdu->u.initResponse->protocolVersion = nopt;
1405         }
1406         apdu->u.initResponse->preferredMessageSize =
1407             odr_intdup(odr_encode(),
1408                        m_client->m_initResponse_preferredMessageSize >
1409                        m_initRequest_preferredMessageSize ?
1410                        m_initRequest_preferredMessageSize :
1411                        m_client->m_initResponse_preferredMessageSize);
1412         apdu->u.initResponse->maximumRecordSize =
1413             odr_intdup(odr_encode(),
1414                        m_client->m_initResponse_maximumRecordSize >
1415                        m_initRequest_maximumRecordSize ?
1416                        m_initRequest_maximumRecordSize :
1417                        m_client->m_initResponse_maximumRecordSize);
1418     }
1419
1420     int r = send_PDU_convert(apdu);
1421     if (r)
1422         return r;
1423     if (kill_session)
1424     {
1425         delete m_client;
1426         m_client = 0;
1427         m_parent->pre_init();
1428     }
1429     return r;
1430 }
1431
1432 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1433 {
1434     int len = 0;
1435     const char *apdu_name_tmp = apdu_name(apdu);
1436     int r = send_Z_PDU(apdu, &len);
1437     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1438         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1439                  get_session_str(),
1440                  apdu_name_tmp, get_hostname(), len);
1441     m_bytes_sent += len;
1442     return r;
1443 }
1444
1445 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1446 {
1447     if (apdu->which == Z_APDU_presentRequest)
1448     {
1449         Z_PresentRequest *pr = apdu->u.presentRequest;
1450         int toget = *pr->numberOfRecordsRequested;
1451         int start = *pr->resultSetStartPoint;
1452
1453         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1454                 pr->resultSetId, start, toget);
1455
1456         if (*m_parent->m_optimize == '0')
1457             return apdu;
1458
1459         if (!m_client->m_last_resultSetId)
1460         {
1461             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1462             new_apdu->u.presentResponse->records =
1463                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1464                                                pr->resultSetId);
1465             send_to_client(new_apdu);
1466             return 0;
1467         }
1468         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1469         {
1470             if (start+toget-1 > m_client->m_last_resultCount)
1471             {
1472                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1473                 new_apdu->u.presentResponse->records =
1474                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1475                 send_to_client(new_apdu);
1476                 return 0;
1477             }
1478             Z_NamePlusRecordList *npr;
1479 #if 0
1480             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1481                     m_session_str, start, toget, yaz_z3950oid_to_str(
1482                         pr->preferredRecordSyntax, &oclass));
1483 #endif
1484             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1485                                           pr->preferredRecordSyntax,
1486                                           pr->recordComposition))
1487             {
1488                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1489                          m_session_str);
1490                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1491                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1492
1493                 new_apdu->u.presentResponse->numberOfRecordsReturned
1494                     = odr_intdup(odr_encode(), toget);
1495
1496                 new_apdu->u.presentResponse->records = (Z_Records*)
1497                     odr_malloc(odr_encode(), sizeof(Z_Records));
1498                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1499                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1500                 new_apdu->u.presentResponse->nextResultSetPosition =
1501                     odr_intdup(odr_encode(), start+toget);
1502
1503                 send_to_client(new_apdu);
1504                 return 0;
1505             }
1506         }
1507     }
1508
1509     if (apdu->which != Z_APDU_searchRequest)
1510         return apdu;
1511     Z_SearchRequest *sr = apdu->u.searchRequest;
1512     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1513     Yaz_Z_Databases this_databases;
1514
1515     this_databases.set(sr->num_databaseNames, (const char **)
1516                        sr->databaseNames);
1517
1518     this_query->set_Z_Query(sr->query);
1519
1520     char query_str[120];
1521     this_query->print(query_str, sizeof(query_str)-1);
1522     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1523
1524     if (*m_parent->m_optimize != '0' &&
1525         m_client->m_last_ok && m_client->m_last_query &&
1526         m_client->m_last_query->match(this_query) &&
1527         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1528         m_client->m_last_databases.match(this_databases))
1529     {
1530         delete this_query;
1531         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1532             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1533         {
1534             Z_NamePlusRecordList *npr;
1535             int toget = *sr->mediumSetPresentNumber;
1536             Z_RecordComposition *comp = 0;
1537
1538             if (toget > m_client->m_last_resultCount)
1539                 toget = m_client->m_last_resultCount;
1540
1541             if (sr->mediumSetElementSetNames)
1542             {
1543                 comp = (Z_RecordComposition *)
1544                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1545                 comp->which = Z_RecordComp_simple;
1546                 comp->u.simple = sr->mediumSetElementSetNames;
1547             }
1548
1549             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1550                                           sr->preferredRecordSyntax, comp))
1551             {
1552                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1553                          m_session_str);
1554                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1555                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1556                 new_apdu->u.searchResponse->resultCount =
1557                     &m_client->m_last_resultCount;
1558
1559                 new_apdu->u.searchResponse->numberOfRecordsReturned
1560                     = odr_intdup(odr_encode(), toget);
1561
1562                 new_apdu->u.searchResponse->presentStatus =
1563                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1564                 new_apdu->u.searchResponse->records = (Z_Records*)
1565                     odr_malloc(odr_encode(), sizeof(Z_Records));
1566                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1567                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1568                 new_apdu->u.searchResponse->nextResultSetPosition =
1569                     odr_intdup(odr_encode(), toget+1);
1570                 send_to_client(new_apdu);
1571                 return 0;
1572             }
1573             else
1574             {
1575                 // medium Set
1576                 // send present request (medium size)
1577                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1578                          m_session_str);
1579
1580                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1581                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1582                 pr->referenceId = sr->referenceId;
1583                 pr->resultSetId = sr->resultSetName;
1584                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1585                 *pr->numberOfRecordsRequested = toget;
1586                 pr->recordComposition = comp;
1587                 m_client->m_sr_transform = 1;
1588                 return new_apdu;
1589             }
1590         }
1591         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1592             m_client->m_last_resultCount <= 0)
1593         {
1594             // large set. Return pseudo-search response immediately
1595             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1596                      m_session_str);
1597             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1598             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1599             new_apdu->u.searchResponse->resultCount =
1600                 &m_client->m_last_resultCount;
1601             send_to_client(new_apdu);
1602             return 0;
1603         }
1604         else
1605         {
1606             Z_NamePlusRecordList *npr;
1607             int toget = m_client->m_last_resultCount;
1608             Z_RecordComposition *comp = 0;
1609             // small set
1610             // send a present request (small set)
1611
1612             if (sr->smallSetElementSetNames)
1613             {
1614                 comp = (Z_RecordComposition *)
1615                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1616                 comp->which = Z_RecordComp_simple;
1617                 comp->u.simple = sr->smallSetElementSetNames;
1618             }
1619
1620             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1621                                           sr->preferredRecordSyntax, comp))
1622             {
1623                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1624                          m_session_str);
1625                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1626                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1627                 new_apdu->u.searchResponse->resultCount =
1628                     &m_client->m_last_resultCount;
1629
1630                 new_apdu->u.searchResponse->numberOfRecordsReturned
1631                     = odr_intdup(odr_encode(), toget);
1632
1633                 new_apdu->u.searchResponse->presentStatus =
1634                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1635                 new_apdu->u.searchResponse->records = (Z_Records*)
1636                     odr_malloc(odr_encode(), sizeof(Z_Records));
1637                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1638                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1639                 new_apdu->u.searchResponse->nextResultSetPosition =
1640                     odr_intdup(odr_encode(), toget+1);
1641                 send_to_client(new_apdu);
1642                 return 0;
1643             }
1644             else
1645             {
1646                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1647                          m_session_str);
1648                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1649                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1650                 pr->referenceId = sr->referenceId;
1651                 pr->resultSetId = sr->resultSetName;
1652                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1653                 *pr->numberOfRecordsRequested = toget;
1654                 pr->recordComposition = comp;
1655                 m_client->m_sr_transform = 1;
1656                 return new_apdu;
1657             }
1658         }
1659     }
1660     else  // query doesn't match
1661     {
1662         delete m_client->m_last_query;
1663         m_client->m_last_query = this_query;
1664         m_client->m_last_ok = 0;
1665         m_client->m_cache.clear();
1666         m_client->m_resultSetStartPoint = 0;
1667
1668         xfree (m_client->m_last_resultSetId);
1669         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1670
1671         m_client->m_last_databases.set(sr->num_databaseNames,
1672                                        (const char **) sr->databaseNames);
1673     }
1674     return apdu;
1675 }
1676
1677
1678 void Yaz_Proxy::inc_request_no()
1679 {
1680     char *cp = strchr(m_session_str, ' ');
1681     m_request_no++;
1682     if (cp)
1683         sprintf(cp+1, "%d ", m_request_no);
1684 }
1685
1686 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1687 {
1688     inc_request_no();
1689
1690     m_bytes_recv += len;
1691
1692     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1693         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1694                  m_session_str, gdu_name(apdu), len);
1695
1696     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1697         shutdown();
1698
1699     m_bw_stat.add_bytes(len);
1700     m_pdu_stat.add_bytes(1);
1701
1702 #if HAVE_GETTIMEOFDAY
1703     gettimeofday((struct timeval *) m_time_tv, 0);
1704 #endif
1705
1706     int bw_total = m_bw_stat.get_total();
1707     int pdu_total = m_pdu_stat.get_total();
1708
1709     int reduce = 0;
1710     if (m_bw_max)
1711     {
1712         if (bw_total > m_bw_max)
1713         {
1714             reduce = (bw_total/m_bw_max);
1715         }
1716     }
1717     if (m_pdu_max)
1718     {
1719         if (pdu_total > m_pdu_max)
1720         {
1721             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1722             reduce = (reduce > nreduce) ? reduce : nreduce;
1723         }
1724     }
1725     m_http_version = 0;
1726     if (reduce)
1727     {
1728         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1729                 m_session_str, reduce, bw_total, pdu_total,
1730                 m_bw_max, m_pdu_max);
1731
1732         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1733         timeout(reduce);       // call us reduce seconds later
1734     }
1735     else if (apdu->which == Z_GDU_Z3950)
1736         handle_incoming_Z_PDU(apdu->u.z3950);
1737     else if (apdu->which == Z_GDU_HTTP_Request)
1738         handle_incoming_HTTP(apdu->u.HTTP_Request);
1739 }
1740
1741 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1742 {
1743     if (m_max_record_retrieve)
1744     {
1745         if (apdu->which == Z_APDU_presentRequest)
1746         {
1747             Z_PresentRequest *pr = apdu->u.presentRequest;
1748             if (pr->numberOfRecordsRequested &&
1749                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1750                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1751         }
1752     }
1753 }
1754 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1755 {
1756     if (apdu->which == Z_APDU_initRequest)
1757     {
1758         yaz_log(YLOG_LOG, "%shandle_charset_lang_negotiation",
1759                 m_session_str);
1760         if (m_initRequest_options &&
1761             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1762             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1763         {
1764             // There is no negotiation proposal from
1765             // client's side. OK. The proxy negotiation
1766             // in use, only.
1767             Z_InitRequest *initRequest = apdu->u.initRequest;
1768             Z_OtherInformation **otherInfo;
1769             Z_OtherInformationUnit *oi;
1770             get_otherInfoAPDU(apdu, &otherInfo);
1771             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1772             if (oi)
1773             {
1774                 ODR_MASK_SET(initRequest->options,
1775                     Z_Options_negotiationModel);
1776                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1777                 oi->information.externallyDefinedInfo =
1778                 yaz_set_proposal_charneg(odr_encode(),
1779                     (const char**)&m_proxy_negotiation_charset,
1780                     m_proxy_negotiation_charset ? 1:0,
1781                     (const char**)&m_proxy_negotiation_lang,
1782                     m_proxy_negotiation_lang ? 1:0,
1783                     1);
1784             }
1785         }
1786         else if (m_initRequest_options &&
1787                  ODR_MASK_GET(m_initRequest_options,
1788                               Z_Options_negotiationModel) &&
1789                  m_charset_converter->get_target_query_charset())
1790         {
1791             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1792                     m_session_str,
1793                     m_charset_converter->get_target_query_charset());
1794             Z_InitRequest *initRequest = apdu->u.initRequest;
1795             Z_CharSetandLanguageNegotiation *negotiation =
1796                 yaz_get_charneg_record (initRequest->otherInfo);
1797             if (negotiation &&
1798                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1799             {
1800                 NMEM nmem = nmem_create();
1801                 char **charsets = 0;
1802                 int num_charsets = 0;
1803                 char **langs = 0;
1804                 int num_langs = 0;
1805                 int selected = 0;
1806                 yaz_get_proposal_charneg (nmem, negotiation,
1807                                           &charsets, &num_charsets,
1808                                           &langs, &num_langs, &selected);
1809                 int i;
1810                 for (i = 0; i<num_charsets; i++)
1811                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
1812                             charsets[i]);
1813                 for (i = 0; i<num_langs; i++)
1814                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
1815                             langs[i]);
1816
1817                 const char *t_charset =
1818                     m_charset_converter->get_target_query_charset();
1819                 // sweep through charsets and pick the first supported
1820                 // conversion
1821                 for (i = 0; i<num_charsets; i++)
1822                 {
1823                     const char *c_charset = charsets[i];
1824                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
1825                         break;
1826                 }
1827                 if (i != num_charsets)
1828                 {
1829                     // got one .. set up ODR for reverse direction
1830                     const char *c_charset = charsets[i];
1831                     odr_set_charset(odr_encode(), c_charset, t_charset);
1832                     m_charset_converter->set_client_query_charset(c_charset);
1833                     m_charset_converter->set_client_charset_selected(selected);
1834                 }
1835                 nmem_destroy(nmem);
1836                 ODR_MASK_CLEAR(m_initRequest_options,
1837                                Z_Options_negotiationModel);
1838                 yaz_del_charneg_record(&initRequest->otherInfo);
1839             }
1840             else
1841             {
1842                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
1843                         m_session_str);
1844             }
1845         }
1846     }
1847     else if (apdu->which == Z_APDU_initResponse)
1848     {
1849         Z_InitResponse *initResponse = apdu->u.initResponse;
1850         Z_OtherInformation **otherInfo;
1851
1852         if (ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel))
1853         {
1854             char *charset = 0;
1855             char *lang = 0;
1856             int selected = 0;
1857
1858             get_otherInfoAPDU(apdu, &otherInfo);
1859
1860             if (!otherInfo && !(*otherInfo))
1861                 return;
1862
1863             Z_CharSetandLanguageNegotiation *charneg =
1864                 yaz_get_charneg_record(*otherInfo);
1865
1866             if (!charneg)
1867                 return;
1868
1869             yaz_get_response_charneg(m_referenceId_mem, charneg,
1870                 &charset, &lang, &selected);
1871
1872             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
1873                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
1874
1875             if (m_initRequest_options &&
1876                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
1877             {
1878                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
1879                     m_session_str);
1880             }
1881             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1882             {
1883                 // negotiation-charset, negotiation-lang
1884                 // elements of config file in use.
1885
1886                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
1887                     m_session_str);
1888
1889                 // clear negotiation option.
1890                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
1891
1892                 // Delete negotiation (charneg-3) entry.
1893                 yaz_del_charneg_record(otherInfo);
1894             }
1895         }
1896         else
1897         {
1898             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1899             {
1900                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
1901                         m_session_str);
1902             }
1903             else if (m_charset_converter->get_client_query_charset())
1904             {
1905                 Z_OtherInformation **otherInfo;
1906                 Z_OtherInformationUnit *oi;
1907                 get_otherInfoAPDU(apdu, &otherInfo);
1908                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1909                 if (oi)
1910                 {
1911                     ODR_MASK_SET(initResponse->options,
1912                                  Z_Options_negotiationModel);
1913                     ODR_MASK_SET(m_initRequest_options,
1914                                  Z_Options_negotiationModel);
1915
1916                     oi->which = Z_OtherInfo_externallyDefinedInfo;
1917                     oi->information.externallyDefinedInfo =
1918                         yaz_set_response_charneg(
1919                             odr_encode(),
1920                             m_charset_converter->get_client_query_charset(),
1921                             0 /* no lang */,
1922                             m_charset_converter->get_client_charset_selected());
1923                 }
1924             }
1925         }
1926     }
1927 }
1928
1929 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1930                                                      int error,
1931                                                      const char *addinfo)
1932 {
1933     Z_Records *rec = (Z_Records *)
1934         odr_malloc (odr, sizeof(*rec));
1935     int *err = (int *)
1936         odr_malloc (odr, sizeof(*err));
1937     Z_DiagRec *drec = (Z_DiagRec *)
1938         odr_malloc (odr, sizeof(*drec));
1939     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1940         odr_malloc (odr, sizeof(*dr));
1941     *err = error;
1942     rec->which = Z_Records_NSD;
1943     rec->u.nonSurrogateDiagnostic = dr;
1944     dr->diagnosticSetId =
1945         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1946     dr->condition = err;
1947     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1948     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1949     return rec;
1950 }
1951
1952 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1953 {
1954     if (apdu->which == Z_APDU_searchRequest &&
1955         apdu->u.searchRequest->query &&
1956         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1957         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1958     {
1959         Z_RPNQuery *rpnquery = 0;
1960         Z_SearchRequest *sr = apdu->u.searchRequest;
1961         char *addinfo = 0;
1962
1963         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
1964                 sr->query->u.type_104->u.cql);
1965
1966         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1967                                           &rpnquery, odr_encode(),
1968                                           &addinfo);
1969         if (r == -3)
1970             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
1971         else if (r)
1972         {
1973             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1974             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1975
1976             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1977             new_apdu->u.searchResponse->records =
1978                 create_nonSurrogateDiagnostics(odr_encode(),
1979                                                yaz_diag_srw_to_bib1(r),
1980                                                addinfo);
1981             *new_apdu->u.searchResponse->searchStatus = 0;
1982
1983             send_to_client(new_apdu);
1984
1985             return 0;
1986         }
1987         else
1988         {
1989             sr->query->which = Z_Query_type_1;
1990             sr->query->u.type_1 = rpnquery;
1991         }
1992         return apdu;
1993     }
1994     return apdu;
1995 }
1996
1997 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
1998 {
1999     if (apdu->which == Z_APDU_searchRequest &&
2000         apdu->u.searchRequest->query)
2001     {
2002         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2003             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2004         {
2005             if (m_http_version)
2006                 m_charset_converter->set_client_query_charset("UTF-8");
2007             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2008             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2009         }
2010     }
2011     return apdu;
2012 }
2013
2014
2015 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2016 {
2017     if (apdu->which == Z_APDU_searchRequest)
2018     {
2019         Z_SearchRequest *sr = apdu->u.searchRequest;
2020         int err = 0;
2021         char *addinfo = 0;
2022
2023         Yaz_ProxyConfig *cfg = check_reconfigure();
2024         if (cfg)
2025             err = cfg->check_query(odr_encode(), m_default_target,
2026                                    sr->query, &addinfo);
2027         if (err)
2028         {
2029             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2030
2031             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2032             new_apdu->u.searchResponse->records =
2033                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2034             *new_apdu->u.searchResponse->searchStatus = 0;
2035
2036             send_to_client(new_apdu);
2037
2038             return 0;
2039         }
2040     }
2041     return apdu;
2042 }
2043
2044 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2045 {
2046     if (apdu->which != Z_APDU_initRequest)
2047         return 1;  // pass if no init request
2048     Z_InitRequest *req = apdu->u.initRequest;
2049
2050     Yaz_ProxyConfig *cfg = check_reconfigure();
2051     if (!cfg)
2052         return 1;  // pass if no config
2053
2054     int ret;
2055     if (req->idAuthentication == 0)
2056     {
2057         ret = cfg->client_authentication(m_default_target, 0, 0, 0);
2058     }
2059     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2060     {
2061         ret = cfg->client_authentication(m_default_target,
2062             req->idAuthentication->u.idPass->userId,
2063             req->idAuthentication->u.idPass->groupId,
2064             req->idAuthentication->u.idPass->password);
2065     }
2066     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2067     {
2068         char user[64], pass[64];
2069         *user = '\0';
2070         *pass = '\0';
2071         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2072         ret = cfg->client_authentication(m_default_target, user, 0, pass);
2073     }
2074     else
2075         ret = cfg->client_authentication(m_default_target, 0, 0, 0);
2076
2077     cfg->target_authentication(m_default_target, odr_encode(), req);
2078
2079     return ret;
2080 }
2081
2082 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2083 {
2084     m_marcxml_mode = none;
2085     if (apdu->which == Z_APDU_searchRequest)
2086     {
2087         Z_SearchRequest *sr = apdu->u.searchRequest;
2088         int err = 0;
2089         char *addinfo = 0;
2090         Yaz_ProxyConfig *cfg = check_reconfigure();
2091
2092         Z_RecordComposition rc_temp, *rc = 0;
2093         if (sr->smallSetElementSetNames)
2094         {
2095             rc_temp.which = Z_RecordComp_simple;
2096             rc_temp.u.simple = sr->smallSetElementSetNames;
2097             rc = &rc_temp;
2098         }
2099
2100         if (sr->preferredRecordSyntax)
2101         {
2102             struct oident *ent;
2103             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2104             m_frontend_type = ent->value;
2105         }
2106         else
2107             m_frontend_type = VAL_NONE;
2108
2109         char *stylesheet_name = 0;
2110         if (cfg)
2111             err = cfg->check_syntax(odr_encode(),
2112                                     m_default_target,
2113                                     sr->preferredRecordSyntax, rc,
2114                                     &addinfo, &stylesheet_name, &m_schema,
2115                                     &m_backend_type, &m_backend_charset,
2116                                     &m_usemarcon_ini_stage1,
2117                                     &m_usemarcon_ini_stage2);
2118         if (stylesheet_name)
2119         {
2120             m_parent->low_socket_close();
2121
2122 #if HAVE_XSLT
2123             if (m_stylesheet_xsp)
2124                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2125             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2126                                                        stylesheet_name);
2127 #endif
2128             m_stylesheet_offset = 0;
2129             xfree(stylesheet_name);
2130
2131             m_parent->low_socket_open();
2132         }
2133         if (err == -1)
2134         {
2135             sr->smallSetElementSetNames = 0;
2136             sr->mediumSetElementSetNames = 0;
2137             m_marcxml_mode = marcxml;
2138             if (m_backend_type)
2139             {
2140
2141                 sr->preferredRecordSyntax =
2142                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2143                                         m_backend_type);
2144             }
2145             else
2146                 sr->preferredRecordSyntax =
2147                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2148                                            VAL_USMARC);
2149         }
2150         else if (err)
2151         {
2152             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2153
2154             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2155             new_apdu->u.searchResponse->records =
2156                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2157             *new_apdu->u.searchResponse->searchStatus = 0;
2158
2159             send_to_client(new_apdu);
2160
2161             return 0;
2162         }
2163         else if (m_backend_type)
2164         {
2165             sr->preferredRecordSyntax =
2166                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2167         }
2168     }
2169     else if (apdu->which == Z_APDU_presentRequest)
2170     {
2171         Z_PresentRequest *pr = apdu->u.presentRequest;
2172         int err = 0;
2173         char *addinfo = 0;
2174         Yaz_ProxyConfig *cfg = check_reconfigure();
2175
2176         if (pr->preferredRecordSyntax)
2177         {
2178             struct oident *ent;
2179             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2180             m_frontend_type = ent->value;
2181         }
2182         else
2183             m_frontend_type = VAL_NONE;
2184
2185         char *stylesheet_name = 0;
2186         if (cfg)
2187             err = cfg->check_syntax(odr_encode(), m_default_target,
2188                                     pr->preferredRecordSyntax,
2189                                     pr->recordComposition,
2190                                     &addinfo, &stylesheet_name, &m_schema,
2191                                     &m_backend_type, &m_backend_charset,
2192                                     &m_usemarcon_ini_stage1,
2193                                     &m_usemarcon_ini_stage2
2194                                     );
2195         if (stylesheet_name)
2196         {
2197             m_parent->low_socket_close();
2198
2199 #if HAVE_XSLT
2200             if (m_stylesheet_xsp)
2201                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2202             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2203                                                        stylesheet_name);
2204 #endif
2205             m_stylesheet_offset = 0;
2206             xfree(stylesheet_name);
2207
2208             m_parent->low_socket_open();
2209         }
2210         if (err == -1)
2211         {
2212             pr->recordComposition = 0;
2213             m_marcxml_mode = marcxml;
2214             if (m_backend_type)
2215             {
2216
2217                 pr->preferredRecordSyntax =
2218                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2219                                         m_backend_type);
2220             }
2221             else
2222                 pr->preferredRecordSyntax =
2223                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2224                                            VAL_USMARC);
2225         }
2226         else if (err)
2227         {
2228             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2229
2230             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2231             new_apdu->u.presentResponse->records =
2232                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2233             *new_apdu->u.presentResponse->presentStatus =
2234                 Z_PresentStatus_failure;
2235
2236             send_to_client(new_apdu);
2237
2238             return 0;
2239         }
2240         else if (m_backend_type)
2241         {
2242             pr->preferredRecordSyntax =
2243                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2244         }
2245     }
2246     return apdu;
2247 }
2248
2249 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2250 {
2251     if (!schema)
2252         return 0;
2253     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2254         odr_malloc(o, sizeof(Z_ElementSetNames));
2255     esn->which = Z_ElementSetNames_generic;
2256     esn->u.generic = odr_strdup(o, schema);
2257     return esn;
2258 }
2259
2260 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2261 {
2262     const char *t = 0;
2263     Yaz_ProxyConfig *cfg = check_reconfigure();
2264     if (cfg)
2265         t = cfg->get_explain_name(db, backend_db);
2266
2267     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2268     {
2269         releaseClient();
2270     }
2271
2272     if (t)
2273     {
2274         xfree(m_default_target);
2275         m_default_target = xstrdup(t);
2276     }
2277 }
2278
2279 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2280 {
2281     struct stat sbuf;
2282     if (strcmp(hreq->method, "GET"))
2283         return 0;
2284     if (hreq->path[0] != '/')
2285     {
2286         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2287         return 0;
2288     }
2289     const char *cp = hreq->path;
2290     while (*cp)
2291     {
2292         if (*cp == '/' && strchr("/.", cp[1]))
2293         {
2294             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2295             return 0;
2296         }
2297         cp++;
2298     }
2299     const char *fname = hreq->path+1;
2300     if (stat(fname, &sbuf))
2301     {
2302         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2303         return 0;
2304     }
2305     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2306     {
2307         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2308         return 0;
2309     }
2310     if (sbuf.st_size > (off_t) 1000000)
2311     {
2312         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2313         return 0;
2314     }
2315
2316     ODR o = odr_encode();
2317     Yaz_ProxyConfig *cfg = check_reconfigure();
2318     const char *ctype = cfg->check_mime_type(fname);
2319     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2320     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2321     if (m_http_version)
2322         hres->version = odr_strdup(o, m_http_version);
2323     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2324     if (m_http_keepalive)
2325         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2326     else
2327         timeout(0);
2328
2329     hres->content_len = sbuf.st_size;
2330     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2331     FILE *f = fopen(fname, "rb");
2332     if (f)
2333     {
2334         fread(hres->content_buf, 1, hres->content_len, f);
2335         fclose(f);
2336     }
2337     else
2338     {
2339         return 0;
2340     }
2341     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2342     {
2343         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2344                  fname);
2345     }
2346     int len;
2347     send_GDU(gdu, &len);
2348     return 1;
2349 }
2350
2351 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2352 {
2353     if (m_s2z_odr_init)
2354     {
2355         odr_destroy(m_s2z_odr_init);
2356         m_s2z_odr_init = 0;
2357     }
2358     if (m_s2z_odr_search)
2359     {
2360         odr_destroy(m_s2z_odr_search);
2361         m_s2z_odr_search = 0;
2362     }
2363
2364     m_http_keepalive = 0;
2365     m_http_version = 0;
2366     if (!strcmp(hreq->version, "1.0"))
2367     {
2368         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2369         if (v && !strcmp(v, "Keep-Alive"))
2370             m_http_keepalive = 1;
2371         else
2372             m_http_keepalive = 0;
2373         m_http_version = "1.0";
2374     }
2375     else
2376     {
2377         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2378         if (v && !strcmp(v, "close"))
2379             m_http_keepalive = 0;
2380         else
2381             m_http_keepalive = 1;
2382         m_http_version = "1.1";
2383     }
2384
2385     Z_SRW_PDU *srw_pdu = 0;
2386     Z_SOAP *soap_package = 0;
2387     char *charset = 0;
2388     Z_SRW_diagnostic *diagnostic = 0;
2389     int num_diagnostic = 0;
2390
2391     if (file_access(hreq))
2392     {
2393         return;
2394     }
2395     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2396                             &charset) == 0
2397              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2398                                &charset, &diagnostic, &num_diagnostic) == 0)
2399     {
2400         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2401         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2402         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2403         m_s2z_init_apdu = 0;
2404         m_s2z_search_apdu = 0;
2405         m_s2z_present_apdu = 0;
2406
2407         m_s2z_stylesheet = 0;
2408
2409         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2410         {
2411             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2412
2413             const char *backend_db = srw_req->database;
2414             srw_get_client(srw_req->database, &backend_db);
2415
2416             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2417             // recordXPath unsupported.
2418             if (srw_req->recordXPath)
2419             {
2420                 yaz_add_srw_diagnostic(odr_decode(),
2421                                        &diagnostic, &num_diagnostic,
2422                                        72, 0);
2423             }
2424             // sort unsupported
2425             if (srw_req->sort_type != Z_SRW_sort_type_none)
2426             {
2427                 yaz_add_srw_diagnostic(odr_decode(),
2428                                        &diagnostic, &num_diagnostic,
2429                                        80, 0);
2430             }
2431             // save stylesheet
2432             if (srw_req->stylesheet)
2433                 m_s2z_stylesheet =
2434                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2435
2436             // set packing for response records ..
2437             if (srw_req->recordPacking &&
2438                 !strcmp(srw_req->recordPacking, "xml"))
2439                 m_s2z_packing = Z_SRW_recordPacking_XML;
2440             else
2441                 m_s2z_packing = Z_SRW_recordPacking_string;
2442
2443             if (num_diagnostic)
2444             {
2445                 Z_SRW_PDU *srw_pdu =
2446                     yaz_srw_get(odr_encode(),
2447                                 Z_SRW_searchRetrieve_response);
2448                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2449
2450                 srw_res->diagnostics = diagnostic;
2451                 srw_res->num_diagnostics = num_diagnostic;
2452                 send_srw_response(srw_pdu);
2453                 return;
2454             }
2455
2456             // prepare search PDU
2457             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2458                                           Z_APDU_searchRequest);
2459             Z_SearchRequest *z_searchRequest =
2460                 m_s2z_search_apdu->u.searchRequest;
2461
2462             z_searchRequest->num_databaseNames = 1;
2463             z_searchRequest->databaseNames = (char**)
2464                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2465             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2466                                                            backend_db);
2467
2468             // query transformation
2469             Z_Query *query = (Z_Query *)
2470                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2471             z_searchRequest->query = query;
2472
2473             if (srw_req->query_type == Z_SRW_query_type_cql)
2474             {
2475                 Z_External *ext = (Z_External *)
2476                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2477                 ext->direct_reference =
2478                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2479                 ext->indirect_reference = 0;
2480                 ext->descriptor = 0;
2481                 ext->which = Z_External_CQL;
2482                 ext->u.cql = srw_req->query.cql;
2483
2484                 query->which = Z_Query_type_104;
2485                 query->u.type_104 =  ext;
2486             }
2487             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2488             {
2489                 Z_RPNQuery *RPNquery;
2490                 YAZ_PQF_Parser pqf_parser;
2491
2492                 pqf_parser = yaz_pqf_create ();
2493
2494                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2495                                           srw_req->query.pqf);
2496                 if (!RPNquery)
2497                 {
2498                     const char *pqf_msg;
2499                     size_t off;
2500                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2501                     yaz_log(YLOG_LOG, "%*s^\n", off+4, "");
2502                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2503
2504                     send_to_srw_client_error(10, 0);
2505                     return;
2506                 }
2507                 query->which = Z_Query_type_1;
2508                 query->u.type_1 =  RPNquery;
2509
2510                 yaz_pqf_destroy (pqf_parser);
2511             }
2512             else
2513             {
2514                 send_to_srw_client_error(7, "query");
2515                 return;
2516             }
2517
2518             // present
2519             m_s2z_present_apdu = 0;
2520             int max = 0;
2521             if (srw_req->maximumRecords)
2522                 max = *srw_req->maximumRecords;
2523             int start = 1;
2524             if (srw_req->startRecord)
2525                 start = *srw_req->startRecord;
2526             if (max > 0)
2527             {
2528                 // Some backend, such as Voyager doesn't honor piggyback
2529                 // So we use present always (0 &&).
2530                 if (0 && start <= 1)  // Z39.50 piggyback
2531                 {
2532                     *z_searchRequest->smallSetUpperBound = max;
2533                     *z_searchRequest->mediumSetPresentNumber = max;
2534                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2535
2536                     z_searchRequest->preferredRecordSyntax =
2537                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2538                                                VAL_TEXT_XML);
2539                     if (srw_req->recordSchema)
2540                     {
2541                         z_searchRequest->smallSetElementSetNames =
2542                             z_searchRequest->mediumSetElementSetNames =
2543                             mk_esn_from_schema(m_s2z_odr_search,
2544                                                srw_req->recordSchema);
2545                     }
2546                 }
2547                 else   // Z39.50 present
2548                 {
2549                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2550                                                    Z_APDU_presentRequest);
2551                     Z_PresentRequest *z_presentRequest =
2552                         m_s2z_present_apdu->u.presentRequest;
2553                     *z_presentRequest->resultSetStartPoint = start;
2554                     *z_presentRequest->numberOfRecordsRequested = max;
2555                     z_presentRequest->preferredRecordSyntax =
2556                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2557                                                VAL_TEXT_XML);
2558                     if (srw_req->recordSchema)
2559                     {
2560                         z_presentRequest->recordComposition =
2561                             (Z_RecordComposition *)
2562                             odr_malloc(m_s2z_odr_search,
2563                                        sizeof(Z_RecordComposition));
2564                         z_presentRequest->recordComposition->which =
2565                             Z_RecordComp_simple;
2566                         z_presentRequest->recordComposition->u.simple =
2567                             mk_esn_from_schema(m_s2z_odr_search,
2568                                                srw_req->recordSchema);
2569                     }
2570                 }
2571             }
2572             if (!m_client)
2573             {
2574                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2575                                             Z_APDU_initRequest);
2576
2577                 // prevent m_initRequest_apdu memory from being grabbed
2578                 // in Yaz_Proxy::handle_incoming_Z_PDU
2579                 m_initRequest_apdu = m_s2z_init_apdu;
2580                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2581                 return;
2582             }
2583             else
2584             {
2585                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2586                 return;
2587             }
2588         }
2589         else if (srw_pdu->which == Z_SRW_explain_request)
2590         {
2591             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2592
2593             const char *backend_db = srw_req->database;
2594             srw_get_client(srw_req->database, &backend_db);
2595
2596             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2597
2598             // save stylesheet
2599             if (srw_req->stylesheet)
2600                 m_s2z_stylesheet =
2601                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2602
2603             if (srw_req->recordPacking &&
2604                 !strcmp(srw_req->recordPacking, "xml"))
2605                 m_s2z_packing = Z_SRW_recordPacking_XML;
2606             else
2607                 m_s2z_packing = Z_SRW_recordPacking_string;
2608
2609             if (num_diagnostic)
2610             {
2611                 send_srw_explain_response(diagnostic, num_diagnostic);
2612                 return;
2613             }
2614
2615             if (!m_client)
2616             {
2617                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2618                                             Z_APDU_initRequest);
2619
2620                 // prevent m_initRequest_apdu memory from being grabbed
2621                 // in Yaz_Proxy::handle_incoming_Z_PDU
2622                 m_initRequest_apdu = m_s2z_init_apdu;
2623                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2624             }
2625             else
2626                 send_srw_explain_response(0, 0);
2627             return;
2628         }
2629         else if (srw_pdu->which == Z_SRW_scan_request)
2630         {
2631             m_s2z_database = odr_strdup(m_s2z_odr_init,
2632                                         srw_pdu->u.scan_request->database);
2633
2634             yaz_add_srw_diagnostic(odr_decode(),
2635                                    &diagnostic, &num_diagnostic,
2636                                    4, "scan");
2637             Z_SRW_PDU *srw_pdu =
2638                 yaz_srw_get(odr_encode(),
2639                             Z_SRW_scan_response);
2640             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2641
2642             srw_res->diagnostics = diagnostic;
2643             srw_res->num_diagnostics = num_diagnostic;
2644             send_srw_response(srw_pdu);
2645             return;
2646         }
2647         else
2648         {
2649             m_s2z_database = 0;
2650
2651             send_to_srw_client_error(4, 0);
2652         }
2653     }
2654     send_http_response(400);
2655 }
2656
2657 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2658 {
2659     Z_OtherInformation **oi;
2660     get_otherInfoAPDU(apdu, &oi);
2661
2662     if (apdu->u.initRequest->implementationId)
2663         yaz_log(YLOG_LOG, "%simplementationId: %s",
2664                 m_session_str, apdu->u.initRequest->implementationId);
2665     if (apdu->u.initRequest->implementationName)
2666         yaz_log(YLOG_LOG, "%simplementationName: %s",
2667                 m_session_str, apdu->u.initRequest->implementationName);
2668     if (apdu->u.initRequest->implementationVersion)
2669         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2670                 m_session_str, apdu->u.initRequest->implementationVersion);
2671     if (m_initRequest_apdu == 0)
2672     {
2673         if (m_initRequest_mem)
2674             nmem_destroy(m_initRequest_mem);
2675
2676         m_initRequest_apdu = apdu;
2677         m_initRequest_mem = odr_extract_mem(odr_decode());
2678
2679         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2680             preferredMessageSize;
2681         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2682         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2683             maximumRecordSize;
2684         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2685
2686         // Save proposal charsets and langs.
2687         if (ODR_MASK_GET(apdu->u.initRequest->options,
2688                          Z_Options_negotiationModel))
2689         {
2690             Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2691                 yaz_get_charneg_record(*oi);
2692
2693             yaz_get_proposal_charneg(m_referenceId_mem,
2694                                      charSetandLangRecord,
2695                                      &m_initRequest_oi_negotiation_charsets,
2696                                      &m_initRequest_oi_negotiation_num_charsets,
2697                                      &m_initRequest_oi_negotiation_langs,
2698                                      &m_initRequest_oi_negotiation_num_langs,
2699                                      &m_initRequest_oi_negotiation_selected);
2700
2701             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2702             {
2703                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2704                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2705                         m_initRequest_oi_negotiation_charsets[i]:"none");
2706             }
2707             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2708             {
2709                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2710                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2711                         m_initRequest_oi_negotiation_langs[i]:"none");
2712             }
2713             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2714                     m_session_str, m_initRequest_oi_negotiation_selected);
2715         }
2716         // save init options for the response..
2717         m_initRequest_options = apdu->u.initRequest->options;
2718
2719         apdu->u.initRequest->options =
2720             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2721                                        sizeof(Odr_bitmask));
2722         ODR_MASK_ZERO(apdu->u.initRequest->options);
2723         int i;
2724         for (i = 0; i<= 24; i++)
2725             ODR_MASK_SET(apdu->u.initRequest->options, i);
2726         // check negotiation option
2727         if (!ODR_MASK_GET(m_initRequest_options,
2728                           Z_Options_negotiationModel))
2729         {
2730             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2731                            Z_Options_negotiationModel);
2732         }
2733         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2734                        Z_Options_concurrentOperations);
2735         // make new version
2736         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2737         apdu->u.initRequest->protocolVersion =
2738             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2739                                        sizeof(Odr_bitmask));
2740         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2741
2742         for (i = 0; i<= 8; i++)
2743             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2744     }
2745     if (m_client->m_init_flag)
2746     {
2747         if (handle_init_response_for_invalid_session(apdu))
2748             return;
2749         if (m_client->m_initResponse)
2750         {
2751             Z_APDU *apdu2 = m_client->m_initResponse;
2752             apdu2->u.initResponse->otherInfo = 0;
2753             if (m_client->m_cookie && *m_client->m_cookie)
2754                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2755                                            m_client->m_cookie);
2756             apdu2->u.initResponse->referenceId =
2757                 apdu->u.initRequest->referenceId;
2758             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2759             apdu2->u.initResponse->protocolVersion =
2760                 m_client->m_initResponse_version;
2761
2762             send_to_client(apdu2);
2763             return;
2764         }
2765     }
2766     m_client->m_init_flag = 1;
2767
2768     Auth_Msg *m = new Auth_Msg;
2769     m->m_apdu = apdu;
2770     m->m_proxy = this;
2771     m_my_thread->put(m);
2772 }
2773
2774 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2775 {
2776     Z_ReferenceId **refid = get_referenceIdP(apdu);
2777     nmem_reset(m_referenceId_mem);
2778     if (refid && *refid)
2779     {
2780         m_referenceId = (Z_ReferenceId *)
2781             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2782         m_referenceId->len = m_referenceId->size = (*refid)->len;
2783         m_referenceId->buf = (unsigned char *)
2784             nmem_malloc(m_referenceId_mem, (*refid)->len);
2785         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2786     }
2787     else
2788         m_referenceId = 0;
2789
2790     if (!m_client && m_invalid_session)
2791     {
2792         m_apdu_invalid_session = apdu;
2793         m_mem_invalid_session = odr_extract_mem(odr_decode());
2794         apdu = m_initRequest_apdu;
2795     }
2796
2797     // Determine our client.
2798     Z_OtherInformation **oi;
2799     get_otherInfoAPDU(apdu, &oi);
2800     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2801     if (!m_client)
2802     {
2803         if (m_http_version)
2804         {
2805             send_http_response(404);
2806             return;
2807         }
2808         else
2809         {
2810             delete this;
2811             return;
2812         }
2813     }
2814
2815     m_client->m_server = this;
2816
2817     if (apdu->which == Z_APDU_initRequest)
2818         handle_init(apdu);
2819     else
2820         handle_incoming_Z_PDU_2(apdu);
2821 }
2822
2823 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
2824 {
2825     handle_max_record_retrieve(apdu);
2826
2827     if (apdu)
2828         apdu = handle_syntax_validation(apdu);
2829
2830     if (apdu)
2831         apdu = handle_query_transformation(apdu);
2832
2833     if (apdu)
2834         apdu = handle_target_charset_conversion(apdu);
2835
2836     if (apdu)
2837         apdu = handle_query_validation(apdu);
2838
2839     if (apdu)
2840         apdu = result_set_optimize(apdu);
2841
2842     if (!apdu)
2843     {
2844         m_client->timeout(m_target_idletime);  // mark it active even
2845         // though we didn't use it
2846         return;
2847     }
2848     // Add otherInformation entry in APDU if
2849     // negotiation is in use.
2850     if (apdu)
2851         handle_charset_lang_negotiation(apdu);
2852
2853     // delete other info construct completely if 0 elements
2854     Z_OtherInformation **oi;
2855     get_otherInfoAPDU(apdu, &oi);
2856     if (oi && *oi && (*oi)->num_elements == 0)
2857         *oi = 0;
2858
2859     if (apdu->which == Z_APDU_presentRequest &&
2860         m_client->m_resultSetStartPoint == 0)
2861     {
2862         Z_PresentRequest *pr = apdu->u.presentRequest;
2863         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2864         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2865     } else {
2866         m_client->m_resultSetStartPoint = 0;
2867     }
2868     if (m_client->send_to_target(apdu) < 0)
2869     {
2870         delete m_client;
2871         m_client = 0;
2872         delete this;
2873     }
2874     else
2875         m_client->m_waiting = 1;
2876 }
2877
2878 void Yaz_Proxy::connectNotify()
2879 {
2880 }
2881
2882 void Yaz_Proxy::releaseClient()
2883 {
2884     xfree(m_proxyTarget);
2885     m_proxyTarget = 0;
2886     m_invalid_session = 0;
2887     // only keep if keep_alive flag is set...
2888     if (m_client &&
2889         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2890         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2891         m_client->m_waiting == 0)
2892     {
2893         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2894                  m_session_str,
2895                  m_client->get_hostname());
2896         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2897                 m_session_str, m_client->m_pdu_recv,
2898                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2899                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2900         assert (m_client->m_waiting != 2);
2901         // Tell client (if any) that no server connection is there..
2902         m_client->m_server = 0;
2903         m_client = 0;
2904     }
2905     else if (m_client)
2906     {
2907         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
2908                  m_session_str,
2909                  m_client->get_hostname());
2910         assert (m_client->m_waiting != 2);
2911         delete m_client;
2912         m_client = 0;
2913     }
2914     else if (!m_parent)
2915     {
2916         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
2917                  m_session_str);
2918         assert (m_parent);
2919     }
2920     else
2921     {
2922         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
2923                  m_session_str);
2924     }
2925     if (m_parent)
2926         m_parent->pre_init();
2927 }
2928
2929 void Yaz_Proxy::shutdown()
2930 {
2931     releaseClient();
2932     delete this;
2933 }
2934
2935 const char *Yaz_ProxyClient::get_session_str()
2936 {
2937     if (!m_server)
2938         return "0 ";
2939     return m_server->get_session_str();
2940 }
2941
2942 void Yaz_ProxyClient::shutdown()
2943 {
2944     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2945              get_hostname());
2946     delete m_server;
2947     delete this;
2948 }
2949
2950 void Yaz_Proxy::failNotify()
2951 {
2952     inc_request_no();
2953     yaz_log (YLOG_LOG, "%sConnection closed by client",
2954              get_session_str());
2955     shutdown();
2956 }
2957
2958 void Yaz_ProxyClient::failNotify()
2959 {
2960     if (m_server)
2961         m_server->inc_request_no();
2962     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
2963              get_session_str(), get_hostname());
2964     shutdown();
2965 }
2966
2967 void Yaz_ProxyClient::connectNotify()
2968 {
2969     const char *s = get_session_str();
2970     const char *h = get_hostname();
2971     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2972              m_target_idletime);
2973     timeout(m_target_idletime);
2974     if (!m_server)
2975         pre_init_client();
2976 }
2977
2978 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
2979                                               *the_PDU_Observable, int fd)
2980 {
2981     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2982 }
2983
2984 Yaz_ProxyClient::~Yaz_ProxyClient()
2985 {
2986     if (m_prev)
2987         *m_prev = m_next;
2988     if (m_next)
2989         m_next->m_prev = m_prev;
2990     m_waiting = 2;     // for debugging purposes only.
2991     odr_destroy(m_init_odr);
2992     delete m_last_query;
2993     xfree (m_last_resultSetId);
2994     xfree (m_cookie);
2995 }
2996
2997 void Yaz_ProxyClient::pre_init_client()
2998 {
2999     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3000     Z_InitRequest *req = apdu->u.initRequest;
3001
3002     int i;
3003     for (i = 0; i<= 24; i++)
3004         ODR_MASK_SET(req->options, i);
3005     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3006                    Z_Options_negotiationModel);
3007     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3008                    Z_Options_concurrentOperations);
3009     for (i = 0; i<= 10; i++)
3010         ODR_MASK_SET(req->protocolVersion, i);
3011
3012     if (send_to_target(apdu) < 0)
3013     {
3014         delete this;
3015     }
3016     else
3017     {
3018         m_waiting = 1;
3019         m_init_flag = 1;
3020     }
3021 }
3022
3023 void Yaz_Proxy::pre_init()
3024 {
3025     int i;
3026     const char *name = 0;
3027     const char *zurl_in_use[MAX_ZURL_PLEX];
3028     int limit_bw, limit_pdu, limit_req;
3029     int target_idletime, client_idletime;
3030     int max_clients;
3031     int keepalive_limit_bw, keepalive_limit_pdu;
3032     int pre_init;
3033     const char *cql2rpn = 0;
3034     const char *authentication = 0;
3035     const char *negotiation_charset = 0;
3036     const char *negotiation_lang = 0;
3037
3038     Yaz_ProxyConfig *cfg = check_reconfigure();
3039
3040     zurl_in_use[0] = 0;
3041
3042     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3043         set_APDU_yazlog(1);
3044     else
3045         set_APDU_yazlog(0);
3046
3047     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3048                                           &limit_bw, &limit_pdu, &limit_req,
3049                                           &target_idletime, &client_idletime,
3050                                           &max_clients,
3051                                           &keepalive_limit_bw,
3052                                           &keepalive_limit_pdu,
3053                                           &pre_init,
3054                                           &cql2rpn,
3055                                           &authentication,
3056                                           &negotiation_charset,
3057                                           &negotiation_lang,
3058                                           0) ; i++)
3059     {
3060         if (pre_init)
3061         {
3062             int j;
3063             for (j = 0; zurl_in_use[j]; j++)
3064             {
3065                 Yaz_ProxyClient *c;
3066                 int spare = 0;
3067                 int spare_waiting = 0;
3068                 int in_use = 0;
3069                 int other = 0;
3070                 for (c = m_clientPool; c; c = c->m_next)
3071                 {
3072                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3073                     {
3074                         if (c->m_cookie == 0)
3075                         {
3076                             if (c->m_server == 0)
3077                                 if (c->m_waiting)
3078                                     spare_waiting++;
3079                                 else
3080                                     spare++;
3081                             else
3082                                 in_use++;
3083                         }
3084                         else
3085                             other++;
3086                     }
3087                 }
3088                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3089                         "sparew=%d preinit=%d",m_session_str,
3090                         name, zurl_in_use[j], in_use, other,
3091                         spare, spare_waiting, pre_init);
3092                 if (spare + spare_waiting < pre_init)
3093                 {
3094                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3095                     c->m_next = m_clientPool;
3096                     if (c->m_next)
3097                         c->m_next->m_prev = &c->m_next;
3098                     m_clientPool = c;
3099                     c->m_prev = &m_clientPool;
3100
3101                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3102                         c->set_APDU_yazlog(1);
3103                     else
3104                         c->set_APDU_yazlog(0);
3105
3106                     if (c->client(zurl_in_use[j]))
3107                     {
3108                         timeout(60);
3109                         delete c;
3110                         return;
3111                     }
3112                     c->timeout(30);
3113                     c->m_waiting = 1;
3114                     c->m_target_idletime = target_idletime;
3115                     c->m_seqno = m_seqno++;
3116                 }
3117             }
3118         }
3119     }
3120 }
3121
3122 void Yaz_Proxy::timeoutNotify()
3123 {
3124     if (m_parent)
3125     {
3126         if (m_bw_hold_PDU)
3127         {
3128             timeout(m_client_idletime);
3129             Z_GDU *apdu = m_bw_hold_PDU;
3130             m_bw_hold_PDU = 0;
3131
3132             if (apdu->which == Z_GDU_Z3950)
3133                 handle_incoming_Z_PDU(apdu->u.z3950);
3134             else if (apdu->which == Z_GDU_HTTP_Request)
3135                 handle_incoming_HTTP(apdu->u.HTTP_Request);
3136         }
3137         else if (m_stylesheet_nprl)
3138             convert_xsl_delay();
3139         else
3140         {
3141             inc_request_no();
3142
3143             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3144             shutdown();
3145         }
3146     }
3147     else
3148     {
3149         timeout(600);
3150         pre_init();
3151     }
3152 }
3153
3154 void Yaz_Proxy::markInvalid()
3155 {
3156     m_client = 0;
3157     m_invalid_session = 1;
3158 }
3159
3160 void Yaz_ProxyClient::timeoutNotify()
3161 {
3162     if (m_server)
3163         m_server->inc_request_no();
3164
3165     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3166              get_hostname());
3167     m_waiting = 1;
3168     m_root->pre_init();
3169     if (m_server && m_init_flag)
3170     {
3171         // target timed out in a session that was properly initialized
3172         // server object stay alive but we mark it as invalid so it
3173         // gets initialized again
3174         m_server->markInvalid();
3175         m_server = 0;
3176     }
3177     shutdown();
3178 }
3179
3180 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
3181                                  Yaz_Proxy *parent) :
3182     Z_Assoc (the_PDU_Observable)
3183 {
3184     m_cookie = 0;
3185     m_next = 0;
3186     m_prev = 0;
3187     m_init_flag = 0;
3188     m_last_query = 0;
3189     m_last_resultSetId = 0;
3190     m_last_resultCount = 0;
3191     m_last_ok = 0;
3192     m_sr_transform = 0;
3193     m_waiting = 0;
3194     m_init_odr = odr_createmem (ODR_DECODE);
3195     m_initResponse = 0;
3196     m_initResponse_options = 0;
3197     m_initResponse_version = 0;
3198     m_initResponse_preferredMessageSize = 0;
3199     m_initResponse_maximumRecordSize = 0;
3200     m_resultSetStartPoint = 0;
3201     m_bytes_sent = m_bytes_recv = 0;
3202     m_pdu_recv = 0;
3203     m_server = 0;
3204     m_seqno = 0;
3205     m_target_idletime = 600;
3206     m_root = parent;
3207 }
3208
3209 const char *Yaz_Proxy::option(const char *name, const char *value)
3210 {
3211     if (!strcmp (name, "optimize")) {
3212         if (value) {
3213             xfree (m_optimize);
3214             m_optimize = xstrdup (value);
3215         }
3216         return m_optimize;
3217     }
3218     return 0;
3219 }
3220
3221 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
3222 {
3223
3224 }
3225
3226 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
3227 {
3228     if (apdu->which == Z_GDU_Z3950)
3229         recv_Z_PDU(apdu->u.z3950, len);
3230     else if (apdu->which == Z_GDU_HTTP_Response)
3231         recv_HTTP_response(apdu->u.HTTP_Response, len);
3232     else
3233         shutdown();
3234 }
3235
3236 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
3237 {
3238     if (!m_invalid_session)
3239         return 0;
3240     m_invalid_session = 0;
3241     handle_incoming_Z_PDU(m_apdu_invalid_session);
3242     assert (m_mem_invalid_session);
3243     nmem_destroy(m_mem_invalid_session);
3244     m_mem_invalid_session = 0;
3245     return 1;
3246 }
3247
3248 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
3249 {
3250     m_bytes_recv += len;
3251
3252     m_pdu_recv++;
3253     m_waiting = 0;
3254     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
3255         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
3256                  apdu_name(apdu), get_hostname(), len);
3257     if (apdu->which == Z_APDU_initResponse)
3258     {
3259         if (!m_server)  // if this is a pre init session , check for more
3260             m_root->pre_init();
3261         NMEM nmem = odr_extract_mem (odr_decode());
3262         odr_reset (m_init_odr);
3263         nmem_transfer (m_init_odr->mem, nmem);
3264         m_initResponse = apdu;
3265         m_initResponse_options = apdu->u.initResponse->options;
3266         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3267         m_initResponse_preferredMessageSize =
3268             *apdu->u.initResponse->preferredMessageSize;
3269         m_initResponse_maximumRecordSize =
3270             *apdu->u.initResponse->maximumRecordSize;
3271
3272         Z_InitResponse *ir = apdu->u.initResponse;
3273         char *im0 = ir->implementationName;
3274
3275         char *im1 = (char*)
3276             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3277         *im1 = '\0';
3278         if (im0)
3279         {
3280             strcat(im1, im0);
3281             strcat(im1, " ");
3282         }
3283         strcat(im1, "(YAZ Proxy)");
3284         ir->implementationName = im1;
3285
3286         nmem_destroy (nmem);
3287
3288         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3289             return;
3290     }
3291     if (apdu->which == Z_APDU_searchResponse)
3292     {
3293         Z_SearchResponse *sr = apdu->u.searchResponse;
3294         m_last_resultCount = *sr->resultCount;
3295         int status = *sr->searchStatus;
3296         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3297         {
3298             m_last_ok = 1;
3299
3300             if (sr->records && sr->records->which == Z_Records_DBOSD)
3301             {
3302                 m_cache.add(odr_decode(),
3303                             sr->records->u.databaseOrSurDiagnostics, 1,
3304                             *sr->resultCount);
3305             }
3306         }
3307     }
3308     if (apdu->which == Z_APDU_presentResponse)
3309     {
3310         Z_PresentResponse *pr = apdu->u.presentResponse;
3311         if (m_sr_transform)
3312         {
3313             m_sr_transform = 0;
3314             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3315             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3316             sr->referenceId = pr->referenceId;
3317             *sr->resultCount = m_last_resultCount;
3318             sr->records = pr->records;
3319             sr->nextResultSetPosition = pr->nextResultSetPosition;
3320             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3321             apdu = new_apdu;
3322         }
3323         if (pr->records &&
3324             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3325         {
3326             m_cache.add(odr_decode(),
3327                         pr->records->u.databaseOrSurDiagnostics,
3328                         m_resultSetStartPoint, -1);
3329             m_resultSetStartPoint = 0;
3330         }
3331     }
3332     if (m_cookie)
3333         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3334     if (m_server)
3335     {
3336         m_server->send_to_client(apdu);
3337     }
3338     if (apdu->which == Z_APDU_close)
3339     {
3340         shutdown();
3341     }
3342 }
3343
3344 void Yaz_Proxy::low_socket_close()
3345 {
3346 #if WIN32
3347 #else
3348     int i;
3349     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3350         if  (m_lo_fd[i] >= 0)
3351             ::close(m_lo_fd[i]);
3352 #endif
3353 }
3354
3355 void Yaz_Proxy::low_socket_open()
3356 {
3357 #if WIN32
3358 #else
3359     int i;
3360     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3361         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3362 #endif
3363 }
3364
3365 int Yaz_Proxy::server(const char *addr)
3366 {
3367     int r = Z_Assoc::server(addr);
3368     if (!r)
3369     {
3370         yaz_log(YLOG_LOG, "%sStarted proxy "
3371 #ifdef VERSION
3372             VERSION
3373 #endif
3374             " on %s", m_session_str, addr);
3375         timeout(1);
3376     }
3377     return r;
3378 }
3379