src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.28 2005-05-30 20:09:21 adam Exp $
   2    Copyright (c) 1998-2005, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <stdlib.h>
  42 #include <time.h>
  43 #include <fcntl.h>
  44
  45 #include <yaz/srw.h>
  46 #include <yaz/marcdisp.h>
  47 #include <yaz/yaz-iconv.h>
  48 #include <yaz/log.h>
  49 #include <yaz/diagbib1.h>
  50 #include "proxyp.h"
  51 #include <yaz/pquery.h>
  52 #include <yaz/otherinfo.h>
  53 #include <yaz/charneg.h>
  54 #include "msg-thread.h"
  55
  56 class Auth_Msg : public IMsg_Thread {
  57 public:
  58     int m_ret;
  59     void destroy();
  60     IMsg_Thread *handle();
  61     void result();
  62     Yaz_Proxy *m_proxy;
  63     Z_APDU *m_apdu;
  64 };
  65
  66 IMsg_Thread *Auth_Msg::handle()
  67 {
  68     yaz_log(YLOG_LOG, "Auth_Msg:handle begin");
  69     m_ret = m_proxy->handle_authentication(m_apdu);
  70     yaz_log(YLOG_LOG, "Auth_Msg:handle end");
  71     return this;
  72 }
  73
  74 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
  75 {
  76     if (ret == 0)
  77     {
  78         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
  79         *apdu_reject->u.initResponse->result = 0;
  80         send_to_client(apdu_reject);
  81         shutdown();
  82     }
  83     else
  84         handle_incoming_Z_PDU_2(apdu);
  85 }
  86
  87 void Auth_Msg::result()
  88 {
  89     yaz_log(YLOG_LOG, "Auth_Msg:result");
  90     m_proxy->result_authentication(m_apdu, m_ret);
  91     delete this;
  92 }
  93
  94 static const char *apdu_name(Z_APDU *apdu)
  95 {
  96     switch (apdu->which)
  97     {
  98     case Z_APDU_initRequest:
  99         return "initRequest";
 100     case Z_APDU_initResponse:
 101         return "initResponse";
 102     case Z_APDU_searchRequest:
 103         return "searchRequest";
 104     case Z_APDU_searchResponse:
 105         return "searchResponse";
 106     case Z_APDU_presentRequest:
 107         return "presentRequest";
 108     case Z_APDU_presentResponse:
 109         return "presentResponse";
 110     case Z_APDU_deleteResultSetRequest:
 111         return "deleteResultSetRequest";
 112     case Z_APDU_deleteResultSetResponse:
 113         return "deleteResultSetResponse";
 114     case Z_APDU_scanRequest:
 115         return "scanRequest";
 116     case Z_APDU_scanResponse:
 117         return "scanResponse";
 118     case Z_APDU_sortRequest:
 119         return "sortRequest";
 120     case Z_APDU_sortResponse:
 121         return "sortResponse";
 122     case Z_APDU_extendedServicesRequest:
 123         return "extendedServicesRequest";
 124     case Z_APDU_extendedServicesResponse:
 125         return "extendedServicesResponse";
 126     case Z_APDU_close:
 127         return "close";
 128     }
 129     return "other";
 130 }
 131
 132 static const char *gdu_name(Z_GDU *gdu)
 133 {
 134     switch(gdu->which)
 135     {
 136     case Z_GDU_Z3950:
 137         return apdu_name(gdu->u.z3950);
 138     case Z_GDU_HTTP_Request:
 139         return "HTTP Request";
 140     case Z_GDU_HTTP_Response:
 141         return "HTTP Response";
 142     }
 143     return "Unknown request/response";
 144 }
 145
 146 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
 147                      IYazSocketObservable *the_socket_observable,
 148                      Yaz_Proxy *parent)
 149     :
 150     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 151 {
 152     m_PDU_Observable = the_PDU_Observable;
 153     m_socket_observable = the_socket_observable;
 154     m_client = 0;
 155     m_parent = parent;
 156     m_clientPool = 0;
 157     m_seqno = 1;
 158     m_keepalive_limit_bw = 500000;
 159     m_keepalive_limit_pdu = 1000;
 160     m_proxyTarget = 0;
 161     m_default_target = 0;
 162     m_proxy_negotiation_charset = 0;
 163     m_proxy_negotiation_lang = 0;
 164     m_charset_converter = new Yaz_CharsetConverter;
 165     m_max_clients = 150;
 166     m_log_mask = 0;
 167     m_seed = time(0);
 168     m_client_idletime = 600;
 169     m_target_idletime = 600;
 170     m_optimize = xstrdup ("1");
 171     strcpy(m_session_str, "0 ");
 172     m_session_no = 0;
 173     m_bytes_sent = 0;
 174     m_bytes_recv = 0;
 175     m_bw_hold_PDU = 0;
 176     m_bw_max = 0;
 177     m_pdu_max = 0;
 178     m_max_record_retrieve = 0;
 179     m_reconfig_flag = 0;
 180     m_config_fname = 0;
 181     m_request_no = 0;
 182     m_invalid_session = 0;
 183     m_referenceId = 0;
 184     m_referenceId_mem = nmem_create();
 185     m_config = 0;
 186     m_marcxml_mode = none;
 187     m_stylesheet_xsp = 0;
 188     m_stylesheet_nprl = 0;
 189     m_s2z_stylesheet = 0;
 190     m_s2z_database = 0;
 191     m_schema = 0;
 192     m_backend_type = 0;
 193     m_backend_charset = 0;
 194     m_frontend_type = 0;
 195     m_initRequest_apdu = 0;
 196     m_initRequest_mem = 0;
 197     m_initRequest_preferredMessageSize = 0;
 198     m_initRequest_maximumRecordSize = 0;
 199     m_initRequest_options = 0;
 200     m_initRequest_version = 0;
 201     m_initRequest_oi_negotiation_charsets = 0;
 202     m_initRequest_oi_negotiation_num_charsets = 0;
 203     m_initRequest_oi_negotiation_langs = 0;
 204     m_initRequest_oi_negotiation_num_langs = 0;
 205     m_initRequest_oi_negotiation_selected = 0;
 206     m_apdu_invalid_session = 0;
 207     m_mem_invalid_session = 0;
 208     m_s2z_odr_init = 0;
 209     m_s2z_odr_search = 0;
 210     m_s2z_init_apdu = 0;
 211     m_s2z_search_apdu = 0;
 212     m_s2z_present_apdu = 0;
 213     m_http_keepalive = 0;
 214     m_http_version = 0;
 215     m_soap_ns = 0;
 216     m_s2z_packing = Z_SRW_recordPacking_string;
 217 #if HAVE_GETTIMEOFDAY
 218     m_time_tv = xmalloc(sizeof(struct timeval));
 219     struct timeval *tv = (struct timeval *) m_time_tv;
 220     tv->tv_sec = 0;
 221     tv->tv_usec = 0;
 222 #else
 223     m_time_tv = 0;
 224 #endif
 225     m_usemarcon_ini_stage1 = 0;
 226     m_usemarcon_ini_stage2 = 0;
 227     m_usemarcon = new Yaz_usemarcon();
 228     if (!m_parent)
 229         low_socket_open();
 230     m_my_thread = 0;
 231 }
 232
 233 Yaz_Proxy::~Yaz_Proxy()
 234 {
 235     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 236             m_bytes_sent, m_bytes_recv);
 237     nmem_destroy(m_initRequest_mem);
 238     nmem_destroy(m_mem_invalid_session);
 239     nmem_destroy(m_referenceId_mem);
 240
 241     xfree(m_proxyTarget);
 242     xfree(m_default_target);
 243     xfree(m_proxy_negotiation_charset);
 244     xfree(m_proxy_negotiation_lang);
 245     delete m_charset_converter;
 246     xfree(m_optimize);
 247
 248 #if HAVE_XSLT
 249     if (m_stylesheet_xsp)
 250         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 251 #endif
 252     xfree (m_time_tv);
 253
 254     xfree (m_schema);
 255     xfree (m_backend_type);
 256     xfree (m_backend_charset);
 257     xfree (m_usemarcon_ini_stage1);
 258     xfree (m_usemarcon_ini_stage2);
 259     delete m_usemarcon;
 260     if (m_s2z_odr_init)
 261         odr_destroy(m_s2z_odr_init);
 262     if (m_s2z_odr_search)
 263         odr_destroy(m_s2z_odr_search);
 264     if (!m_parent)
 265         low_socket_close();
 266     if (!m_parent)
 267         delete m_my_thread;
 268     delete m_config;
 269 }
 270
 271 void Yaz_Proxy::set_debug_mode(int mode)
 272 {
 273     m_debug_mode = mode;
 274 }
 275
 276 int Yaz_Proxy::set_config(const char *config)
 277 {
 278     delete m_config;
 279     m_config = new Yaz_ProxyConfig();
 280     xfree(m_config_fname);
 281     m_config_fname = xstrdup(config);
 282     int r = m_config->read_xml(config);
 283     if (!r)
 284         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 285     return r;
 286 }
 287
 288 void Yaz_Proxy::set_default_target(const char *target)
 289 {
 290     xfree (m_default_target);
 291     m_default_target = 0;
 292     if (target)
 293         m_default_target = (char *) xstrdup (target);
 294 }
 295
 296 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang)
 297 {
 298     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
 299         "language to '%s'", m_session_str, charset?charset:"none",
 300         lang?lang:"none");
 301     xfree (m_proxy_negotiation_charset);
 302     xfree (m_proxy_negotiation_lang);
 303     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 304     if (charset)
 305         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 306     if (lang)
 307         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 308 }
 309
 310 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 311 {
 312     if (m_parent)
 313         return m_parent->check_reconfigure();
 314
 315     Yaz_ProxyConfig *cfg = m_config;
 316     if (m_reconfig_flag)
 317     {
 318         yaz_log(YLOG_LOG, "reconfigure");
 319         yaz_log_reopen();
 320         if (m_config_fname && cfg)
 321         {
 322             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 323             int r = cfg->read_xml(m_config_fname);
 324             if (r)
 325                 yaz_log(YLOG_WARN, "reconfigure failed");
 326             else
 327             {
 328                 m_log_mask = 0;
 329                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 330             }
 331         }
 332         else
 333             yaz_log(YLOG_LOG, "reconfigure");
 334         m_reconfig_flag = 0;
 335     }
 336     return cfg;
 337 }
 338
 339 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 340                                             *the_PDU_Observable, int fd)
 341 {
 342     check_reconfigure();
 343     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
 344                                          m_socket_observable, this);
 345     new_proxy->m_config = 0;
 346     new_proxy->m_config_fname = 0;
 347     new_proxy->timeout(m_client_idletime);
 348     new_proxy->m_target_idletime = m_target_idletime;
 349     new_proxy->set_default_target(m_default_target);
 350     new_proxy->m_max_clients = m_max_clients;
 351     new_proxy->m_log_mask = m_log_mask;
 352     new_proxy->set_APDU_log(get_APDU_log());
 353     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 354         new_proxy->set_APDU_yazlog(1);
 355     else
 356         new_proxy->set_APDU_yazlog(0);
 357     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 358     m_session_no++;
 359     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 360              the_PDU_Observable->getpeername());
 361     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 362         m_proxy_negotiation_lang);
 363     // create thread object the first time we get an incoming connection
 364     if (!m_my_thread)
 365         m_my_thread = new Msg_Thread(m_socket_observable);
 366     new_proxy->m_my_thread = m_my_thread;
 367     return new_proxy;
 368 }
 369
 370 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 371 {
 372     int oid[OID_SIZE];
 373     Z_OtherInformationUnit *oi;
 374     struct oident ent;
 375     ent.proto = PROTO_Z3950;
 376     ent.oclass = CLASS_USERINFO;
 377     ent.value = (oid_value) VAL_COOKIE;
 378     assert (oid_ent_to_oid (&ent, oid));
 379
 380     if (oid_ent_to_oid (&ent, oid) &&
 381         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 382         oi->which == Z_OtherInfo_characterInfo)
 383         return oi->information.characterInfo;
 384     return 0;
 385 }
 386 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 387 {
 388     int oid[OID_SIZE];
 389     Z_OtherInformationUnit *oi;
 390     struct oident ent;
 391     ent.proto = PROTO_Z3950;
 392     ent.oclass = CLASS_USERINFO;
 393     ent.value = (oid_value) VAL_PROXY;
 394     if (oid_ent_to_oid (&ent, oid) &&
 395         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 396         oi->which == Z_OtherInfo_characterInfo)
 397         return oi->information.characterInfo;
 398     return 0;
 399 }
 400 const char *Yaz_Proxy::load_balance(const char **url)
 401 {
 402     int zurl_in_use[MAX_ZURL_PLEX];
 403     int zurl_in_spare[MAX_ZURL_PLEX];
 404     Yaz_ProxyClient *c;
 405     int i;
 406
 407     for (i = 0; i<MAX_ZURL_PLEX; i++)
 408     {
 409         zurl_in_use[i] = 0;
 410         zurl_in_spare[i] = 0;
 411     }
 412     for (c = m_parent->m_clientPool; c; c = c->m_next)
 413     {
 414         for (i = 0; url[i]; i++)
 415             if (!strcmp(url[i], c->get_hostname()))
 416             {
 417                 zurl_in_use[i]++;
 418                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 419                     zurl_in_spare[i]++;
 420             }
 421     }
 422     int min_use = 100000;
 423     int spare_for_min = 0;
 424     int max_spare = 0;
 425     const char *ret_min = 0;
 426     const char *ret_spare = 0;
 427     for (i = 0; url[i]; i++)
 428     {
 429         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 430                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 431         if (min_use > zurl_in_use[i])
 432         {
 433             ret_min = url[i];
 434             min_use = zurl_in_use[i];
 435             spare_for_min = zurl_in_spare[i];
 436         }
 437         if (max_spare < zurl_in_spare[i])
 438         {
 439             ret_spare = url[i];
 440             max_spare = zurl_in_spare[i];
 441         }
 442     }
 443     return ret_min;
 444 }
 445
 446 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 447                                        const char *proxy_host)
 448 {
 449     assert (m_parent);
 450     Yaz_Proxy *parent = m_parent;
 451     Yaz_ProxyClient *c = m_client;
 452
 453     if (!m_proxyTarget)
 454     {
 455         const char *url[MAX_ZURL_PLEX];
 456         Yaz_ProxyConfig *cfg = check_reconfigure();
 457         if (proxy_host)
 458         {
 459             if (parent && parent->m_debug_mode)
 460             {
 461                 // only to be enabled for debugging...
 462                 if (!strcmp(proxy_host, "stop"))
 463                     exit(0);
 464             }
 465             xfree(m_default_target);
 466             m_default_target = xstrdup(proxy_host);
 467         }
 468         proxy_host = m_default_target;
 469         int client_idletime = -1;
 470         const char *cql2rpn_fname = 0;
 471         const char *negotiation_charset = 0;
 472         const char *negotiation_lang = 0;
 473         const char *query_charset = 0;
 474         url[0] = m_default_target;
 475         url[1] = 0;
 476         if (cfg)
 477         {
 478             int pre_init = 0;
 479             cfg->get_target_info(proxy_host, url, &m_bw_max,
 480                                  &m_pdu_max, &m_max_record_retrieve,
 481                                  &m_target_idletime, &client_idletime,
 482                                  &parent->m_max_clients,
 483                                  &m_keepalive_limit_bw,
 484                                  &m_keepalive_limit_pdu,
 485                                  &pre_init,
 486                                  &cql2rpn_fname,
 487                                  &negotiation_charset,
 488                                  &negotiation_lang,
 489                                  &query_charset);
 490         }
 491         if (client_idletime != -1)
 492         {
 493             m_client_idletime = client_idletime;
 494             timeout(m_client_idletime);
 495         }
 496         if (cql2rpn_fname)
 497             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 498         if (negotiation_charset || negotiation_lang)
 499         {
 500             set_proxy_negotiation(negotiation_charset,
 501                 negotiation_lang);
 502         }
 503         m_charset_converter->set_target_query_charset(query_charset);
 504         if (!url[0])
 505         {
 506             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 507             return 0;
 508         }
 509         // we don't handle multiplexing for cookie session, so we just
 510         // pick the first one in this case (anonymous users will be able
 511         // to use any backend)
 512         if (cookie && *cookie)
 513             m_proxyTarget = (char*) xstrdup(url[0]);
 514         else
 515             m_proxyTarget = (char*) xstrdup(load_balance(url));
 516     }
 517     if (cookie && *cookie)
 518     {   // search in sessions with a cookie
 519         for (c = parent->m_clientPool; c; c = c->m_next)
 520         {
 521             assert (c->m_prev);
 522             assert (*c->m_prev == c);
 523             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 524                 !strcmp(m_proxyTarget, c->get_hostname()))
 525             {
 526                 // Found it in cache
 527                 // The following handles "cancel"
 528                 // If connection is busy (waiting for PDU) and
 529                 // we have an initRequest we can safely do re-open
 530                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 531                 {
 532                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 533                              c->get_hostname());
 534                     c->close();
 535                     c->m_init_flag = 0;
 536
 537                     c->m_last_ok = 0;
 538                     c->m_cache.clear();
 539                     c->m_last_resultCount = 0;
 540                     c->m_sr_transform = 0;
 541                     c->m_waiting = 0;
 542                     c->m_resultSetStartPoint = 0;
 543                     c->m_target_idletime = m_target_idletime;
 544                     if (c->client(m_proxyTarget))
 545                     {
 546                         delete c;
 547                         return 0;
 548                     }
 549                     c->timeout(30);
 550                 }
 551                 c->m_seqno = parent->m_seqno;
 552                 if (c->m_server && c->m_server != this)
 553                     c->m_server->m_client = 0;
 554                 c->m_server = this;
 555                 (parent->m_seqno)++;
 556                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 557                 return c;
 558             }
 559         }
 560     }
 561     else if (!c &&
 562             apdu->which == Z_APDU_initRequest &&
 563             apdu->u.initRequest->idAuthentication == 0 &&
 564             !ODR_MASK_GET(apdu->u.initRequest->options, Z_Options_negotiationModel))
 565     {
 566         // anonymous sessions without cookie.
 567         // if authentication is set it is NOT anonymous se we can't share them.
 568         // If charset and lang negotiation is use it is NOT anonymous session too.
 569         for (c = parent->m_clientPool; c; c = c->m_next)
 570         {
 571             assert(c->m_prev);
 572             assert(*c->m_prev == c);
 573             if (c->m_server == 0 && c->m_cookie == 0 &&
 574                 c->m_waiting == 0 &&
 575                 !strcmp(m_proxyTarget, c->get_hostname()))
 576             {
 577                 // found it in cache
 578                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 579                          m_session_str, parent->m_seqno, c->get_hostname());
 580
 581                 c->m_seqno = parent->m_seqno;
 582                 assert(c->m_server == 0);
 583                 c->m_server = this;
 584
 585                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 586                     c->set_APDU_yazlog(1);
 587                 else
 588                     c->set_APDU_yazlog(0);
 589
 590                 (parent->m_seqno)++;
 591
 592                 parent->pre_init();
 593
 594                 return c;
 595             }
 596         }
 597     }
 598     if (!m_client)
 599     {
 600         if (apdu->which != Z_APDU_initRequest)
 601         {
 602             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 603             return 0;
 604         }
 605         Z_InitRequest *initRequest = apdu->u.initRequest;
 606
 607         if (initRequest->idAuthentication)
 608         {
 609             // the client uses authentication. We set the keepalive PDU
 610             // to 0 so we don't cache it in releaseClient
 611             m_keepalive_limit_pdu = 0;
 612         }
 613         // go through list of clients - and find the lowest/oldest one.
 614         Yaz_ProxyClient *c_min = 0;
 615         int min_seq = -1;
 616         int no_of_clients = 0;
 617         if (parent->m_clientPool)
 618             yaz_log (YLOG_DEBUG, "Existing sessions");
 619         for (c = parent->m_clientPool; c; c = c->m_next)
 620         {
 621             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 622                                c->m_waiting, c->get_hostname(),
 623                                c->m_cookie ? c->m_cookie : "");
 624             no_of_clients++;
 625             if (min_seq < 0 || c->m_seqno < min_seq)
 626             {
 627                 min_seq = c->m_seqno;
 628                 c_min = c;
 629             }
 630         }
 631         if (no_of_clients >= parent->m_max_clients)
 632         {
 633             c = c_min;
 634             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 635             {
 636                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 637                          m_session_str, parent->m_max_clients, c->m_seqno);
 638                 if (c->m_server && c->m_server != this)
 639                     delete c->m_server;
 640                 c->m_server = 0;
 641             }
 642             else
 643             {
 644                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 645                          m_session_str, parent->m_max_clients,
 646                          c->m_seqno, parent->m_seqno, c->get_hostname());
 647                 xfree (c->m_cookie);
 648                 c->m_cookie = 0;
 649                 if (cookie)
 650                     c->m_cookie = xstrdup(cookie);
 651                 c->m_seqno = parent->m_seqno;
 652                 if (c->m_server && c->m_server != this)
 653                 {
 654                     c->m_server->m_client = 0;
 655                     delete c->m_server;
 656                 }
 657                 (parent->m_seqno)++;
 658                 c->m_target_idletime = m_target_idletime;
 659                 c->timeout(m_target_idletime);
 660
 661                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 662                     c->set_APDU_yazlog(1);
 663                 else
 664                     c->set_APDU_yazlog(0);
 665
 666                 return c;
 667             }
 668         }
 669         else
 670         {
 671             yaz_log (YLOG_LOG, "%sNEW %d %s",
 672                      m_session_str, parent->m_seqno, m_proxyTarget);
 673             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 674             c->m_next = parent->m_clientPool;
 675             if (c->m_next)
 676                 c->m_next->m_prev = &c->m_next;
 677             parent->m_clientPool = c;
 678             c->m_prev = &parent->m_clientPool;
 679         }
 680
 681         xfree (c->m_cookie);
 682         c->m_cookie = 0;
 683         if (cookie)
 684             c->m_cookie = xstrdup(cookie);
 685
 686         c->m_seqno = parent->m_seqno;
 687         c->m_init_flag = 0;
 688         c->m_last_resultCount = 0;
 689         c->m_last_ok = 0;
 690         c->m_cache.clear();
 691         c->m_sr_transform = 0;
 692         c->m_waiting = 0;
 693         c->m_resultSetStartPoint = 0;
 694         (parent->m_seqno)++;
 695         if (c->client(m_proxyTarget))
 696         {
 697             delete c;
 698             return 0;
 699         }
 700         c->m_target_idletime = m_target_idletime;
 701         c->timeout(30);
 702
 703         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 704             c->set_APDU_yazlog(1);
 705         else
 706             c->set_APDU_yazlog(0);
 707     }
 708     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 709     return c;
 710 }
 711
 712 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 713 {
 714     int i;
 715     for (i = 0; i<num; i++)
 716     {
 717         oident *ent;
 718         Z_DefaultDiagFormat *r;
 719         Z_DiagRec *p = pp[i];
 720         if (p->which != Z_DiagRec_defaultFormat)
 721         {
 722             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 723             return;
 724         }
 725         else
 726             r = p->u.defaultFormat;
 727         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 728             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 729             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 730         switch (r->which)
 731         {
 732         case Z_DefaultDiagFormat_v2Addinfo:
 733             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 734                     m_session_str,
 735                     *r->condition, diagbib1_str(*r->condition),
 736                     r->u.v2Addinfo);
 737             break;
 738         case Z_DefaultDiagFormat_v3Addinfo:
 739             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 740                     m_session_str,
 741                     *r->condition, diagbib1_str(*r->condition),
 742                     r->u.v3Addinfo);
 743             break;
 744         }
 745     }
 746 }
 747
 748 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 749 {
 750     if (!m_stylesheet_xsp || p->num_records <= 0)
 751         return 0;  /* no XSLT to be done ... */
 752
 753     m_stylesheet_offset = 0;
 754     m_stylesheet_nprl = p;
 755     m_stylesheet_apdu = apdu;
 756     timeout(0);
 757     return 1;
 758 }
 759
 760 void Yaz_Proxy::convert_xsl_delay()
 761 {
 762 #if HAVE_XSLT
 763     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 764     if (npr->which == Z_NamePlusRecord_databaseRecord)
 765     {
 766         Z_External *r = npr->u.databaseRecord;
 767         if (r->which == Z_External_octet)
 768         {
 769 #if 0
 770             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 771 #endif
 772             xmlDocPtr res, doc = xmlParseMemory(
 773                 (char*) r->u.octet_aligned->buf,
 774                 r->u.octet_aligned->len);
 775
 776
 777             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 778                     m_session_str, m_stylesheet_offset);
 779             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 780                                       doc, 0);
 781
 782             if (res)
 783             {
 784                 xmlChar *out_buf;
 785                 int out_len;
 786                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 787
 788                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 789                     u.databaseRecord =
 790                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 791                                  (char*) out_buf, out_len);
 792                 xmlFree(out_buf);
 793                 xmlFreeDoc(res);
 794             }
 795
 796             xmlFreeDoc(doc);
 797         }
 798     }
 799 #endif
 800     m_stylesheet_offset++;
 801     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 802     {
 803         m_stylesheet_nprl = 0;
 804 #if HAVE_XSLT
 805         if (m_stylesheet_xsp)
 806             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 807 #endif
 808         m_stylesheet_xsp = 0;
 809         timeout(m_client_idletime);
 810         send_PDU_convert(m_stylesheet_apdu);
 811     }
 812     else
 813         timeout(0);
 814 }
 815
 816 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 817 {
 818     if (m_frontend_type != VAL_NONE)
 819     {
 820         int i;
 821         for (i = 0; i < p->num_records; i++)
 822         {
 823             Z_NamePlusRecord *npr = p->records[i];
 824             if (npr->which == Z_NamePlusRecord_databaseRecord)
 825             {
 826                 Z_External *r = npr->u.databaseRecord;
 827                 if (r->which == Z_External_octet)
 828                 {
 829 #if HAVE_USEMARCON
 830                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 831                     {
 832                         if (!m_usemarcon->m_stage1)
 833                         {
 834                             m_usemarcon->m_stage1 = new CDetails();
 835                         }
 836                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 837                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 838                         int res = m_usemarcon->m_stage1->Start();
 839                         if (res == 0)
 840                         {
 841                             char *converted;
 842                             int convlen;
 843                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 844                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 845                             {
 846                                 if (!m_usemarcon->m_stage2)
 847                                 {
 848                                     m_usemarcon->m_stage2 = new CDetails();
 849                                 }
 850                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 851                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 852                                 res = m_usemarcon->m_stage2->Start();
 853                                 if (res == 0)
 854                                 {
 855                                     free(converted);
 856                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 857                                 }
 858                                 else
 859                                 {
 860                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 861                                 }
 862                             }
 863                             npr->u.databaseRecord =
 864                                 z_ext_record(odr_encode(),
 865                                              m_frontend_type,
 866                                              converted,
 867                                              strlen(converted));
 868                             free(converted);
 869                         }
 870                         else
 871                         {
 872                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 873                         }
 874                         continue;
 875                     }
 876 #endif
 877 /* HAVE_USEMARCON */
 878                     npr->u.databaseRecord =
 879                         z_ext_record(odr_encode(),
 880                                      m_frontend_type,
 881                                      (char*) r->u.octet_aligned->buf,
 882                                      r->u.octet_aligned->len);
 883                 }
 884             }
 885         }
 886     }
 887 }
 888
 889 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
 890                                         const char *backend_charset)
 891 {
 892     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
 893     int sel =   m_charset_converter->get_client_charset_selected();
 894     const char *client_record_charset =
 895         m_charset_converter->get_client_query_charset();
 896     if (sel && backend_charset && client_record_charset &&
 897         strcmp(backend_charset, client_record_charset))
 898     {
 899         int i;
 900         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
 901                                         backend_charset);
 902         yaz_marc_t mt = yaz_marc_create();
 903         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
 904         yaz_marc_iconv(mt, cd);
 905         for (i = 0; i < p->num_records; i++)
 906         {
 907             Z_NamePlusRecord *npr = p->records[i];
 908             if (npr->which == Z_NamePlusRecord_databaseRecord)
 909             {
 910                 Z_External *r = npr->u.databaseRecord;
 911                 oident *ent = oid_getentbyoid(r->direct_reference);
 912                 if (!ent || ent->value == VAL_NONE)
 913                     continue;
 914
 915                 if (ent->value == VAL_SUTRS)
 916                 {
 917                     WRBUF w = wrbuf_alloc();
 918
 919                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
 920                                       r->u.octet_aligned->len);
 921                     npr->u.databaseRecord =
 922                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
 923                                      wrbuf_len(w));
 924                     wrbuf_free(w, 1);
 925                 }
 926                 else if (ent->value == VAL_TEXT_XML)
 927                 {
 928                     ;
 929                 }
 930                 else if (r->which == Z_External_octet)
 931                 {
 932                     int rlen;
 933                     char *result;
 934                     if (yaz_marc_decode_buf(mt,
 935                                             (char*) r->u.octet_aligned->buf,
 936                                             r->u.octet_aligned->len,
 937                                             &result, &rlen))
 938                     {
 939                         npr->u.databaseRecord =
 940                             z_ext_record(odr_encode(), ent->value, result, rlen);
 941                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
 942                                 m_session_str);
 943                     }
 944                 }
 945             }
 946         }
 947         if (cd)
 948             yaz_iconv_close(cd);
 949         yaz_marc_destroy(mt);
 950     }
 951     else
 952     {
 953         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
 954     }
 955 }
 956
 957 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
 958                                    const char *backend_charset)
 959 {
 960     int i;
 961     if (!backend_charset)
 962         backend_charset = "MARC-8";
 963     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
 964     yaz_marc_t mt = yaz_marc_create();
 965     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 966     yaz_marc_iconv(mt, cd);
 967     for (i = 0; i < p->num_records; i++)
 968     {
 969         Z_NamePlusRecord *npr = p->records[i];
 970         if (npr->which == Z_NamePlusRecord_databaseRecord)
 971         {
 972             Z_External *r = npr->u.databaseRecord;
 973             if (r->which == Z_External_OPAC)
 974             {
 975                 WRBUF w = wrbuf_alloc();
 976
 977                 yaz_display_OPAC(w, r->u.opac, 0);
 978                 npr->u.databaseRecord = z_ext_record(
 979                     odr_encode(), VAL_TEXT_XML,
 980                     wrbuf_buf(w), wrbuf_len(w)
 981                     );
 982                 wrbuf_free(w, 1);
 983             }
 984             else if (r->which == Z_External_octet)
 985             {
 986                 int rlen;
 987                 char *result;
 988                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 989                                         r->u.octet_aligned->len,
 990                                         &result, &rlen))
 991                 {
 992                     npr->u.databaseRecord =
 993                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
 994                 }
 995             }
 996         }
 997     }
 998     if (cd)
 999         yaz_iconv_close(cd);
1000     yaz_marc_destroy(mt);
1001 }
1002
1003 void Yaz_Proxy::logtime()
1004 {
1005 #if HAVE_GETTIMEOFDAY
1006     struct timeval *tv = (struct timeval*) m_time_tv;
1007     if (tv->tv_sec)
1008     {
1009         struct timeval tv1;
1010         gettimeofday(&tv1, 0);
1011         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1012             (tv1.tv_usec - tv->tv_usec);
1013         if (diff >= 0)
1014             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1015                     diff/1000000, (diff/1000)%1000);
1016     }
1017     tv->tv_sec = 0;
1018     tv->tv_usec = 0;
1019 #endif
1020 }
1021
1022 int Yaz_Proxy::send_http_response(int code)
1023 {
1024     ODR o = odr_encode();
1025     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1026     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1027     if (m_http_version)
1028         hres->version = odr_strdup(o, m_http_version);
1029     if (m_http_keepalive)
1030         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1031     else
1032         timeout(0);
1033
1034     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1035     {
1036         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1037                  gdu_name(gdu));
1038     }
1039     int len;
1040     int r = send_GDU(gdu, &len);
1041     m_bytes_sent += len;
1042     m_bw_stat.add_bytes(len);
1043     logtime();
1044     return r;
1045 }
1046
1047 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
1048 {
1049     ODR o = odr_encode();
1050     const char *ctype = "text/xml";
1051     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
1052     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1053     if (m_http_version)
1054         hres->version = odr_strdup(o, m_http_version);
1055     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1056     if (m_http_keepalive)
1057         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1058     else
1059         timeout(0);
1060
1061     static Z_SOAP_Handler soap_handlers[2] = {
1062 #if HAVE_XSLT
1063         {"http://www.loc.gov/zing/srw/", 0,
1064          (Z_SOAP_fun) yaz_srw_codec},
1065 #endif
1066         {0, 0, 0}
1067     };
1068
1069     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1070     soap_package->which = Z_SOAP_generic;
1071     soap_package->u.generic =
1072         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1073     soap_package->u.generic->no = 0;
1074     soap_package->u.generic->ns = soap_handlers[0].ns;
1075     soap_package->u.generic->p = (void *) srw_pdu;
1076     soap_package->ns = m_soap_ns;
1077     z_soap_codec_enc_xsl(o, &soap_package,
1078                          &hres->content_buf, &hres->content_len,
1079                          soap_handlers, 0, m_s2z_stylesheet);
1080     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1081     {
1082         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1083                  gdu_name(gdu));
1084     }
1085     int len;
1086     int r = send_GDU(gdu, &len);
1087     m_bytes_sent += len;
1088     m_bw_stat.add_bytes(len);
1089     logtime();
1090     return r;
1091 }
1092
1093 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1094 {
1095     ODR o = odr_encode();
1096     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1097     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1098
1099     srw_res->num_diagnostics = 1;
1100     srw_res->diagnostics = (Z_SRW_diagnostic *)
1101         odr_malloc(o, sizeof(*srw_res->diagnostics));
1102     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
1103     return send_srw_response(srw_pdu);
1104 }
1105
1106 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1107                              Z_DefaultDiagFormat *ddf)
1108 {
1109     int bib1_code = *ddf->condition;
1110     if (bib1_code == 109)
1111         return 404;
1112     srw_res->num_diagnostics = 1;
1113     srw_res->diagnostics = (Z_SRW_diagnostic *)
1114         odr_malloc(o, sizeof(*srw_res->diagnostics));
1115     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1116                           yaz_diag_bib1_to_srw(*ddf->condition),
1117                           ddf->u.v2Addinfo);
1118     return 0;
1119 }
1120
1121 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1122 {
1123     ODR o = odr_encode();
1124     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1125     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1126
1127     srw_res->numberOfRecords = odr_intdup (o, hits);
1128     if (records && records->which == Z_Records_DBOSD)
1129     {
1130         srw_res->num_records =
1131             records->u.databaseOrSurDiagnostics->num_records;
1132         int i;
1133         srw_res->records = (Z_SRW_record *)
1134             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1135         for (i = 0; i < srw_res->num_records; i++)
1136         {
1137             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1138             if (npr->which != Z_NamePlusRecord_databaseRecord)
1139             {
1140                 srw_res->records[i].recordSchema = "diagnostic";
1141                 srw_res->records[i].recordPacking = m_s2z_packing;
1142                 srw_res->records[i].recordData_buf = "67";
1143                 srw_res->records[i].recordData_len = 2;
1144                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1145                 continue;
1146             }
1147             Z_External *r = npr->u.databaseRecord;
1148             oident *ent = oid_getentbyoid(r->direct_reference);
1149             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1150             {
1151                 srw_res->records[i].recordSchema = m_schema;
1152                 srw_res->records[i].recordPacking = m_s2z_packing;
1153                 srw_res->records[i].recordData_buf = (char*)
1154                     r->u.octet_aligned->buf;
1155                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1156                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1157             }
1158             else
1159             {
1160                 srw_res->records[i].recordSchema = "diagnostic";
1161                 srw_res->records[i].recordPacking = m_s2z_packing;
1162                 srw_res->records[i].recordData_buf = "67";
1163                 srw_res->records[i].recordData_len = 2;
1164                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1165             }
1166         }
1167     }
1168     if (records && records->which == Z_Records_NSD)
1169     {
1170         int http_code;
1171         http_code = z_to_srw_diag(odr_encode(), srw_res,
1172                                    records->u.nonSurrogateDiagnostic);
1173         if (http_code)
1174             return send_http_response(http_code);
1175     }
1176     return send_srw_response(srw_pdu);
1177
1178 }
1179
1180 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1181                                         int num_diagnostics)
1182 {
1183     Yaz_ProxyConfig *cfg = check_reconfigure();
1184     if (cfg)
1185     {
1186         int len;
1187         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1188                                        m_s2z_database, &len);
1189         if (b)
1190         {
1191             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1192             Z_SRW_explainResponse *er = res->u.explain_response;
1193
1194             er->record.recordData_buf = b;
1195             er->record.recordData_len = len;
1196             er->record.recordPacking = m_s2z_packing;
1197             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1198
1199             er->diagnostics = diagnostics;
1200             er->num_diagnostics = num_diagnostics;
1201             return send_srw_response(res);
1202         }
1203     }
1204     return send_http_response(404);
1205 }
1206
1207 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1208 {
1209     if (m_http_version)
1210     {
1211         if (apdu->which == Z_APDU_initResponse)
1212         {
1213             Z_InitResponse *res = apdu->u.initResponse;
1214             if (*res->result == 0)
1215             {
1216                 send_to_srw_client_error(3, 0);
1217             }
1218             else if (!m_s2z_search_apdu)
1219             {
1220                 send_srw_explain_response(0, 0);
1221             }
1222             else
1223             {
1224                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1225             }
1226         }
1227         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1228         {
1229             m_s2z_search_apdu = 0;
1230             Z_SearchResponse *res = apdu->u.searchResponse;
1231             m_s2z_hit_count = *res->resultCount;
1232             if (res->records && res->records->which == Z_Records_NSD)
1233             {
1234                 send_to_srw_client_ok(0, res->records, 1);
1235             }
1236             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1237             {
1238                 // adjust
1239                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1240
1241                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1242                 {
1243                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1244                         > m_s2z_hit_count)
1245                         *pr->numberOfRecordsRequested =
1246                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1247                 }
1248                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1249             }
1250             else
1251             {
1252                 m_s2z_present_apdu = 0;
1253                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1254             }
1255         }
1256         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1257         {
1258             int start =
1259                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1260
1261             m_s2z_present_apdu = 0;
1262             Z_PresentResponse *res = apdu->u.presentResponse;
1263             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1264         }
1265     }
1266     else
1267     {
1268         int len = 0;
1269         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1270             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1271                      apdu_name(apdu));
1272         int r = send_Z_PDU(apdu, &len);
1273         m_bytes_sent += len;
1274         m_bw_stat.add_bytes(len);
1275         logtime();
1276         return r;
1277     }
1278     return 0;
1279 }
1280
1281 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1282 {
1283     int kill_session = 0;
1284     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1285
1286     if (new_id)
1287         *new_id = m_referenceId;
1288
1289     if (apdu->which == Z_APDU_searchResponse)
1290     {
1291         Z_SearchResponse *sr = apdu->u.searchResponse;
1292         Z_Records *p = sr->records;
1293         if (p && p->which == Z_Records_NSD)
1294         {
1295             Z_DiagRec dr, *dr_p = &dr;
1296             dr.which = Z_DiagRec_defaultFormat;
1297             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1298
1299             *sr->searchStatus = 0;
1300             display_diagrecs(&dr_p, 1);
1301         }
1302         else
1303         {
1304             if (p && p->which == Z_Records_DBOSD)
1305             {
1306                 if (m_backend_type
1307 #if HAVE_USEMARCON
1308                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1309 #endif
1310                     )
1311                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1312                 if (m_marcxml_mode == marcxml)
1313                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1314                                        m_backend_charset);
1315                 else
1316                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1317                                             m_backend_charset);
1318                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1319                     return 0;
1320
1321             }
1322             if (sr->resultCount)
1323             {
1324                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1325                         *sr->resultCount);
1326                 if (*sr->resultCount < 0)
1327                 {
1328                     m_invalid_session = 1;
1329                     kill_session = 1;
1330
1331                     *sr->searchStatus = 0;
1332                     sr->records =
1333                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1334                     *sr->resultCount = 0;
1335                 }
1336             }
1337         }
1338     }
1339     else if (apdu->which == Z_APDU_presentResponse)
1340     {
1341         Z_PresentResponse *sr = apdu->u.presentResponse;
1342         Z_Records *p = sr->records;
1343         if (p && p->which == Z_Records_NSD)
1344         {
1345             Z_DiagRec dr, *dr_p = &dr;
1346             dr.which = Z_DiagRec_defaultFormat;
1347             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1348             if (*sr->presentStatus == Z_PresentStatus_success)
1349                 *sr->presentStatus = Z_PresentStatus_failure;
1350             display_diagrecs(&dr_p, 1);
1351         }
1352         if (p && p->which == Z_Records_DBOSD)
1353         {
1354             if (m_backend_type
1355 #if HAVE_USEMARCON
1356                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1357 #endif
1358                 )
1359                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1360             if (m_marcxml_mode == marcxml)
1361                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1362                                    m_backend_charset);
1363             else
1364                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1365                                         m_backend_charset);
1366             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1367                 return 0;
1368         }
1369     }
1370     else if (apdu->which == Z_APDU_initResponse)
1371     {
1372         //Get and check negotiation record
1373         //from init response.
1374         handle_charset_lang_negotiation(apdu);
1375
1376         if (m_initRequest_options)
1377         {
1378             Z_Options *nopt =
1379                 (Odr_bitmask *)odr_malloc(odr_encode(),
1380                                           sizeof(Odr_bitmask));
1381             ODR_MASK_ZERO(nopt);
1382
1383             int i;
1384             for (i = 0; i<24; i++)
1385                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1386                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1387                     ODR_MASK_SET(nopt, i);
1388             apdu->u.initResponse->options = nopt;
1389         }
1390         if (m_initRequest_version)
1391         {
1392             Z_ProtocolVersion *nopt =
1393                 (Odr_bitmask *)odr_malloc(odr_encode(),
1394                                           sizeof(Odr_bitmask));
1395             ODR_MASK_ZERO(nopt);
1396
1397             int i;
1398             for (i = 0; i<8; i++)
1399                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1400                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1401                     ODR_MASK_SET(nopt, i);
1402             apdu->u.initResponse->protocolVersion = nopt;
1403         }
1404         apdu->u.initResponse->preferredMessageSize =
1405             odr_intdup(odr_encode(),
1406                        m_client->m_initResponse_preferredMessageSize >
1407                        m_initRequest_preferredMessageSize ?
1408                        m_initRequest_preferredMessageSize :
1409                        m_client->m_initResponse_preferredMessageSize);
1410         apdu->u.initResponse->maximumRecordSize =
1411             odr_intdup(odr_encode(),
1412                        m_client->m_initResponse_maximumRecordSize >
1413                        m_initRequest_maximumRecordSize ?
1414                        m_initRequest_maximumRecordSize :
1415                        m_client->m_initResponse_maximumRecordSize);
1416     }
1417
1418     int r = send_PDU_convert(apdu);
1419     if (r)
1420         return r;
1421     if (kill_session)
1422     {
1423         delete m_client;
1424         m_client = 0;
1425         m_parent->pre_init();
1426     }
1427     return r;
1428 }
1429
1430 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1431 {
1432     int len = 0;
1433     const char *apdu_name_tmp = apdu_name(apdu);
1434     int r = send_Z_PDU(apdu, &len);
1435     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1436         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1437                  get_session_str(),
1438                  apdu_name_tmp, get_hostname(), len);
1439     m_bytes_sent += len;
1440     return r;
1441 }
1442
1443 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1444 {
1445     if (apdu->which == Z_APDU_presentRequest)
1446     {
1447         Z_PresentRequest *pr = apdu->u.presentRequest;
1448         int toget = *pr->numberOfRecordsRequested;
1449         int start = *pr->resultSetStartPoint;
1450
1451         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1452                 pr->resultSetId, start, toget);
1453
1454         if (*m_parent->m_optimize == '0')
1455             return apdu;
1456
1457         if (!m_client->m_last_resultSetId)
1458         {
1459             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1460             new_apdu->u.presentResponse->records =
1461                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1462                                                pr->resultSetId);
1463             send_to_client(new_apdu);
1464             return 0;
1465         }
1466         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1467         {
1468             if (start+toget-1 > m_client->m_last_resultCount)
1469             {
1470                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1471                 new_apdu->u.presentResponse->records =
1472                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1473                 send_to_client(new_apdu);
1474                 return 0;
1475             }
1476             Z_NamePlusRecordList *npr;
1477 #if 0
1478             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1479                     m_session_str, start, toget, yaz_z3950oid_to_str(
1480                         pr->preferredRecordSyntax, &oclass));
1481 #endif
1482             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1483                                           pr->preferredRecordSyntax,
1484                                           pr->recordComposition))
1485             {
1486                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1487                          m_session_str);
1488                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1489                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1490
1491                 new_apdu->u.presentResponse->numberOfRecordsReturned
1492                     = odr_intdup(odr_encode(), toget);
1493
1494                 new_apdu->u.presentResponse->records = (Z_Records*)
1495                     odr_malloc(odr_encode(), sizeof(Z_Records));
1496                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1497                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1498                 new_apdu->u.presentResponse->nextResultSetPosition =
1499                     odr_intdup(odr_encode(), start+toget);
1500
1501                 send_to_client(new_apdu);
1502                 return 0;
1503             }
1504         }
1505     }
1506
1507     if (apdu->which != Z_APDU_searchRequest)
1508         return apdu;
1509     Z_SearchRequest *sr = apdu->u.searchRequest;
1510     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1511     Yaz_Z_Databases this_databases;
1512
1513     this_databases.set(sr->num_databaseNames, (const char **)
1514                        sr->databaseNames);
1515
1516     this_query->set_Z_Query(sr->query);
1517
1518     char query_str[120];
1519     this_query->print(query_str, sizeof(query_str)-1);
1520     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1521
1522     if (*m_parent->m_optimize != '0' &&
1523         m_client->m_last_ok && m_client->m_last_query &&
1524         m_client->m_last_query->match(this_query) &&
1525         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1526         m_client->m_last_databases.match(this_databases))
1527     {
1528         delete this_query;
1529         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1530             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1531         {
1532             Z_NamePlusRecordList *npr;
1533             int toget = *sr->mediumSetPresentNumber;
1534             Z_RecordComposition *comp = 0;
1535
1536             if (toget > m_client->m_last_resultCount)
1537                 toget = m_client->m_last_resultCount;
1538
1539             if (sr->mediumSetElementSetNames)
1540             {
1541                 comp = (Z_RecordComposition *)
1542                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1543                 comp->which = Z_RecordComp_simple;
1544                 comp->u.simple = sr->mediumSetElementSetNames;
1545             }
1546
1547             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1548                                           sr->preferredRecordSyntax, comp))
1549             {
1550                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1551                          m_session_str);
1552                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1553                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1554                 new_apdu->u.searchResponse->resultCount =
1555                     &m_client->m_last_resultCount;
1556
1557                 new_apdu->u.searchResponse->numberOfRecordsReturned
1558                     = odr_intdup(odr_encode(), toget);
1559
1560                 new_apdu->u.searchResponse->presentStatus =
1561                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1562                 new_apdu->u.searchResponse->records = (Z_Records*)
1563                     odr_malloc(odr_encode(), sizeof(Z_Records));
1564                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1565                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1566                 new_apdu->u.searchResponse->nextResultSetPosition =
1567                     odr_intdup(odr_encode(), toget+1);
1568                 send_to_client(new_apdu);
1569                 return 0;
1570             }
1571             else
1572             {
1573                 // medium Set
1574                 // send present request (medium size)
1575                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1576                          m_session_str);
1577
1578                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1579                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1580                 pr->referenceId = sr->referenceId;
1581                 pr->resultSetId = sr->resultSetName;
1582                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1583                 *pr->numberOfRecordsRequested = toget;
1584                 pr->recordComposition = comp;
1585                 m_client->m_sr_transform = 1;
1586                 return new_apdu;
1587             }
1588         }
1589         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1590             m_client->m_last_resultCount <= 0)
1591         {
1592             // large set. Return pseudo-search response immediately
1593             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1594                      m_session_str);
1595             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1596             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1597             new_apdu->u.searchResponse->resultCount =
1598                 &m_client->m_last_resultCount;
1599             send_to_client(new_apdu);
1600             return 0;
1601         }
1602         else
1603         {
1604             Z_NamePlusRecordList *npr;
1605             int toget = m_client->m_last_resultCount;
1606             Z_RecordComposition *comp = 0;
1607             // small set
1608             // send a present request (small set)
1609
1610             if (sr->smallSetElementSetNames)
1611             {
1612                 comp = (Z_RecordComposition *)
1613                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1614                 comp->which = Z_RecordComp_simple;
1615                 comp->u.simple = sr->smallSetElementSetNames;
1616             }
1617
1618             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1619                                           sr->preferredRecordSyntax, comp))
1620             {
1621                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1622                          m_session_str);
1623                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1624                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1625                 new_apdu->u.searchResponse->resultCount =
1626                     &m_client->m_last_resultCount;
1627
1628                 new_apdu->u.searchResponse->numberOfRecordsReturned
1629                     = odr_intdup(odr_encode(), toget);
1630
1631                 new_apdu->u.searchResponse->presentStatus =
1632                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1633                 new_apdu->u.searchResponse->records = (Z_Records*)
1634                     odr_malloc(odr_encode(), sizeof(Z_Records));
1635                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1636                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1637                 new_apdu->u.searchResponse->nextResultSetPosition =
1638                     odr_intdup(odr_encode(), toget+1);
1639                 send_to_client(new_apdu);
1640                 return 0;
1641             }
1642             else
1643             {
1644                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1645                          m_session_str);
1646                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1647                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1648                 pr->referenceId = sr->referenceId;
1649                 pr->resultSetId = sr->resultSetName;
1650                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1651                 *pr->numberOfRecordsRequested = toget;
1652                 pr->recordComposition = comp;
1653                 m_client->m_sr_transform = 1;
1654                 return new_apdu;
1655             }
1656         }
1657     }
1658     else  // query doesn't match
1659     {
1660         delete m_client->m_last_query;
1661         m_client->m_last_query = this_query;
1662         m_client->m_last_ok = 0;
1663         m_client->m_cache.clear();
1664         m_client->m_resultSetStartPoint = 0;
1665
1666         xfree (m_client->m_last_resultSetId);
1667         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1668
1669         m_client->m_last_databases.set(sr->num_databaseNames,
1670                                        (const char **) sr->databaseNames);
1671     }
1672     return apdu;
1673 }
1674
1675
1676 void Yaz_Proxy::inc_request_no()
1677 {
1678     char *cp = strchr(m_session_str, ' ');
1679     m_request_no++;
1680     if (cp)
1681         sprintf(cp+1, "%d ", m_request_no);
1682 }
1683
1684 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1685 {
1686     inc_request_no();
1687
1688     m_bytes_recv += len;
1689
1690     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1691         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1692                  m_session_str, gdu_name(apdu), len);
1693
1694     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1695         shutdown();
1696
1697     m_bw_stat.add_bytes(len);
1698     m_pdu_stat.add_bytes(1);
1699
1700 #if HAVE_GETTIMEOFDAY
1701     gettimeofday((struct timeval *) m_time_tv, 0);
1702 #endif
1703
1704     int bw_total = m_bw_stat.get_total();
1705     int pdu_total = m_pdu_stat.get_total();
1706
1707     int reduce = 0;
1708     if (m_bw_max)
1709     {
1710         if (bw_total > m_bw_max)
1711         {
1712             reduce = (bw_total/m_bw_max);
1713         }
1714     }
1715     if (m_pdu_max)
1716     {
1717         if (pdu_total > m_pdu_max)
1718         {
1719             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1720             reduce = (reduce > nreduce) ? reduce : nreduce;
1721         }
1722     }
1723     m_http_version = 0;
1724     if (reduce)
1725     {
1726         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1727                 m_session_str, reduce, bw_total, pdu_total,
1728                 m_bw_max, m_pdu_max);
1729
1730         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1731         timeout(reduce);       // call us reduce seconds later
1732     }
1733     else if (apdu->which == Z_GDU_Z3950)
1734         handle_incoming_Z_PDU(apdu->u.z3950);
1735     else if (apdu->which == Z_GDU_HTTP_Request)
1736         handle_incoming_HTTP(apdu->u.HTTP_Request);
1737 }
1738
1739 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1740 {
1741     if (m_max_record_retrieve)
1742     {
1743         if (apdu->which == Z_APDU_presentRequest)
1744         {
1745             Z_PresentRequest *pr = apdu->u.presentRequest;
1746             if (pr->numberOfRecordsRequested &&
1747                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1748                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1749         }
1750     }
1751 }
1752 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1753 {
1754     if (apdu->which == Z_APDU_initRequest)
1755     {
1756         yaz_log(YLOG_LOG, "%shandle_charset_lang_negotiation",
1757                 m_session_str);
1758         if (m_initRequest_options &&
1759             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1760             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1761         {
1762             // There is no negotiation proposal from
1763             // client's side. OK. The proxy negotiation
1764             // in use, only.
1765             Z_InitRequest *initRequest = apdu->u.initRequest;
1766             Z_OtherInformation **otherInfo;
1767             Z_OtherInformationUnit *oi;
1768             get_otherInfoAPDU(apdu, &otherInfo);
1769             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1770             if (oi)
1771             {
1772                 ODR_MASK_SET(initRequest->options,
1773                     Z_Options_negotiationModel);
1774                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1775                 oi->information.externallyDefinedInfo =
1776                 yaz_set_proposal_charneg(odr_encode(),
1777                     (const char**)&m_proxy_negotiation_charset,
1778                     m_proxy_negotiation_charset ? 1:0,
1779                     (const char**)&m_proxy_negotiation_lang,
1780                     m_proxy_negotiation_lang ? 1:0,
1781                     1);
1782             }
1783         }
1784         else if (m_initRequest_options &&
1785                  ODR_MASK_GET(m_initRequest_options,
1786                               Z_Options_negotiationModel) &&
1787                  m_charset_converter->get_target_query_charset())
1788         {
1789             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1790                     m_session_str,
1791                     m_charset_converter->get_target_query_charset());
1792             Z_InitRequest *initRequest = apdu->u.initRequest;
1793             Z_CharSetandLanguageNegotiation *negotiation =
1794                 yaz_get_charneg_record (initRequest->otherInfo);
1795             if (negotiation &&
1796                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1797             {
1798                 NMEM nmem = nmem_create();
1799                 char **charsets = 0;
1800                 int num_charsets = 0;
1801                 char **langs = 0;
1802                 int num_langs = 0;
1803                 int selected = 0;
1804                 yaz_get_proposal_charneg (nmem, negotiation,
1805                                           &charsets, &num_charsets,
1806                                           &langs, &num_langs, &selected);
1807                 int i;
1808                 for (i = 0; i<num_charsets; i++)
1809                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
1810                             charsets[i]);
1811                 for (i = 0; i<num_langs; i++)
1812                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
1813                             langs[i]);
1814
1815                 const char *t_charset =
1816                     m_charset_converter->get_target_query_charset();
1817                 // sweep through charsets and pick the first supported
1818                 // conversion
1819                 for (i = 0; i<num_charsets; i++)
1820                 {
1821                     const char *c_charset = charsets[i];
1822                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
1823                         break;
1824                 }
1825                 if (i != num_charsets)
1826                 {
1827                     // got one .. set up ODR for reverse direction
1828                     const char *c_charset = charsets[i];
1829                     odr_set_charset(odr_encode(), c_charset, t_charset);
1830                     m_charset_converter->set_client_query_charset(c_charset);
1831                     m_charset_converter->set_client_charset_selected(selected);
1832                 }
1833                 nmem_destroy(nmem);
1834                 ODR_MASK_CLEAR(m_initRequest_options,
1835                                Z_Options_negotiationModel);
1836                 yaz_del_charneg_record(&initRequest->otherInfo);
1837             }
1838             else
1839             {
1840                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
1841                         m_session_str);
1842             }
1843         }
1844     }
1845     else if (apdu->which == Z_APDU_initResponse)
1846     {
1847         Z_InitResponse *initResponse = apdu->u.initResponse;
1848         Z_OtherInformation **otherInfo;
1849
1850         if (ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel))
1851         {
1852             char *charset = 0;
1853             char *lang = 0;
1854             int selected = 0;
1855
1856             get_otherInfoAPDU(apdu, &otherInfo);
1857
1858             if (!otherInfo && !(*otherInfo))
1859                 return;
1860
1861             Z_CharSetandLanguageNegotiation *charneg =
1862                 yaz_get_charneg_record(*otherInfo);
1863
1864             if (!charneg)
1865                 return;
1866
1867             yaz_get_response_charneg(m_referenceId_mem, charneg,
1868                 &charset, &lang, &selected);
1869
1870             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
1871                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
1872
1873             if (m_initRequest_options &&
1874                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
1875             {
1876                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
1877                     m_session_str);
1878             }
1879             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1880             {
1881                 // negotiation-charset, negotiation-lang
1882                 // elements of config file in use.
1883
1884                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
1885                     m_session_str);
1886
1887                 // clear negotiation option.
1888                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
1889
1890                 // Delete negotiation (charneg-3) entry.
1891                 yaz_del_charneg_record(otherInfo);
1892             }
1893         }
1894         else
1895         {
1896             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1897             {
1898                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
1899                         m_session_str);
1900             }
1901             else if (m_charset_converter->get_client_query_charset())
1902             {
1903                 Z_OtherInformation **otherInfo;
1904                 Z_OtherInformationUnit *oi;
1905                 get_otherInfoAPDU(apdu, &otherInfo);
1906                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1907                 if (oi)
1908                 {
1909                     ODR_MASK_SET(initResponse->options,
1910                                  Z_Options_negotiationModel);
1911                     ODR_MASK_SET(m_initRequest_options,
1912                                  Z_Options_negotiationModel);
1913
1914                     oi->which = Z_OtherInfo_externallyDefinedInfo;
1915                     oi->information.externallyDefinedInfo =
1916                         yaz_set_response_charneg(
1917                             odr_encode(),
1918                             m_charset_converter->get_client_query_charset(),
1919                             0 /* no lang */,
1920                             m_charset_converter->get_client_charset_selected());
1921                 }
1922             }
1923         }
1924     }
1925 }
1926
1927 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1928                                                      int error,
1929                                                      const char *addinfo)
1930 {
1931     Z_Records *rec = (Z_Records *)
1932         odr_malloc (odr, sizeof(*rec));
1933     int *err = (int *)
1934         odr_malloc (odr, sizeof(*err));
1935     Z_DiagRec *drec = (Z_DiagRec *)
1936         odr_malloc (odr, sizeof(*drec));
1937     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1938         odr_malloc (odr, sizeof(*dr));
1939     *err = error;
1940     rec->which = Z_Records_NSD;
1941     rec->u.nonSurrogateDiagnostic = dr;
1942     dr->diagnosticSetId =
1943         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1944     dr->condition = err;
1945     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1946     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1947     return rec;
1948 }
1949
1950 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1951 {
1952     if (apdu->which == Z_APDU_searchRequest &&
1953         apdu->u.searchRequest->query &&
1954         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1955         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1956     {
1957         Z_RPNQuery *rpnquery = 0;
1958         Z_SearchRequest *sr = apdu->u.searchRequest;
1959         char *addinfo = 0;
1960
1961         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
1962                 sr->query->u.type_104->u.cql);
1963
1964         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1965                                           &rpnquery, odr_encode(),
1966                                           &addinfo);
1967         if (r == -3)
1968             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
1969         else if (r)
1970         {
1971             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1972             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1973
1974             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1975             new_apdu->u.searchResponse->records =
1976                 create_nonSurrogateDiagnostics(odr_encode(),
1977                                                yaz_diag_srw_to_bib1(r),
1978                                                addinfo);
1979             *new_apdu->u.searchResponse->searchStatus = 0;
1980
1981             send_to_client(new_apdu);
1982
1983             return 0;
1984         }
1985         else
1986         {
1987             sr->query->which = Z_Query_type_1;
1988             sr->query->u.type_1 = rpnquery;
1989         }
1990         return apdu;
1991     }
1992     return apdu;
1993 }
1994
1995 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
1996 {
1997     if (apdu->which == Z_APDU_searchRequest &&
1998         apdu->u.searchRequest->query)
1999     {
2000         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2001             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2002         {
2003             if (m_http_version)
2004                 m_charset_converter->set_client_query_charset("UTF-8");
2005             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2006             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2007         }
2008     }
2009     return apdu;
2010 }
2011
2012
2013 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2014 {
2015     if (apdu->which == Z_APDU_searchRequest)
2016     {
2017         Z_SearchRequest *sr = apdu->u.searchRequest;
2018         int err = 0;
2019         char *addinfo = 0;
2020
2021         Yaz_ProxyConfig *cfg = check_reconfigure();
2022         if (cfg)
2023             err = cfg->check_query(odr_encode(), m_default_target,
2024                                    sr->query, &addinfo);
2025         if (err)
2026         {
2027             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2028
2029             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2030             new_apdu->u.searchResponse->records =
2031                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2032             *new_apdu->u.searchResponse->searchStatus = 0;
2033
2034             send_to_client(new_apdu);
2035
2036             return 0;
2037         }
2038     }
2039     return apdu;
2040 }
2041
2042 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2043 {
2044     if (apdu->which != Z_APDU_initRequest)
2045         return 1;  // pass if no init request
2046     Z_InitRequest *req = apdu->u.initRequest;
2047
2048     Yaz_ProxyConfig *cfg = check_reconfigure();
2049     if (!cfg)
2050         return 1;  // pass if no config
2051
2052     int ret;
2053     if (req->idAuthentication == 0)
2054     {
2055         ret = cfg->client_authentication(m_default_target, 0, 0, 0);
2056     }
2057     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2058     {
2059         ret = cfg->client_authentication(m_default_target,
2060             req->idAuthentication->u.idPass->userId,
2061             req->idAuthentication->u.idPass->groupId,
2062             req->idAuthentication->u.idPass->password);
2063     }
2064     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2065     {
2066         char user[64], pass[64];
2067         *user = '\0';
2068         *pass = '\0';
2069         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2070         ret = cfg->client_authentication(m_default_target, user, 0, pass);
2071     }
2072     else
2073         ret = cfg->client_authentication(m_default_target, 0, 0, 0);
2074
2075     cfg->target_authentication(m_default_target, odr_encode(), req);
2076
2077     return ret;
2078 }
2079
2080 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2081 {
2082     m_marcxml_mode = none;
2083     if (apdu->which == Z_APDU_searchRequest)
2084     {
2085         Z_SearchRequest *sr = apdu->u.searchRequest;
2086         int err = 0;
2087         char *addinfo = 0;
2088         Yaz_ProxyConfig *cfg = check_reconfigure();
2089
2090         Z_RecordComposition rc_temp, *rc = 0;
2091         if (sr->smallSetElementSetNames)
2092         {
2093             rc_temp.which = Z_RecordComp_simple;
2094             rc_temp.u.simple = sr->smallSetElementSetNames;
2095             rc = &rc_temp;
2096         }
2097
2098         if (sr->preferredRecordSyntax)
2099         {
2100             struct oident *ent;
2101             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2102             m_frontend_type = ent->value;
2103         }
2104         else
2105             m_frontend_type = VAL_NONE;
2106
2107         char *stylesheet_name = 0;
2108         if (cfg)
2109             err = cfg->check_syntax(odr_encode(),
2110                                     m_default_target,
2111                                     sr->preferredRecordSyntax, rc,
2112                                     &addinfo, &stylesheet_name, &m_schema,
2113                                     &m_backend_type, &m_backend_charset,
2114                                     &m_usemarcon_ini_stage1,
2115                                     &m_usemarcon_ini_stage2);
2116         if (stylesheet_name)
2117         {
2118             m_parent->low_socket_close();
2119
2120 #if HAVE_XSLT
2121             if (m_stylesheet_xsp)
2122                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2123             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2124                                                        stylesheet_name);
2125 #endif
2126             m_stylesheet_offset = 0;
2127             xfree(stylesheet_name);
2128
2129             m_parent->low_socket_open();
2130         }
2131         if (err == -1)
2132         {
2133             sr->smallSetElementSetNames = 0;
2134             sr->mediumSetElementSetNames = 0;
2135             m_marcxml_mode = marcxml;
2136             if (m_backend_type)
2137             {
2138
2139                 sr->preferredRecordSyntax =
2140                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2141                                         m_backend_type);
2142             }
2143             else
2144                 sr->preferredRecordSyntax =
2145                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2146                                            VAL_USMARC);
2147         }
2148         else if (err)
2149         {
2150             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2151
2152             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2153             new_apdu->u.searchResponse->records =
2154                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2155             *new_apdu->u.searchResponse->searchStatus = 0;
2156
2157             send_to_client(new_apdu);
2158
2159             return 0;
2160         }
2161         else if (m_backend_type)
2162         {
2163             sr->preferredRecordSyntax =
2164                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2165         }
2166     }
2167     else if (apdu->which == Z_APDU_presentRequest)
2168     {
2169         Z_PresentRequest *pr = apdu->u.presentRequest;
2170         int err = 0;
2171         char *addinfo = 0;
2172         Yaz_ProxyConfig *cfg = check_reconfigure();
2173
2174         if (pr->preferredRecordSyntax)
2175         {
2176             struct oident *ent;
2177             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2178             m_frontend_type = ent->value;
2179         }
2180         else
2181             m_frontend_type = VAL_NONE;
2182
2183         char *stylesheet_name = 0;
2184         if (cfg)
2185             err = cfg->check_syntax(odr_encode(), m_default_target,
2186                                     pr->preferredRecordSyntax,
2187                                     pr->recordComposition,
2188                                     &addinfo, &stylesheet_name, &m_schema,
2189                                     &m_backend_type, &m_backend_charset,
2190                                     &m_usemarcon_ini_stage1,
2191                                     &m_usemarcon_ini_stage2
2192                                     );
2193         if (stylesheet_name)
2194         {
2195             m_parent->low_socket_close();
2196
2197 #if HAVE_XSLT
2198             if (m_stylesheet_xsp)
2199                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2200             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2201                                                        stylesheet_name);
2202 #endif
2203             m_stylesheet_offset = 0;
2204             xfree(stylesheet_name);
2205
2206             m_parent->low_socket_open();
2207         }
2208         if (err == -1)
2209         {
2210             pr->recordComposition = 0;
2211             m_marcxml_mode = marcxml;
2212             if (m_backend_type)
2213             {
2214
2215                 pr->preferredRecordSyntax =
2216                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2217                                         m_backend_type);
2218             }
2219             else
2220                 pr->preferredRecordSyntax =
2221                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2222                                            VAL_USMARC);
2223         }
2224         else if (err)
2225         {
2226             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2227
2228             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2229             new_apdu->u.presentResponse->records =
2230                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2231             *new_apdu->u.presentResponse->presentStatus =
2232                 Z_PresentStatus_failure;
2233
2234             send_to_client(new_apdu);
2235
2236             return 0;
2237         }
2238         else if (m_backend_type)
2239         {
2240             pr->preferredRecordSyntax =
2241                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2242         }
2243     }
2244     return apdu;
2245 }
2246
2247 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2248 {
2249     if (!schema)
2250         return 0;
2251     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2252         odr_malloc(o, sizeof(Z_ElementSetNames));
2253     esn->which = Z_ElementSetNames_generic;
2254     esn->u.generic = odr_strdup(o, schema);
2255     return esn;
2256 }
2257
2258 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2259 {
2260     const char *t = 0;
2261     Yaz_ProxyConfig *cfg = check_reconfigure();
2262     if (cfg)
2263         t = cfg->get_explain_name(db, backend_db);
2264
2265     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2266     {
2267         releaseClient();
2268     }
2269
2270     if (t)
2271     {
2272         xfree(m_default_target);
2273         m_default_target = xstrdup(t);
2274     }
2275 }
2276
2277 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2278 {
2279     struct stat sbuf;
2280     if (strcmp(hreq->method, "GET"))
2281         return 0;
2282     if (hreq->path[0] != '/')
2283     {
2284         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2285         return 0;
2286     }
2287     const char *cp = hreq->path;
2288     while (*cp)
2289     {
2290         if (*cp == '/' && strchr("/.", cp[1]))
2291         {
2292             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2293             return 0;
2294         }
2295         cp++;
2296     }
2297     const char *fname = hreq->path+1;
2298     if (stat(fname, &sbuf))
2299     {
2300         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2301         return 0;
2302     }
2303     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2304     {
2305         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2306         return 0;
2307     }
2308     if (sbuf.st_size > (off_t) 1000000)
2309     {
2310         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2311         return 0;
2312     }
2313
2314     ODR o = odr_encode();
2315     Yaz_ProxyConfig *cfg = check_reconfigure();
2316     const char *ctype = cfg->check_mime_type(fname);
2317     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2318     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2319     if (m_http_version)
2320         hres->version = odr_strdup(o, m_http_version);
2321     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2322     if (m_http_keepalive)
2323         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2324     else
2325         timeout(0);
2326
2327     hres->content_len = sbuf.st_size;
2328     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2329     FILE *f = fopen(fname, "rb");
2330     if (f)
2331     {
2332         fread(hres->content_buf, 1, hres->content_len, f);
2333         fclose(f);
2334     }
2335     else
2336     {
2337         return 0;
2338     }
2339     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2340     {
2341         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2342                  fname);
2343     }
2344     int len;
2345     send_GDU(gdu, &len);
2346     return 1;
2347 }
2348
2349 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2350 {
2351     if (m_s2z_odr_init)
2352     {
2353         odr_destroy(m_s2z_odr_init);
2354         m_s2z_odr_init = 0;
2355     }
2356     if (m_s2z_odr_search)
2357     {
2358         odr_destroy(m_s2z_odr_search);
2359         m_s2z_odr_search = 0;
2360     }
2361
2362     m_http_keepalive = 0;
2363     m_http_version = 0;
2364     if (!strcmp(hreq->version, "1.0"))
2365     {
2366         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2367         if (v && !strcmp(v, "Keep-Alive"))
2368             m_http_keepalive = 1;
2369         else
2370             m_http_keepalive = 0;
2371         m_http_version = "1.0";
2372     }
2373     else
2374     {
2375         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2376         if (v && !strcmp(v, "close"))
2377             m_http_keepalive = 0;
2378         else
2379             m_http_keepalive = 1;
2380         m_http_version = "1.1";
2381     }
2382
2383     Z_SRW_PDU *srw_pdu = 0;
2384     Z_SOAP *soap_package = 0;
2385     char *charset = 0;
2386     Z_SRW_diagnostic *diagnostic = 0;
2387     int num_diagnostic = 0;
2388
2389     if (file_access(hreq))
2390     {
2391         return;
2392     }
2393     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2394                             &charset) == 0
2395              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2396                                &charset, &diagnostic, &num_diagnostic) == 0)
2397     {
2398         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2399         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2400         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2401         m_s2z_init_apdu = 0;
2402         m_s2z_search_apdu = 0;
2403         m_s2z_present_apdu = 0;
2404
2405         m_s2z_stylesheet = 0;
2406
2407         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2408         {
2409             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2410
2411             const char *backend_db = srw_req->database;
2412             srw_get_client(srw_req->database, &backend_db);
2413
2414             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2415             // recordXPath unsupported.
2416             if (srw_req->recordXPath)
2417             {
2418                 yaz_add_srw_diagnostic(odr_decode(),
2419                                        &diagnostic, &num_diagnostic,
2420                                        72, 0);
2421             }
2422             // sort unsupported
2423             if (srw_req->sort_type != Z_SRW_sort_type_none)
2424             {
2425                 yaz_add_srw_diagnostic(odr_decode(),
2426                                        &diagnostic, &num_diagnostic,
2427                                        80, 0);
2428             }
2429             // save stylesheet
2430             if (srw_req->stylesheet)
2431                 m_s2z_stylesheet =
2432                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2433
2434             // set packing for response records ..
2435             if (srw_req->recordPacking &&
2436                 !strcmp(srw_req->recordPacking, "xml"))
2437                 m_s2z_packing = Z_SRW_recordPacking_XML;
2438             else
2439                 m_s2z_packing = Z_SRW_recordPacking_string;
2440
2441             if (num_diagnostic)
2442             {
2443                 Z_SRW_PDU *srw_pdu =
2444                     yaz_srw_get(odr_encode(),
2445                                 Z_SRW_searchRetrieve_response);
2446                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2447
2448                 srw_res->diagnostics = diagnostic;
2449                 srw_res->num_diagnostics = num_diagnostic;
2450                 send_srw_response(srw_pdu);
2451                 return;
2452             }
2453
2454             // prepare search PDU
2455             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2456                                           Z_APDU_searchRequest);
2457             Z_SearchRequest *z_searchRequest =
2458                 m_s2z_search_apdu->u.searchRequest;
2459
2460             z_searchRequest->num_databaseNames = 1;
2461             z_searchRequest->databaseNames = (char**)
2462                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2463             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2464                                                            backend_db);
2465
2466             // query transformation
2467             Z_Query *query = (Z_Query *)
2468                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2469             z_searchRequest->query = query;
2470
2471             if (srw_req->query_type == Z_SRW_query_type_cql)
2472             {
2473                 Z_External *ext = (Z_External *)
2474                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2475                 ext->direct_reference =
2476                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2477                 ext->indirect_reference = 0;
2478                 ext->descriptor = 0;
2479                 ext->which = Z_External_CQL;
2480                 ext->u.cql = srw_req->query.cql;
2481
2482                 query->which = Z_Query_type_104;
2483                 query->u.type_104 =  ext;
2484             }
2485             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2486             {
2487                 Z_RPNQuery *RPNquery;
2488                 YAZ_PQF_Parser pqf_parser;
2489
2490                 pqf_parser = yaz_pqf_create ();
2491
2492                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2493                                           srw_req->query.pqf);
2494                 if (!RPNquery)
2495                 {
2496                     const char *pqf_msg;
2497                     size_t off;
2498                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2499                     yaz_log(YLOG_LOG, "%*s^\n", off+4, "");
2500                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2501
2502                     send_to_srw_client_error(10, 0);
2503                     return;
2504                 }
2505                 query->which = Z_Query_type_1;
2506                 query->u.type_1 =  RPNquery;
2507
2508                 yaz_pqf_destroy (pqf_parser);
2509             }
2510             else
2511             {
2512                 send_to_srw_client_error(7, "query");
2513                 return;
2514             }
2515
2516             // present
2517             m_s2z_present_apdu = 0;
2518             int max = 0;
2519             if (srw_req->maximumRecords)
2520                 max = *srw_req->maximumRecords;
2521             int start = 1;
2522             if (srw_req->startRecord)
2523                 start = *srw_req->startRecord;
2524             if (max > 0)
2525             {
2526                 // Some backend, such as Voyager doesn't honor piggyback
2527                 // So we use present always (0 &&).
2528                 if (0 && start <= 1)  // Z39.50 piggyback
2529                 {
2530                     *z_searchRequest->smallSetUpperBound = max;
2531                     *z_searchRequest->mediumSetPresentNumber = max;
2532                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2533
2534                     z_searchRequest->preferredRecordSyntax =
2535                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2536                                                VAL_TEXT_XML);
2537                     if (srw_req->recordSchema)
2538                     {
2539                         z_searchRequest->smallSetElementSetNames =
2540                             z_searchRequest->mediumSetElementSetNames =
2541                             mk_esn_from_schema(m_s2z_odr_search,
2542                                                srw_req->recordSchema);
2543                     }
2544                 }
2545                 else   // Z39.50 present
2546                 {
2547                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2548                                                    Z_APDU_presentRequest);
2549                     Z_PresentRequest *z_presentRequest =
2550                         m_s2z_present_apdu->u.presentRequest;
2551                     *z_presentRequest->resultSetStartPoint = start;
2552                     *z_presentRequest->numberOfRecordsRequested = max;
2553                     z_presentRequest->preferredRecordSyntax =
2554                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2555                                                VAL_TEXT_XML);
2556                     if (srw_req->recordSchema)
2557                     {
2558                         z_presentRequest->recordComposition =
2559                             (Z_RecordComposition *)
2560                             odr_malloc(m_s2z_odr_search,
2561                                        sizeof(Z_RecordComposition));
2562                         z_presentRequest->recordComposition->which =
2563                             Z_RecordComp_simple;
2564                         z_presentRequest->recordComposition->u.simple =
2565                             mk_esn_from_schema(m_s2z_odr_search,
2566                                                srw_req->recordSchema);
2567                     }
2568                 }
2569             }
2570             if (!m_client)
2571             {
2572                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2573                                             Z_APDU_initRequest);
2574
2575                 // prevent m_initRequest_apdu memory from being grabbed
2576                 // in Yaz_Proxy::handle_incoming_Z_PDU
2577                 m_initRequest_apdu = m_s2z_init_apdu;
2578                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2579                 return;
2580             }
2581             else
2582             {
2583                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2584                 return;
2585             }
2586         }
2587         else if (srw_pdu->which == Z_SRW_explain_request)
2588         {
2589             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2590
2591             const char *backend_db = srw_req->database;
2592             srw_get_client(srw_req->database, &backend_db);
2593
2594             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2595
2596             // save stylesheet
2597             if (srw_req->stylesheet)
2598                 m_s2z_stylesheet =
2599                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2600
2601             if (srw_req->recordPacking &&
2602                 !strcmp(srw_req->recordPacking, "xml"))
2603                 m_s2z_packing = Z_SRW_recordPacking_XML;
2604             else
2605                 m_s2z_packing = Z_SRW_recordPacking_string;
2606
2607             if (num_diagnostic)
2608             {
2609                 send_srw_explain_response(diagnostic, num_diagnostic);
2610                 return;
2611             }
2612
2613             if (!m_client)
2614             {
2615                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2616                                             Z_APDU_initRequest);
2617
2618                 // prevent m_initRequest_apdu memory from being grabbed
2619                 // in Yaz_Proxy::handle_incoming_Z_PDU
2620                 m_initRequest_apdu = m_s2z_init_apdu;
2621                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2622             }
2623             else
2624                 send_srw_explain_response(0, 0);
2625             return;
2626         }
2627         else if (srw_pdu->which == Z_SRW_scan_request)
2628         {
2629             m_s2z_database = odr_strdup(m_s2z_odr_init,
2630                                         srw_pdu->u.scan_request->database);
2631
2632             yaz_add_srw_diagnostic(odr_decode(),
2633                                    &diagnostic, &num_diagnostic,
2634                                    4, "scan");
2635             Z_SRW_PDU *srw_pdu =
2636                 yaz_srw_get(odr_encode(),
2637                             Z_SRW_scan_response);
2638             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2639
2640             srw_res->diagnostics = diagnostic;
2641             srw_res->num_diagnostics = num_diagnostic;
2642             send_srw_response(srw_pdu);
2643             return;
2644         }
2645         else
2646         {
2647             m_s2z_database = 0;
2648
2649             send_to_srw_client_error(4, 0);
2650         }
2651     }
2652     send_http_response(400);
2653 }
2654
2655 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2656 {
2657     Z_OtherInformation **oi;
2658     get_otherInfoAPDU(apdu, &oi);
2659
2660     if (apdu->u.initRequest->implementationId)
2661         yaz_log(YLOG_LOG, "%simplementationId: %s",
2662                 m_session_str, apdu->u.initRequest->implementationId);
2663     if (apdu->u.initRequest->implementationName)
2664         yaz_log(YLOG_LOG, "%simplementationName: %s",
2665                 m_session_str, apdu->u.initRequest->implementationName);
2666     if (apdu->u.initRequest->implementationVersion)
2667         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2668                 m_session_str, apdu->u.initRequest->implementationVersion);
2669     if (m_initRequest_apdu == 0)
2670     {
2671         if (m_initRequest_mem)
2672             nmem_destroy(m_initRequest_mem);
2673
2674         m_initRequest_apdu = apdu;
2675         m_initRequest_mem = odr_extract_mem(odr_decode());
2676
2677         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2678             preferredMessageSize;
2679         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2680         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2681             maximumRecordSize;
2682         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2683
2684         // Save proposal charsets and langs.
2685         if (ODR_MASK_GET(apdu->u.initRequest->options,
2686                          Z_Options_negotiationModel))
2687         {
2688             Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2689                 yaz_get_charneg_record(*oi);
2690
2691             yaz_get_proposal_charneg(m_referenceId_mem,
2692                                      charSetandLangRecord,
2693                                      &m_initRequest_oi_negotiation_charsets,
2694                                      &m_initRequest_oi_negotiation_num_charsets,
2695                                      &m_initRequest_oi_negotiation_langs,
2696                                      &m_initRequest_oi_negotiation_num_langs,
2697                                      &m_initRequest_oi_negotiation_selected);
2698
2699             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2700             {
2701                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2702                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2703                         m_initRequest_oi_negotiation_charsets[i]:"none");
2704             }
2705             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2706             {
2707                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2708                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2709                         m_initRequest_oi_negotiation_langs[i]:"none");
2710             }
2711             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2712                     m_session_str, m_initRequest_oi_negotiation_selected);
2713         }
2714         // save init options for the response..
2715         m_initRequest_options = apdu->u.initRequest->options;
2716
2717         apdu->u.initRequest->options =
2718             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2719                                        sizeof(Odr_bitmask));
2720         ODR_MASK_ZERO(apdu->u.initRequest->options);
2721         int i;
2722         for (i = 0; i<= 24; i++)
2723             ODR_MASK_SET(apdu->u.initRequest->options, i);
2724         // check negotiation option
2725         if (!ODR_MASK_GET(m_initRequest_options,
2726                           Z_Options_negotiationModel))
2727         {
2728             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2729                            Z_Options_negotiationModel);
2730         }
2731         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2732                        Z_Options_concurrentOperations);
2733         // make new version
2734         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2735         apdu->u.initRequest->protocolVersion =
2736             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2737                                        sizeof(Odr_bitmask));
2738         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2739
2740         for (i = 0; i<= 8; i++)
2741             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2742     }
2743     if (m_client->m_init_flag)
2744     {
2745         if (handle_init_response_for_invalid_session(apdu))
2746             return;
2747         if (m_client->m_initResponse)
2748         {
2749             Z_APDU *apdu2 = m_client->m_initResponse;
2750             apdu2->u.initResponse->otherInfo = 0;
2751             if (m_client->m_cookie && *m_client->m_cookie)
2752                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2753                                            m_client->m_cookie);
2754             apdu2->u.initResponse->referenceId =
2755                 apdu->u.initRequest->referenceId;
2756             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2757             apdu2->u.initResponse->protocolVersion =
2758                 m_client->m_initResponse_version;
2759
2760             send_to_client(apdu2);
2761             return;
2762         }
2763     }
2764     m_client->m_init_flag = 1;
2765
2766     Auth_Msg *m = new Auth_Msg;
2767     m->m_apdu = apdu;
2768     m->m_proxy = this;
2769     m_my_thread->put(m);
2770 }
2771
2772 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2773 {
2774     Z_ReferenceId **refid = get_referenceIdP(apdu);
2775     nmem_reset(m_referenceId_mem);
2776     if (refid && *refid)
2777     {
2778         m_referenceId = (Z_ReferenceId *)
2779             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2780         m_referenceId->len = m_referenceId->size = (*refid)->len;
2781         m_referenceId->buf = (unsigned char *)
2782             nmem_malloc(m_referenceId_mem, (*refid)->len);
2783         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2784     }
2785     else
2786         m_referenceId = 0;
2787
2788     if (!m_client && m_invalid_session)
2789     {
2790         m_apdu_invalid_session = apdu;
2791         m_mem_invalid_session = odr_extract_mem(odr_decode());
2792         apdu = m_initRequest_apdu;
2793     }
2794
2795     // Determine our client.
2796     Z_OtherInformation **oi;
2797     get_otherInfoAPDU(apdu, &oi);
2798     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2799     if (!m_client)
2800     {
2801         if (m_http_version)
2802         {
2803             send_http_response(404);
2804             return;
2805         }
2806         else
2807         {
2808             delete this;
2809             return;
2810         }
2811     }
2812
2813     m_client->m_server = this;
2814
2815     if (apdu->which == Z_APDU_initRequest)
2816         handle_init(apdu);
2817     else
2818         handle_incoming_Z_PDU_2(apdu);
2819 }
2820
2821 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
2822 {
2823     handle_max_record_retrieve(apdu);
2824
2825     if (apdu)
2826         apdu = handle_syntax_validation(apdu);
2827
2828     if (apdu)
2829         apdu = handle_query_transformation(apdu);
2830
2831     if (apdu)
2832         apdu = handle_target_charset_conversion(apdu);
2833
2834     if (apdu)
2835         apdu = handle_query_validation(apdu);
2836
2837     if (apdu)
2838         apdu = result_set_optimize(apdu);
2839
2840     if (!apdu)
2841     {
2842         m_client->timeout(m_target_idletime);  // mark it active even
2843         // though we didn't use it
2844         return;
2845     }
2846     // Add otherInformation entry in APDU if
2847     // negotiation is in use.
2848     if (apdu)
2849         handle_charset_lang_negotiation(apdu);
2850
2851     // delete other info construct completely if 0 elements
2852     Z_OtherInformation **oi;
2853     get_otherInfoAPDU(apdu, &oi);
2854     if (oi && *oi && (*oi)->num_elements == 0)
2855         *oi = 0;
2856
2857     if (apdu->which == Z_APDU_presentRequest &&
2858         m_client->m_resultSetStartPoint == 0)
2859     {
2860         Z_PresentRequest *pr = apdu->u.presentRequest;
2861         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2862         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2863     } else {
2864         m_client->m_resultSetStartPoint = 0;
2865     }
2866     if (m_client->send_to_target(apdu) < 0)
2867     {
2868         delete m_client;
2869         m_client = 0;
2870         delete this;
2871     }
2872     else
2873         m_client->m_waiting = 1;
2874 }
2875
2876 void Yaz_Proxy::connectNotify()
2877 {
2878 }
2879
2880 void Yaz_Proxy::releaseClient()
2881 {
2882     xfree(m_proxyTarget);
2883     m_proxyTarget = 0;
2884     m_invalid_session = 0;
2885     // only keep if keep_alive flag is set...
2886     if (m_client &&
2887         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2888         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2889         m_client->m_waiting == 0)
2890     {
2891         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2892                  m_session_str,
2893                  m_client->get_hostname());
2894         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2895                 m_session_str, m_client->m_pdu_recv,
2896                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2897                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2898         assert (m_client->m_waiting != 2);
2899         // Tell client (if any) that no server connection is there..
2900         m_client->m_server = 0;
2901         m_client = 0;
2902     }
2903     else if (m_client)
2904     {
2905         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
2906                  m_session_str,
2907                  m_client->get_hostname());
2908         assert (m_client->m_waiting != 2);
2909         delete m_client;
2910         m_client = 0;
2911     }
2912     else if (!m_parent)
2913     {
2914         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
2915                  m_session_str);
2916         assert (m_parent);
2917     }
2918     else
2919     {
2920         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
2921                  m_session_str);
2922     }
2923     if (m_parent)
2924         m_parent->pre_init();
2925 }
2926
2927 void Yaz_Proxy::shutdown()
2928 {
2929     releaseClient();
2930     delete this;
2931 }
2932
2933 const char *Yaz_ProxyClient::get_session_str()
2934 {
2935     if (!m_server)
2936         return "0 ";
2937     return m_server->get_session_str();
2938 }
2939
2940 void Yaz_ProxyClient::shutdown()
2941 {
2942     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2943              get_hostname());
2944     delete m_server;
2945     delete this;
2946 }
2947
2948 void Yaz_Proxy::failNotify()
2949 {
2950     inc_request_no();
2951     yaz_log (YLOG_LOG, "%sConnection closed by client",
2952              get_session_str());
2953     shutdown();
2954 }
2955
2956 void Yaz_ProxyClient::failNotify()
2957 {
2958     if (m_server)
2959         m_server->inc_request_no();
2960     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
2961              get_session_str(), get_hostname());
2962     shutdown();
2963 }
2964
2965 void Yaz_ProxyClient::connectNotify()
2966 {
2967     const char *s = get_session_str();
2968     const char *h = get_hostname();
2969     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2970              m_target_idletime);
2971     timeout(m_target_idletime);
2972     if (!m_server)
2973         pre_init_client();
2974 }
2975
2976 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2977                                                   *the_PDU_Observable, int fd)
2978 {
2979     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2980 }
2981
2982 Yaz_ProxyClient::~Yaz_ProxyClient()
2983 {
2984     if (m_prev)
2985         *m_prev = m_next;
2986     if (m_next)
2987         m_next->m_prev = m_prev;
2988     m_waiting = 2;     // for debugging purposes only.
2989     odr_destroy(m_init_odr);
2990     delete m_last_query;
2991     xfree (m_last_resultSetId);
2992     xfree (m_cookie);
2993 }
2994
2995 void Yaz_ProxyClient::pre_init_client()
2996 {
2997     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2998     Z_InitRequest *req = apdu->u.initRequest;
2999
3000     int i;
3001     for (i = 0; i<= 24; i++)
3002         ODR_MASK_SET(req->options, i);
3003     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3004                    Z_Options_negotiationModel);
3005     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3006                    Z_Options_concurrentOperations);
3007     for (i = 0; i<= 10; i++)
3008         ODR_MASK_SET(req->protocolVersion, i);
3009
3010     if (send_to_target(apdu) < 0)
3011     {
3012         delete this;
3013     }
3014     else
3015     {
3016         m_waiting = 1;
3017         m_init_flag = 1;
3018     }
3019 }
3020
3021 void Yaz_Proxy::pre_init()
3022 {
3023     int i;
3024     const char *name = 0;
3025     const char *zurl_in_use[MAX_ZURL_PLEX];
3026     int limit_bw, limit_pdu, limit_req;
3027     int target_idletime, client_idletime;
3028     int max_clients;
3029     int keepalive_limit_bw, keepalive_limit_pdu;
3030     int pre_init;
3031     const char *cql2rpn = 0;
3032     const char *authentication = 0;
3033     const char *negotiation_charset = 0;
3034     const char *negotiation_lang = 0;
3035
3036     Yaz_ProxyConfig *cfg = check_reconfigure();
3037
3038     zurl_in_use[0] = 0;
3039
3040     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3041         set_APDU_yazlog(1);
3042     else
3043         set_APDU_yazlog(0);
3044
3045     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3046                                           &limit_bw, &limit_pdu, &limit_req,
3047                                           &target_idletime, &client_idletime,
3048                                           &max_clients,
3049                                           &keepalive_limit_bw,
3050                                           &keepalive_limit_pdu,
3051                                           &pre_init,
3052                                           &cql2rpn,
3053                                           &authentication,
3054                                           &negotiation_charset,
3055                                           &negotiation_lang,
3056                                           0) ; i++)
3057     {
3058         if (pre_init)
3059         {
3060             int j;
3061             for (j = 0; zurl_in_use[j]; j++)
3062             {
3063                 Yaz_ProxyClient *c;
3064                 int spare = 0;
3065                 int spare_waiting = 0;
3066                 int in_use = 0;
3067                 int other = 0;
3068                 for (c = m_clientPool; c; c = c->m_next)
3069                 {
3070                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3071                     {
3072                         if (c->m_cookie == 0)
3073                         {
3074                             if (c->m_server == 0)
3075                                 if (c->m_waiting)
3076                                     spare_waiting++;
3077                                 else
3078                                     spare++;
3079                             else
3080                                 in_use++;
3081                         }
3082                         else
3083                             other++;
3084                     }
3085                 }
3086                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3087                         "sparew=%d preinit=%d",m_session_str,
3088                         name, zurl_in_use[j], in_use, other,
3089                         spare, spare_waiting, pre_init);
3090                 if (spare + spare_waiting < pre_init)
3091                 {
3092                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3093                     c->m_next = m_clientPool;
3094                     if (c->m_next)
3095                         c->m_next->m_prev = &c->m_next;
3096                     m_clientPool = c;
3097                     c->m_prev = &m_clientPool;
3098
3099                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3100                         c->set_APDU_yazlog(1);
3101                     else
3102                         c->set_APDU_yazlog(0);
3103
3104                     if (c->client(zurl_in_use[j]))
3105                     {
3106                         timeout(60);
3107                         delete c;
3108                         return;
3109                     }
3110                     c->timeout(30);
3111                     c->m_waiting = 1;
3112                     c->m_target_idletime = target_idletime;
3113                     c->m_seqno = m_seqno++;
3114                 }
3115             }
3116         }
3117     }
3118 }
3119
3120 void Yaz_Proxy::timeoutNotify()
3121 {
3122     if (m_parent)
3123     {
3124         if (m_bw_hold_PDU)
3125         {
3126             timeout(m_client_idletime);
3127             Z_GDU *apdu = m_bw_hold_PDU;
3128             m_bw_hold_PDU = 0;
3129
3130             if (apdu->which == Z_GDU_Z3950)
3131                 handle_incoming_Z_PDU(apdu->u.z3950);
3132             else if (apdu->which == Z_GDU_HTTP_Request)
3133                 handle_incoming_HTTP(apdu->u.HTTP_Request);
3134         }
3135         else if (m_stylesheet_nprl)
3136             convert_xsl_delay();
3137         else
3138         {
3139             inc_request_no();
3140
3141             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3142             shutdown();
3143         }
3144     }
3145     else
3146     {
3147         timeout(600);
3148         pre_init();
3149     }
3150 }
3151
3152 void Yaz_Proxy::markInvalid()
3153 {
3154     m_client = 0;
3155     m_invalid_session = 1;
3156 }
3157
3158 void Yaz_ProxyClient::timeoutNotify()
3159 {
3160     if (m_server)
3161         m_server->inc_request_no();
3162
3163     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3164              get_hostname());
3165     m_waiting = 1;
3166     m_root->pre_init();
3167     if (m_server && m_init_flag)
3168     {
3169         // target timed out in a session that was properly initialized
3170         // server object stay alive but we mark it as invalid so it
3171         // gets initialized again
3172         m_server->markInvalid();
3173         m_server = 0;
3174     }
3175     shutdown();
3176 }
3177
3178 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
3179                                  Yaz_Proxy *parent) :
3180     Yaz_Z_Assoc (the_PDU_Observable)
3181 {
3182     m_cookie = 0;
3183     m_next = 0;
3184     m_prev = 0;
3185     m_init_flag = 0;
3186     m_last_query = 0;
3187     m_last_resultSetId = 0;
3188     m_last_resultCount = 0;
3189     m_last_ok = 0;
3190     m_sr_transform = 0;
3191     m_waiting = 0;
3192     m_init_odr = odr_createmem (ODR_DECODE);
3193     m_initResponse = 0;
3194     m_initResponse_options = 0;
3195     m_initResponse_version = 0;
3196     m_initResponse_preferredMessageSize = 0;
3197     m_initResponse_maximumRecordSize = 0;
3198     m_resultSetStartPoint = 0;
3199     m_bytes_sent = m_bytes_recv = 0;
3200     m_pdu_recv = 0;
3201     m_server = 0;
3202     m_seqno = 0;
3203     m_target_idletime = 600;
3204     m_root = parent;
3205 }
3206
3207 const char *Yaz_Proxy::option(const char *name, const char *value)
3208 {
3209     if (!strcmp (name, "optimize")) {
3210         if (value) {
3211             xfree (m_optimize);
3212             m_optimize = xstrdup (value);
3213         }
3214         return m_optimize;
3215     }
3216     return 0;
3217 }
3218
3219 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
3220 {
3221
3222 }
3223
3224 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
3225 {
3226     if (apdu->which == Z_GDU_Z3950)
3227         recv_Z_PDU(apdu->u.z3950, len);
3228     else if (apdu->which == Z_GDU_HTTP_Response)
3229         recv_HTTP_response(apdu->u.HTTP_Response, len);
3230     else
3231         shutdown();
3232 }
3233
3234 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
3235 {
3236     if (!m_invalid_session)
3237         return 0;
3238     m_invalid_session = 0;
3239     handle_incoming_Z_PDU(m_apdu_invalid_session);
3240     assert (m_mem_invalid_session);
3241     nmem_destroy(m_mem_invalid_session);
3242     m_mem_invalid_session = 0;
3243     return 1;
3244 }
3245
3246 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
3247 {
3248     m_bytes_recv += len;
3249
3250     m_pdu_recv++;
3251     m_waiting = 0;
3252     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
3253         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
3254                  apdu_name(apdu), get_hostname(), len);
3255     if (apdu->which == Z_APDU_initResponse)
3256     {
3257         if (!m_server)  // if this is a pre init session , check for more
3258             m_root->pre_init();
3259         NMEM nmem = odr_extract_mem (odr_decode());
3260         odr_reset (m_init_odr);
3261         nmem_transfer (m_init_odr->mem, nmem);
3262         m_initResponse = apdu;
3263         m_initResponse_options = apdu->u.initResponse->options;
3264         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3265         m_initResponse_preferredMessageSize =
3266             *apdu->u.initResponse->preferredMessageSize;
3267         m_initResponse_maximumRecordSize =
3268             *apdu->u.initResponse->maximumRecordSize;
3269
3270         Z_InitResponse *ir = apdu->u.initResponse;
3271         char *im0 = ir->implementationName;
3272
3273         char *im1 = (char*)
3274             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3275         *im1 = '\0';
3276         if (im0)
3277         {
3278             strcat(im1, im0);
3279             strcat(im1, " ");
3280         }
3281         strcat(im1, "(YAZ Proxy)");
3282         ir->implementationName = im1;
3283
3284         nmem_destroy (nmem);
3285
3286         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3287             return;
3288     }
3289     if (apdu->which == Z_APDU_searchResponse)
3290     {
3291         Z_SearchResponse *sr = apdu->u.searchResponse;
3292         m_last_resultCount = *sr->resultCount;
3293         int status = *sr->searchStatus;
3294         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3295         {
3296             m_last_ok = 1;
3297
3298             if (sr->records && sr->records->which == Z_Records_DBOSD)
3299             {
3300                 m_cache.add(odr_decode(),
3301                             sr->records->u.databaseOrSurDiagnostics, 1,
3302                             *sr->resultCount);
3303             }
3304         }
3305     }
3306     if (apdu->which == Z_APDU_presentResponse)
3307     {
3308         Z_PresentResponse *pr = apdu->u.presentResponse;
3309         if (m_sr_transform)
3310         {
3311             m_sr_transform = 0;
3312             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3313             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3314             sr->referenceId = pr->referenceId;
3315             *sr->resultCount = m_last_resultCount;
3316             sr->records = pr->records;
3317             sr->nextResultSetPosition = pr->nextResultSetPosition;
3318             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3319             apdu = new_apdu;
3320         }
3321         if (pr->records &&
3322             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3323         {
3324             m_cache.add(odr_decode(),
3325                         pr->records->u.databaseOrSurDiagnostics,
3326                         m_resultSetStartPoint, -1);
3327             m_resultSetStartPoint = 0;
3328         }
3329     }
3330     if (m_cookie)
3331         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3332     if (m_server)
3333     {
3334         m_server->send_to_client(apdu);
3335     }
3336     if (apdu->which == Z_APDU_close)
3337     {
3338         shutdown();
3339     }
3340 }
3341
3342 void Yaz_Proxy::low_socket_close()
3343 {
3344 #if WIN32
3345 #else
3346     int i;
3347     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3348         if  (m_lo_fd[i] >= 0)
3349             ::close(m_lo_fd[i]);
3350 #endif
3351 }
3352
3353 void Yaz_Proxy::low_socket_open()
3354 {
3355 #if WIN32
3356 #else
3357     int i;
3358     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3359         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3360 #endif
3361 }
3362
3363 int Yaz_Proxy::server(const char *addr)
3364 {
3365     int r = Yaz_Z_Assoc::server(addr);
3366     if (!r)
3367     {
3368         yaz_log(YLOG_LOG, "%sStarted proxy "
3369 #ifdef VERSION
3370             VERSION
3371 #endif
3372             " on %s", m_session_str, addr);
3373         timeout(1);
3374     }
3375     return r;
3376 }
3377