src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.27 2005-05-27 18:07:49 adam Exp $
   2    Copyright (c) 1998-2005, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <stdlib.h>
  42 #include <time.h>
  43 #include <fcntl.h>
  44
  45 #include <yaz/srw.h>
  46 #include <yaz/marcdisp.h>
  47 #include <yaz/yaz-iconv.h>
  48 #include <yaz/log.h>
  49 #include <yaz/diagbib1.h>
  50 #include "proxyp.h"
  51 #include <yaz/pquery.h>
  52 #include <yaz/otherinfo.h>
  53 #include <yaz/charneg.h>
  54
  55
  56 static const char *apdu_name(Z_APDU *apdu)
  57 {
  58     switch (apdu->which)
  59     {
  60     case Z_APDU_initRequest:
  61         return "initRequest";
  62     case Z_APDU_initResponse:
  63         return "initResponse";
  64     case Z_APDU_searchRequest:
  65         return "searchRequest";
  66     case Z_APDU_searchResponse:
  67         return "searchResponse";
  68     case Z_APDU_presentRequest:
  69         return "presentRequest";
  70     case Z_APDU_presentResponse:
  71         return "presentResponse";
  72     case Z_APDU_deleteResultSetRequest:
  73         return "deleteResultSetRequest";
  74     case Z_APDU_deleteResultSetResponse:
  75         return "deleteResultSetResponse";
  76     case Z_APDU_scanRequest:
  77         return "scanRequest";
  78     case Z_APDU_scanResponse:
  79         return "scanResponse";
  80     case Z_APDU_sortRequest:
  81         return "sortRequest";
  82     case Z_APDU_sortResponse:
  83         return "sortResponse";
  84     case Z_APDU_extendedServicesRequest:
  85         return "extendedServicesRequest";
  86     case Z_APDU_extendedServicesResponse:
  87         return "extendedServicesResponse";
  88     case Z_APDU_close:
  89         return "close";
  90     }
  91     return "other";
  92 }
  93
  94 static const char *gdu_name(Z_GDU *gdu)
  95 {
  96     switch(gdu->which)
  97     {
  98     case Z_GDU_Z3950:
  99         return apdu_name(gdu->u.z3950);
 100     case Z_GDU_HTTP_Request:
 101         return "HTTP Request";
 102     case Z_GDU_HTTP_Response:
 103         return "HTTP Response";
 104     }
 105     return "Unknown request/response";
 106 }
 107 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
 108                      Yaz_Proxy *parent) :
 109     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 110 {
 111     m_PDU_Observable = the_PDU_Observable;
 112     m_client = 0;
 113     m_parent = parent;
 114     m_clientPool = 0;
 115     m_seqno = 1;
 116     m_keepalive_limit_bw = 500000;
 117     m_keepalive_limit_pdu = 1000;
 118     m_proxyTarget = 0;
 119     m_default_target = 0;
 120     m_proxy_negotiation_charset = 0;
 121     m_proxy_negotiation_lang = 0;
 122     m_charset_converter = new Yaz_CharsetConverter;
 123     m_max_clients = 150;
 124     m_log_mask = 0;
 125     m_seed = time(0);
 126     m_client_idletime = 600;
 127     m_target_idletime = 600;
 128     m_optimize = xstrdup ("1");
 129     strcpy(m_session_str, "0 ");
 130     m_session_no=0;
 131     m_bytes_sent = 0;
 132     m_bytes_recv = 0;
 133     m_bw_hold_PDU = 0;
 134     m_bw_max = 0;
 135     m_pdu_max = 0;
 136     m_max_record_retrieve = 0;
 137     m_reconfig_flag = 0;
 138     m_config_fname = 0;
 139     m_request_no = 0;
 140     m_invalid_session = 0;
 141     m_referenceId = 0;
 142     m_referenceId_mem = nmem_create();
 143     m_config = 0;
 144     m_marcxml_mode = none;
 145     m_stylesheet_xsp = 0;
 146     m_stylesheet_nprl = 0;
 147     m_s2z_stylesheet = 0;
 148     m_s2z_database = 0;
 149     m_schema = 0;
 150     m_backend_type = 0;
 151     m_backend_charset = 0;
 152     m_frontend_type = 0;
 153     m_initRequest_apdu = 0;
 154     m_initRequest_mem = 0;
 155     m_initRequest_preferredMessageSize = 0;
 156     m_initRequest_maximumRecordSize = 0;
 157     m_initRequest_options = 0;
 158     m_initRequest_version = 0;
 159     m_initRequest_oi_negotiation_charsets = 0;
 160     m_initRequest_oi_negotiation_num_charsets = 0;
 161     m_initRequest_oi_negotiation_langs = 0;
 162     m_initRequest_oi_negotiation_num_langs = 0;
 163     m_initRequest_oi_negotiation_selected = 0;
 164     m_apdu_invalid_session = 0;
 165     m_mem_invalid_session = 0;
 166     m_s2z_odr_init = 0;
 167     m_s2z_odr_search = 0;
 168     m_s2z_init_apdu = 0;
 169     m_s2z_search_apdu = 0;
 170     m_s2z_present_apdu = 0;
 171     m_http_keepalive = 0;
 172     m_http_version = 0;
 173     m_soap_ns = 0;
 174     m_s2z_packing = Z_SRW_recordPacking_string;
 175 #if HAVE_GETTIMEOFDAY
 176     m_time_tv = xmalloc(sizeof(struct timeval));
 177     struct timeval *tv = (struct timeval *) m_time_tv;
 178     tv->tv_sec = 0;
 179     tv->tv_usec = 0;
 180 #else
 181     m_time_tv = 0;
 182 #endif
 183     m_usemarcon_ini_stage1 = 0;
 184     m_usemarcon_ini_stage2 = 0;
 185     m_usemarcon = new Yaz_usemarcon();
 186     if (!m_parent)
 187         low_socket_open();
 188 }
 189
 190 Yaz_Proxy::~Yaz_Proxy()
 191 {
 192     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 193             m_bytes_sent, m_bytes_recv);
 194     nmem_destroy(m_initRequest_mem);
 195     nmem_destroy(m_mem_invalid_session);
 196     nmem_destroy(m_referenceId_mem);
 197
 198     xfree(m_proxyTarget);
 199     xfree(m_default_target);
 200     xfree(m_proxy_negotiation_charset);
 201     xfree(m_proxy_negotiation_lang);
 202     delete m_charset_converter;
 203     xfree(m_optimize);
 204
 205 #if HAVE_XSLT
 206     if (m_stylesheet_xsp)
 207         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 208 #endif
 209     xfree (m_time_tv);
 210
 211     xfree (m_schema);
 212     xfree (m_backend_type);
 213     xfree (m_backend_charset);
 214     xfree (m_usemarcon_ini_stage1);
 215     xfree (m_usemarcon_ini_stage2);
 216     delete m_usemarcon;
 217     if (m_s2z_odr_init)
 218         odr_destroy(m_s2z_odr_init);
 219     if (m_s2z_odr_search)
 220         odr_destroy(m_s2z_odr_search);
 221     if (!m_parent)
 222         low_socket_close();
 223     delete m_config;
 224 }
 225
 226 void Yaz_Proxy::set_debug_mode(int mode)
 227 {
 228     m_debug_mode = mode;
 229 }
 230
 231 int Yaz_Proxy::set_config(const char *config)
 232 {
 233     delete m_config;
 234     m_config = new Yaz_ProxyConfig();
 235     xfree(m_config_fname);
 236     m_config_fname = xstrdup(config);
 237     int r = m_config->read_xml(config);
 238     if (!r)
 239         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 240     return r;
 241 }
 242
 243 void Yaz_Proxy::set_default_target(const char *target)
 244 {
 245     xfree (m_default_target);
 246     m_default_target = 0;
 247     if (target)
 248         m_default_target = (char *) xstrdup (target);
 249 }
 250
 251 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang)
 252 {
 253     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
 254         "language to '%s'", m_session_str, charset?charset:"none",
 255         lang?lang:"none");
 256     xfree (m_proxy_negotiation_charset);
 257     xfree (m_proxy_negotiation_lang);
 258     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 259     if (charset)
 260         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 261     if (lang)
 262         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 263 }
 264
 265 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 266 {
 267     if (m_parent)
 268         return m_parent->check_reconfigure();
 269
 270     Yaz_ProxyConfig *cfg = m_config;
 271     if (m_reconfig_flag)
 272     {
 273         yaz_log(YLOG_LOG, "reconfigure");
 274         yaz_log_reopen();
 275         if (m_config_fname && cfg)
 276         {
 277             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 278             int r = cfg->read_xml(m_config_fname);
 279             if (r)
 280                 yaz_log(YLOG_WARN, "reconfigure failed");
 281             else
 282             {
 283                 m_log_mask = 0;
 284                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 285             }
 286         }
 287         else
 288             yaz_log(YLOG_LOG, "reconfigure");
 289         m_reconfig_flag = 0;
 290     }
 291     return cfg;
 292 }
 293
 294 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 295                                             *the_PDU_Observable, int fd)
 296 {
 297     check_reconfigure();
 298     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 299     new_proxy->m_config = 0;
 300     new_proxy->m_config_fname = 0;
 301     new_proxy->timeout(m_client_idletime);
 302     new_proxy->m_target_idletime = m_target_idletime;
 303     new_proxy->set_default_target(m_default_target);
 304     new_proxy->m_max_clients = m_max_clients;
 305     new_proxy->m_log_mask = m_log_mask;
 306     new_proxy->set_APDU_log(get_APDU_log());
 307     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 308         new_proxy->set_APDU_yazlog(1);
 309     else
 310         new_proxy->set_APDU_yazlog(0);
 311     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 312     m_session_no++;
 313     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 314              the_PDU_Observable->getpeername());
 315     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 316         m_proxy_negotiation_lang);
 317     return new_proxy;
 318 }
 319
 320 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 321 {
 322     int oid[OID_SIZE];
 323     Z_OtherInformationUnit *oi;
 324     struct oident ent;
 325     ent.proto = PROTO_Z3950;
 326     ent.oclass = CLASS_USERINFO;
 327     ent.value = (oid_value) VAL_COOKIE;
 328     assert (oid_ent_to_oid (&ent, oid));
 329
 330     if (oid_ent_to_oid (&ent, oid) &&
 331         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 332         oi->which == Z_OtherInfo_characterInfo)
 333         return oi->information.characterInfo;
 334     return 0;
 335 }
 336 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 337 {
 338     int oid[OID_SIZE];
 339     Z_OtherInformationUnit *oi;
 340     struct oident ent;
 341     ent.proto = PROTO_Z3950;
 342     ent.oclass = CLASS_USERINFO;
 343     ent.value = (oid_value) VAL_PROXY;
 344     if (oid_ent_to_oid (&ent, oid) &&
 345         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 346         oi->which == Z_OtherInfo_characterInfo)
 347         return oi->information.characterInfo;
 348     return 0;
 349 }
 350 const char *Yaz_Proxy::load_balance(const char **url)
 351 {
 352     int zurl_in_use[MAX_ZURL_PLEX];
 353     int zurl_in_spare[MAX_ZURL_PLEX];
 354     Yaz_ProxyClient *c;
 355     int i;
 356
 357     for (i = 0; i<MAX_ZURL_PLEX; i++)
 358     {
 359         zurl_in_use[i] = 0;
 360         zurl_in_spare[i] = 0;
 361     }
 362     for (c = m_parent->m_clientPool; c; c = c->m_next)
 363     {
 364         for (i = 0; url[i]; i++)
 365             if (!strcmp(url[i], c->get_hostname()))
 366             {
 367                 zurl_in_use[i]++;
 368                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 369                     zurl_in_spare[i]++;
 370             }
 371     }
 372     int min_use = 100000;
 373     int spare_for_min = 0;
 374     int max_spare = 0;
 375     const char *ret_min = 0;
 376     const char *ret_spare = 0;
 377     for (i = 0; url[i]; i++)
 378     {
 379         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 380                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 381         if (min_use > zurl_in_use[i])
 382         {
 383             ret_min = url[i];
 384             min_use = zurl_in_use[i];
 385             spare_for_min = zurl_in_spare[i];
 386         }
 387         if (max_spare < zurl_in_spare[i])
 388         {
 389             ret_spare = url[i];
 390             max_spare = zurl_in_spare[i];
 391         }
 392     }
 393     return ret_min;
 394 }
 395
 396 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 397                                        const char *proxy_host)
 398 {
 399     assert (m_parent);
 400     Yaz_Proxy *parent = m_parent;
 401     Yaz_ProxyClient *c = m_client;
 402
 403     if (!m_proxyTarget)
 404     {
 405         const char *url[MAX_ZURL_PLEX];
 406         Yaz_ProxyConfig *cfg = check_reconfigure();
 407         if (proxy_host)
 408         {
 409             if (parent && parent->m_debug_mode)
 410             {
 411                 // only to be enabled for debugging...
 412                 if (!strcmp(proxy_host, "stop"))
 413                     exit(0);
 414             }
 415             xfree(m_default_target);
 416             m_default_target = xstrdup(proxy_host);
 417         }
 418         proxy_host = m_default_target;
 419         int client_idletime = -1;
 420         const char *cql2rpn_fname = 0;
 421         const char *negotiation_charset = 0;
 422         const char *negotiation_lang = 0;
 423         const char *query_charset = 0;
 424         url[0] = m_default_target;
 425         url[1] = 0;
 426         if (cfg)
 427         {
 428             int pre_init = 0;
 429             cfg->get_target_info(proxy_host, url, &m_bw_max,
 430                                  &m_pdu_max, &m_max_record_retrieve,
 431                                  &m_target_idletime, &client_idletime,
 432                                  &parent->m_max_clients,
 433                                  &m_keepalive_limit_bw,
 434                                  &m_keepalive_limit_pdu,
 435                                  &pre_init,
 436                                  &cql2rpn_fname,
 437                                  &negotiation_charset,
 438                                  &negotiation_lang,
 439                                  &query_charset);
 440         }
 441         if (client_idletime != -1)
 442         {
 443             m_client_idletime = client_idletime;
 444             timeout(m_client_idletime);
 445         }
 446         if (cql2rpn_fname)
 447             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 448         if (negotiation_charset || negotiation_lang)
 449         {
 450             set_proxy_negotiation(negotiation_charset,
 451                 negotiation_lang);
 452         }
 453         m_charset_converter->set_target_query_charset(query_charset);
 454         if (!url[0])
 455         {
 456             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 457             return 0;
 458         }
 459         // we don't handle multiplexing for cookie session, so we just
 460         // pick the first one in this case (anonymous users will be able
 461         // to use any backend)
 462         if (cookie && *cookie)
 463             m_proxyTarget = (char*) xstrdup(url[0]);
 464         else
 465             m_proxyTarget = (char*) xstrdup(load_balance(url));
 466     }
 467     if (cookie && *cookie)
 468     {   // search in sessions with a cookie
 469         for (c = parent->m_clientPool; c; c = c->m_next)
 470         {
 471             assert (c->m_prev);
 472             assert (*c->m_prev == c);
 473             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 474                 !strcmp(m_proxyTarget, c->get_hostname()))
 475             {
 476                 // Found it in cache
 477                 // The following handles "cancel"
 478                 // If connection is busy (waiting for PDU) and
 479                 // we have an initRequest we can safely do re-open
 480                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 481                 {
 482                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 483                              c->get_hostname());
 484                     c->close();
 485                     c->m_init_flag = 0;
 486
 487                     c->m_last_ok = 0;
 488                     c->m_cache.clear();
 489                     c->m_last_resultCount = 0;
 490                     c->m_sr_transform = 0;
 491                     c->m_waiting = 0;
 492                     c->m_resultSetStartPoint = 0;
 493                     c->m_target_idletime = m_target_idletime;
 494                     if (c->client(m_proxyTarget))
 495                     {
 496                         delete c;
 497                         return 0;
 498                     }
 499                     c->timeout(30);
 500                 }
 501                 c->m_seqno = parent->m_seqno;
 502                 if (c->m_server && c->m_server != this)
 503                     c->m_server->m_client = 0;
 504                 c->m_server = this;
 505                 (parent->m_seqno)++;
 506                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 507                 return c;
 508             }
 509         }
 510     }
 511     else if (!c &&
 512             apdu->which == Z_APDU_initRequest &&
 513             apdu->u.initRequest->idAuthentication == 0 &&
 514             !ODR_MASK_GET(apdu->u.initRequest->options, Z_Options_negotiationModel))
 515     {
 516         // anonymous sessions without cookie.
 517         // if authentication is set it is NOT anonymous se we can't share them.
 518         // If charset and lang negotiation is use it is NOT anonymous session too.
 519         for (c = parent->m_clientPool; c; c = c->m_next)
 520         {
 521             assert(c->m_prev);
 522             assert(*c->m_prev == c);
 523             if (c->m_server == 0 && c->m_cookie == 0 &&
 524                 c->m_waiting == 0 &&
 525                 !strcmp(m_proxyTarget, c->get_hostname()))
 526             {
 527                 // found it in cache
 528                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 529                          m_session_str, parent->m_seqno, c->get_hostname());
 530
 531                 c->m_seqno = parent->m_seqno;
 532                 assert(c->m_server == 0);
 533                 c->m_server = this;
 534
 535                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 536                     c->set_APDU_yazlog(1);
 537                 else
 538                     c->set_APDU_yazlog(0);
 539
 540                 (parent->m_seqno)++;
 541
 542                 parent->pre_init();
 543
 544                 return c;
 545             }
 546         }
 547     }
 548     if (!m_client)
 549     {
 550         if (apdu->which != Z_APDU_initRequest)
 551         {
 552             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 553             return 0;
 554         }
 555         Z_InitRequest *initRequest = apdu->u.initRequest;
 556
 557         if (initRequest->idAuthentication)
 558         {
 559             // the client uses authentication. We set the keepalive PDU
 560             // to 0 so we don't cache it in releaseClient
 561             m_keepalive_limit_pdu = 0;
 562         }
 563         // go through list of clients - and find the lowest/oldest one.
 564         Yaz_ProxyClient *c_min = 0;
 565         int min_seq = -1;
 566         int no_of_clients = 0;
 567         if (parent->m_clientPool)
 568             yaz_log (YLOG_DEBUG, "Existing sessions");
 569         for (c = parent->m_clientPool; c; c = c->m_next)
 570         {
 571             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 572                                c->m_waiting, c->get_hostname(),
 573                                c->m_cookie ? c->m_cookie : "");
 574             no_of_clients++;
 575             if (min_seq < 0 || c->m_seqno < min_seq)
 576             {
 577                 min_seq = c->m_seqno;
 578                 c_min = c;
 579             }
 580         }
 581         if (no_of_clients >= parent->m_max_clients)
 582         {
 583             c = c_min;
 584             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 585             {
 586                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 587                          m_session_str, parent->m_max_clients, c->m_seqno);
 588                 if (c->m_server && c->m_server != this)
 589                     delete c->m_server;
 590                 c->m_server = 0;
 591             }
 592             else
 593             {
 594                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 595                          m_session_str, parent->m_max_clients,
 596                          c->m_seqno, parent->m_seqno, c->get_hostname());
 597                 xfree (c->m_cookie);
 598                 c->m_cookie = 0;
 599                 if (cookie)
 600                     c->m_cookie = xstrdup(cookie);
 601                 c->m_seqno = parent->m_seqno;
 602                 if (c->m_server && c->m_server != this)
 603                 {
 604                     c->m_server->m_client = 0;
 605                     delete c->m_server;
 606                 }
 607                 (parent->m_seqno)++;
 608                 c->m_target_idletime = m_target_idletime;
 609                 c->timeout(m_target_idletime);
 610
 611                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 612                     c->set_APDU_yazlog(1);
 613                 else
 614                     c->set_APDU_yazlog(0);
 615
 616                 return c;
 617             }
 618         }
 619         else
 620         {
 621             yaz_log (YLOG_LOG, "%sNEW %d %s",
 622                      m_session_str, parent->m_seqno, m_proxyTarget);
 623             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 624             c->m_next = parent->m_clientPool;
 625             if (c->m_next)
 626                 c->m_next->m_prev = &c->m_next;
 627             parent->m_clientPool = c;
 628             c->m_prev = &parent->m_clientPool;
 629         }
 630
 631         xfree (c->m_cookie);
 632         c->m_cookie = 0;
 633         if (cookie)
 634             c->m_cookie = xstrdup(cookie);
 635
 636         c->m_seqno = parent->m_seqno;
 637         c->m_init_flag = 0;
 638         c->m_last_resultCount = 0;
 639         c->m_last_ok = 0;
 640         c->m_cache.clear();
 641         c->m_sr_transform = 0;
 642         c->m_waiting = 0;
 643         c->m_resultSetStartPoint = 0;
 644         (parent->m_seqno)++;
 645         if (c->client(m_proxyTarget))
 646         {
 647             delete c;
 648             return 0;
 649         }
 650         c->m_target_idletime = m_target_idletime;
 651         c->timeout(30);
 652
 653         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 654             c->set_APDU_yazlog(1);
 655         else
 656             c->set_APDU_yazlog(0);
 657     }
 658     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 659     return c;
 660 }
 661 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 662 {
 663     int i;
 664     for (i = 0; i<num; i++)
 665     {
 666         oident *ent;
 667         Z_DefaultDiagFormat *r;
 668         Z_DiagRec *p = pp[i];
 669         if (p->which != Z_DiagRec_defaultFormat)
 670         {
 671             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 672             return;
 673         }
 674         else
 675             r = p->u.defaultFormat;
 676         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 677             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 678             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 679         switch (r->which)
 680         {
 681         case Z_DefaultDiagFormat_v2Addinfo:
 682             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 683                     m_session_str,
 684                     *r->condition, diagbib1_str(*r->condition),
 685                     r->u.v2Addinfo);
 686             break;
 687         case Z_DefaultDiagFormat_v3Addinfo:
 688             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 689                     m_session_str,
 690                     *r->condition, diagbib1_str(*r->condition),
 691                     r->u.v3Addinfo);
 692             break;
 693         }
 694     }
 695 }
 696
 697 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 698 {
 699     if (!m_stylesheet_xsp || p->num_records <= 0)
 700         return 0;  /* no XSLT to be done ... */
 701
 702     m_stylesheet_offset = 0;
 703     m_stylesheet_nprl = p;
 704     m_stylesheet_apdu = apdu;
 705     timeout(0);
 706     return 1;
 707 }
 708
 709 void Yaz_Proxy::convert_xsl_delay()
 710 {
 711 #if HAVE_XSLT
 712     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 713     if (npr->which == Z_NamePlusRecord_databaseRecord)
 714     {
 715         Z_External *r = npr->u.databaseRecord;
 716         if (r->which == Z_External_octet)
 717         {
 718 #if 0
 719             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 720 #endif
 721             xmlDocPtr res, doc = xmlParseMemory(
 722                 (char*) r->u.octet_aligned->buf,
 723                 r->u.octet_aligned->len);
 724
 725
 726             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 727                     m_session_str, m_stylesheet_offset);
 728             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 729                                       doc, 0);
 730
 731             if (res)
 732             {
 733                 xmlChar *out_buf;
 734                 int out_len;
 735                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 736
 737                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 738                     u.databaseRecord =
 739                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 740                                  (char*) out_buf, out_len);
 741                 xmlFree(out_buf);
 742                 xmlFreeDoc(res);
 743             }
 744
 745             xmlFreeDoc(doc);
 746         }
 747     }
 748 #endif
 749     m_stylesheet_offset++;
 750     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 751     {
 752         m_stylesheet_nprl = 0;
 753 #if HAVE_XSLT
 754         if (m_stylesheet_xsp)
 755             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 756 #endif
 757         m_stylesheet_xsp = 0;
 758         timeout(m_client_idletime);
 759         send_PDU_convert(m_stylesheet_apdu);
 760     }
 761     else
 762         timeout(0);
 763 }
 764
 765 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 766 {
 767     if (m_frontend_type != VAL_NONE)
 768     {
 769         int i;
 770         for (i = 0; i < p->num_records; i++)
 771         {
 772             Z_NamePlusRecord *npr = p->records[i];
 773             if (npr->which == Z_NamePlusRecord_databaseRecord)
 774             {
 775                 Z_External *r = npr->u.databaseRecord;
 776                 if (r->which == Z_External_octet)
 777                 {
 778 #if HAVE_USEMARCON
 779                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 780                     {
 781                         if (!m_usemarcon->m_stage1)
 782                         {
 783                             m_usemarcon->m_stage1 = new CDetails();
 784                         }
 785                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 786                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 787                         int res = m_usemarcon->m_stage1->Start();
 788                         if (res == 0)
 789                         {
 790                             char *converted;
 791                             int convlen;
 792                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 793                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 794                             {
 795                                 if (!m_usemarcon->m_stage2)
 796                                 {
 797                                     m_usemarcon->m_stage2 = new CDetails();
 798                                 }
 799                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 800                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 801                                 res = m_usemarcon->m_stage2->Start();
 802                                 if (res == 0)
 803                                 {
 804                                     free(converted);
 805                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 806                                 }
 807                                 else
 808                                 {
 809                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 810                                 }
 811                             }
 812                             npr->u.databaseRecord =
 813                                 z_ext_record(odr_encode(),
 814                                              m_frontend_type,
 815                                              converted,
 816                                              strlen(converted));
 817                             free(converted);
 818                         }
 819                         else
 820                         {
 821                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 822                         }
 823                         continue;
 824                     }
 825 #endif
 826 /* HAVE_USEMARCON */
 827                     npr->u.databaseRecord =
 828                         z_ext_record(odr_encode(),
 829                                      m_frontend_type,
 830                                      (char*) r->u.octet_aligned->buf,
 831                                      r->u.octet_aligned->len);
 832                 }
 833             }
 834         }
 835     }
 836 }
 837
 838 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
 839                                         const char *backend_charset)
 840 {
 841     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
 842     int sel =   m_charset_converter->get_client_charset_selected();
 843     const char *client_record_charset =
 844         m_charset_converter->get_client_query_charset();
 845     if (sel && backend_charset && client_record_charset &&
 846         strcmp(backend_charset, client_record_charset))
 847     {
 848         int i;
 849         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
 850                                         backend_charset);
 851         yaz_marc_t mt = yaz_marc_create();
 852         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
 853         yaz_marc_iconv(mt, cd);
 854         for (i = 0; i < p->num_records; i++)
 855         {
 856             Z_NamePlusRecord *npr = p->records[i];
 857             if (npr->which == Z_NamePlusRecord_databaseRecord)
 858             {
 859                 Z_External *r = npr->u.databaseRecord;
 860                 oident *ent = oid_getentbyoid(r->direct_reference);
 861                 if (!ent || ent->value == VAL_NONE)
 862                     continue;
 863
 864                 if (ent->value == VAL_SUTRS)
 865                 {
 866                     WRBUF w = wrbuf_alloc();
 867
 868                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
 869                                       r->u.octet_aligned->len);
 870                     npr->u.databaseRecord =
 871                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
 872                                      wrbuf_len(w));
 873                     wrbuf_free(w, 1);
 874                 }
 875                 else if (ent->value == VAL_TEXT_XML)
 876                 {
 877                     ;
 878                 }
 879                 else if (r->which == Z_External_octet)
 880                 {
 881                     int rlen;
 882                     char *result;
 883                     if (yaz_marc_decode_buf(mt,
 884                                             (char*) r->u.octet_aligned->buf,
 885                                             r->u.octet_aligned->len,
 886                                             &result, &rlen))
 887                     {
 888                         npr->u.databaseRecord =
 889                             z_ext_record(odr_encode(), ent->value, result, rlen);
 890                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
 891                                 m_session_str);
 892                     }
 893                 }
 894             }
 895         }
 896         if (cd)
 897             yaz_iconv_close(cd);
 898         yaz_marc_destroy(mt);
 899     }
 900     else
 901     {
 902         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
 903     }
 904 }
 905
 906 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
 907                                    const char *backend_charset)
 908 {
 909     int i;
 910     if (!backend_charset)
 911         backend_charset = "MARC-8";
 912     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
 913     yaz_marc_t mt = yaz_marc_create();
 914     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 915     yaz_marc_iconv(mt, cd);
 916     for (i = 0; i < p->num_records; i++)
 917     {
 918         Z_NamePlusRecord *npr = p->records[i];
 919         if (npr->which == Z_NamePlusRecord_databaseRecord)
 920         {
 921             Z_External *r = npr->u.databaseRecord;
 922             if (r->which == Z_External_OPAC)
 923             {
 924                 WRBUF w = wrbuf_alloc();
 925
 926                 yaz_display_OPAC(w, r->u.opac, 0);
 927                 npr->u.databaseRecord = z_ext_record(
 928                     odr_encode(), VAL_TEXT_XML,
 929                     wrbuf_buf(w), wrbuf_len(w)
 930                     );
 931                 wrbuf_free(w, 1);
 932             }
 933             else if (r->which == Z_External_octet)
 934             {
 935                 int rlen;
 936                 char *result;
 937                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 938                                         r->u.octet_aligned->len,
 939                                         &result, &rlen))
 940                 {
 941                     npr->u.databaseRecord =
 942                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
 943                 }
 944             }
 945         }
 946     }
 947     if (cd)
 948         yaz_iconv_close(cd);
 949     yaz_marc_destroy(mt);
 950 }
 951
 952 void Yaz_Proxy::logtime()
 953 {
 954 #if HAVE_GETTIMEOFDAY
 955     struct timeval *tv = (struct timeval*) m_time_tv;
 956     if (tv->tv_sec)
 957     {
 958         struct timeval tv1;
 959         gettimeofday(&tv1, 0);
 960         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
 961             (tv1.tv_usec - tv->tv_usec);
 962         if (diff >= 0)
 963             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 964                     diff/1000000, (diff/1000)%1000);
 965     }
 966     tv->tv_sec = 0;
 967     tv->tv_usec = 0;
 968 #endif
 969 }
 970
 971 int Yaz_Proxy::send_http_response(int code)
 972 {
 973     ODR o = odr_encode();
 974     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 975     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 976     if (m_http_version)
 977         hres->version = odr_strdup(o, m_http_version);
 978     if (m_http_keepalive)
 979         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 980     else
 981         timeout(0);
 982
 983     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 984     {
 985         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
 986                  gdu_name(gdu));
 987     }
 988     int len;
 989     int r = send_GDU(gdu, &len);
 990     m_bytes_sent += len;
 991     m_bw_stat.add_bytes(len);
 992     logtime();
 993     return r;
 994 }
 995
 996 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 997 {
 998     ODR o = odr_encode();
 999     const char *ctype = "text/xml";
1000     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
1001     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1002     if (m_http_version)
1003         hres->version = odr_strdup(o, m_http_version);
1004     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1005     if (m_http_keepalive)
1006         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1007     else
1008         timeout(0);
1009
1010     static Z_SOAP_Handler soap_handlers[2] = {
1011 #if HAVE_XSLT
1012         {"http://www.loc.gov/zing/srw/", 0,
1013          (Z_SOAP_fun) yaz_srw_codec},
1014 #endif
1015         {0, 0, 0}
1016     };
1017
1018     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1019     soap_package->which = Z_SOAP_generic;
1020     soap_package->u.generic =
1021         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1022     soap_package->u.generic->no = 0;
1023     soap_package->u.generic->ns = soap_handlers[0].ns;
1024     soap_package->u.generic->p = (void *) srw_pdu;
1025     soap_package->ns = m_soap_ns;
1026     z_soap_codec_enc_xsl(o, &soap_package,
1027                          &hres->content_buf, &hres->content_len,
1028                          soap_handlers, 0, m_s2z_stylesheet);
1029     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1030     {
1031         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1032                  gdu_name(gdu));
1033     }
1034     int len;
1035     int r = send_GDU(gdu, &len);
1036     m_bytes_sent += len;
1037     m_bw_stat.add_bytes(len);
1038     logtime();
1039     return r;
1040 }
1041
1042 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1043 {
1044     ODR o = odr_encode();
1045     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1046     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1047
1048     srw_res->num_diagnostics = 1;
1049     srw_res->diagnostics = (Z_SRW_diagnostic *)
1050         odr_malloc(o, sizeof(*srw_res->diagnostics));
1051     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
1052     return send_srw_response(srw_pdu);
1053 }
1054
1055 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1056                              Z_DefaultDiagFormat *ddf)
1057 {
1058     int bib1_code = *ddf->condition;
1059     if (bib1_code == 109)
1060         return 404;
1061     srw_res->num_diagnostics = 1;
1062     srw_res->diagnostics = (Z_SRW_diagnostic *)
1063         odr_malloc(o, sizeof(*srw_res->diagnostics));
1064     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1065                           yaz_diag_bib1_to_srw(*ddf->condition),
1066                           ddf->u.v2Addinfo);
1067     return 0;
1068 }
1069
1070 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1071 {
1072     ODR o = odr_encode();
1073     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1074     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1075
1076     srw_res->numberOfRecords = odr_intdup (o, hits);
1077     if (records && records->which == Z_Records_DBOSD)
1078     {
1079         srw_res->num_records =
1080             records->u.databaseOrSurDiagnostics->num_records;
1081         int i;
1082         srw_res->records = (Z_SRW_record *)
1083             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1084         for (i = 0; i < srw_res->num_records; i++)
1085         {
1086             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1087             if (npr->which != Z_NamePlusRecord_databaseRecord)
1088             {
1089                 srw_res->records[i].recordSchema = "diagnostic";
1090                 srw_res->records[i].recordPacking = m_s2z_packing;
1091                 srw_res->records[i].recordData_buf = "67";
1092                 srw_res->records[i].recordData_len = 2;
1093                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1094                 continue;
1095             }
1096             Z_External *r = npr->u.databaseRecord;
1097             oident *ent = oid_getentbyoid(r->direct_reference);
1098             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1099             {
1100                 srw_res->records[i].recordSchema = m_schema;
1101                 srw_res->records[i].recordPacking = m_s2z_packing;
1102                 srw_res->records[i].recordData_buf = (char*)
1103                     r->u.octet_aligned->buf;
1104                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1105                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1106             }
1107             else
1108             {
1109                 srw_res->records[i].recordSchema = "diagnostic";
1110                 srw_res->records[i].recordPacking = m_s2z_packing;
1111                 srw_res->records[i].recordData_buf = "67";
1112                 srw_res->records[i].recordData_len = 2;
1113                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1114             }
1115         }
1116     }
1117     if (records && records->which == Z_Records_NSD)
1118     {
1119         int http_code;
1120         http_code = z_to_srw_diag(odr_encode(), srw_res,
1121                                    records->u.nonSurrogateDiagnostic);
1122         if (http_code)
1123             return send_http_response(http_code);
1124     }
1125     return send_srw_response(srw_pdu);
1126
1127 }
1128
1129 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1130                                         int num_diagnostics)
1131 {
1132     Yaz_ProxyConfig *cfg = check_reconfigure();
1133     if (cfg)
1134     {
1135         int len;
1136         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1137                                        m_s2z_database, &len);
1138         if (b)
1139         {
1140             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1141             Z_SRW_explainResponse *er = res->u.explain_response;
1142
1143             er->record.recordData_buf = b;
1144             er->record.recordData_len = len;
1145             er->record.recordPacking = m_s2z_packing;
1146             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1147
1148             er->diagnostics = diagnostics;
1149             er->num_diagnostics = num_diagnostics;
1150             return send_srw_response(res);
1151         }
1152     }
1153     return send_http_response(404);
1154 }
1155
1156 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1157 {
1158     if (m_http_version)
1159     {
1160         if (apdu->which == Z_APDU_initResponse)
1161         {
1162             Z_InitResponse *res = apdu->u.initResponse;
1163             if (*res->result == 0)
1164             {
1165                 send_to_srw_client_error(3, 0);
1166             }
1167             else if (!m_s2z_search_apdu)
1168             {
1169                 send_srw_explain_response(0, 0);
1170             }
1171             else
1172             {
1173                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1174             }
1175         }
1176         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1177         {
1178             m_s2z_search_apdu = 0;
1179             Z_SearchResponse *res = apdu->u.searchResponse;
1180             m_s2z_hit_count = *res->resultCount;
1181             if (res->records && res->records->which == Z_Records_NSD)
1182             {
1183                 send_to_srw_client_ok(0, res->records, 1);
1184             }
1185             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1186             {
1187                 // adjust
1188                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1189
1190                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1191                 {
1192                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1193                         > m_s2z_hit_count)
1194                         *pr->numberOfRecordsRequested =
1195                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1196                 }
1197                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1198             }
1199             else
1200             {
1201                 m_s2z_present_apdu = 0;
1202                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1203             }
1204         }
1205         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1206         {
1207             int start =
1208                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1209
1210             m_s2z_present_apdu = 0;
1211             Z_PresentResponse *res = apdu->u.presentResponse;
1212             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1213         }
1214     }
1215     else
1216     {
1217         int len = 0;
1218         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1219             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1220                      apdu_name(apdu));
1221         int r = send_Z_PDU(apdu, &len);
1222         m_bytes_sent += len;
1223         m_bw_stat.add_bytes(len);
1224         logtime();
1225         return r;
1226     }
1227     return 0;
1228 }
1229
1230 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1231 {
1232     int kill_session = 0;
1233     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1234
1235     if (new_id)
1236         *new_id = m_referenceId;
1237
1238     if (apdu->which == Z_APDU_searchResponse)
1239     {
1240         Z_SearchResponse *sr = apdu->u.searchResponse;
1241         Z_Records *p = sr->records;
1242         if (p && p->which == Z_Records_NSD)
1243         {
1244             Z_DiagRec dr, *dr_p = &dr;
1245             dr.which = Z_DiagRec_defaultFormat;
1246             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1247
1248             *sr->searchStatus = 0;
1249             display_diagrecs(&dr_p, 1);
1250         }
1251         else
1252         {
1253             if (p && p->which == Z_Records_DBOSD)
1254             {
1255                 if (m_backend_type
1256 #if HAVE_USEMARCON
1257                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1258 #endif
1259                     )
1260                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1261                 if (m_marcxml_mode == marcxml)
1262                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1263                                        m_backend_charset);
1264                 else
1265                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1266                                             m_backend_charset);
1267                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1268                     return 0;
1269
1270             }
1271             if (sr->resultCount)
1272             {
1273                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1274                         *sr->resultCount);
1275                 if (*sr->resultCount < 0)
1276                 {
1277                     m_invalid_session = 1;
1278                     kill_session = 1;
1279
1280                     *sr->searchStatus = 0;
1281                     sr->records =
1282                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1283                     *sr->resultCount = 0;
1284                 }
1285             }
1286         }
1287     }
1288     else if (apdu->which == Z_APDU_presentResponse)
1289     {
1290         Z_PresentResponse *sr = apdu->u.presentResponse;
1291         Z_Records *p = sr->records;
1292         if (p && p->which == Z_Records_NSD)
1293         {
1294             Z_DiagRec dr, *dr_p = &dr;
1295             dr.which = Z_DiagRec_defaultFormat;
1296             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1297             if (*sr->presentStatus == Z_PresentStatus_success)
1298                 *sr->presentStatus = Z_PresentStatus_failure;
1299             display_diagrecs(&dr_p, 1);
1300         }
1301         if (p && p->which == Z_Records_DBOSD)
1302         {
1303             if (m_backend_type
1304 #if HAVE_USEMARCON
1305                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1306 #endif
1307                 )
1308                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1309             if (m_marcxml_mode == marcxml)
1310                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1311                                    m_backend_charset);
1312             else
1313                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1314                                         m_backend_charset);
1315             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1316                 return 0;
1317         }
1318     }
1319     else if (apdu->which == Z_APDU_initResponse)
1320     {
1321         //Get and check negotiation record
1322         //from init response.
1323         handle_charset_lang_negotiation(apdu);
1324
1325         if (m_initRequest_options)
1326         {
1327             Z_Options *nopt =
1328                 (Odr_bitmask *)odr_malloc(odr_encode(),
1329                                           sizeof(Odr_bitmask));
1330             ODR_MASK_ZERO(nopt);
1331
1332             int i;
1333             for (i = 0; i<24; i++)
1334                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1335                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1336                     ODR_MASK_SET(nopt, i);
1337             apdu->u.initResponse->options = nopt;
1338         }
1339         if (m_initRequest_version)
1340         {
1341             Z_ProtocolVersion *nopt =
1342                 (Odr_bitmask *)odr_malloc(odr_encode(),
1343                                           sizeof(Odr_bitmask));
1344             ODR_MASK_ZERO(nopt);
1345
1346             int i;
1347             for (i = 0; i<8; i++)
1348                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1349                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1350                     ODR_MASK_SET(nopt, i);
1351             apdu->u.initResponse->protocolVersion = nopt;
1352         }
1353         apdu->u.initResponse->preferredMessageSize =
1354             odr_intdup(odr_encode(),
1355                        m_client->m_initResponse_preferredMessageSize >
1356                        m_initRequest_preferredMessageSize ?
1357                        m_initRequest_preferredMessageSize :
1358                        m_client->m_initResponse_preferredMessageSize);
1359         apdu->u.initResponse->maximumRecordSize =
1360             odr_intdup(odr_encode(),
1361                        m_client->m_initResponse_maximumRecordSize >
1362                        m_initRequest_maximumRecordSize ?
1363                        m_initRequest_maximumRecordSize :
1364                        m_client->m_initResponse_maximumRecordSize);
1365     }
1366
1367     int r = send_PDU_convert(apdu);
1368     if (r)
1369         return r;
1370     if (kill_session)
1371     {
1372         delete m_client;
1373         m_client = 0;
1374         m_parent->pre_init();
1375     }
1376     return r;
1377 }
1378
1379 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1380 {
1381     int len = 0;
1382     const char *apdu_name_tmp = apdu_name(apdu);
1383     int r = send_Z_PDU(apdu, &len);
1384     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1385         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1386                  get_session_str(),
1387                  apdu_name_tmp, get_hostname(), len);
1388     m_bytes_sent += len;
1389     return r;
1390 }
1391
1392 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1393 {
1394     if (apdu->which == Z_APDU_presentRequest)
1395     {
1396         Z_PresentRequest *pr = apdu->u.presentRequest;
1397         int toget = *pr->numberOfRecordsRequested;
1398         int start = *pr->resultSetStartPoint;
1399
1400         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1401                 pr->resultSetId, start, toget);
1402
1403         if (*m_parent->m_optimize == '0')
1404             return apdu;
1405
1406         if (!m_client->m_last_resultSetId)
1407         {
1408             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1409             new_apdu->u.presentResponse->records =
1410                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1411                                                pr->resultSetId);
1412             send_to_client(new_apdu);
1413             return 0;
1414         }
1415         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1416         {
1417             if (start+toget-1 > m_client->m_last_resultCount)
1418             {
1419                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1420                 new_apdu->u.presentResponse->records =
1421                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1422                 send_to_client(new_apdu);
1423                 return 0;
1424             }
1425             Z_NamePlusRecordList *npr;
1426 #if 0
1427             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1428                     m_session_str, start, toget, yaz_z3950oid_to_str(
1429                         pr->preferredRecordSyntax, &oclass));
1430 #endif
1431             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1432                                           pr->preferredRecordSyntax,
1433                                           pr->recordComposition))
1434             {
1435                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1436                          m_session_str);
1437                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1438                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1439
1440                 new_apdu->u.presentResponse->numberOfRecordsReturned
1441                     = odr_intdup(odr_encode(), toget);
1442
1443                 new_apdu->u.presentResponse->records = (Z_Records*)
1444                     odr_malloc(odr_encode(), sizeof(Z_Records));
1445                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1446                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1447                 new_apdu->u.presentResponse->nextResultSetPosition =
1448                     odr_intdup(odr_encode(), start+toget);
1449
1450                 send_to_client(new_apdu);
1451                 return 0;
1452             }
1453         }
1454     }
1455
1456     if (apdu->which != Z_APDU_searchRequest)
1457         return apdu;
1458     Z_SearchRequest *sr = apdu->u.searchRequest;
1459     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1460     Yaz_Z_Databases this_databases;
1461
1462     this_databases.set(sr->num_databaseNames, (const char **)
1463                        sr->databaseNames);
1464
1465     this_query->set_Z_Query(sr->query);
1466
1467     char query_str[120];
1468     this_query->print(query_str, sizeof(query_str)-1);
1469     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1470
1471     if (*m_parent->m_optimize != '0' &&
1472         m_client->m_last_ok && m_client->m_last_query &&
1473         m_client->m_last_query->match(this_query) &&
1474         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1475         m_client->m_last_databases.match(this_databases))
1476     {
1477         delete this_query;
1478         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1479             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1480         {
1481             Z_NamePlusRecordList *npr;
1482             int toget = *sr->mediumSetPresentNumber;
1483             Z_RecordComposition *comp = 0;
1484
1485             if (toget > m_client->m_last_resultCount)
1486                 toget = m_client->m_last_resultCount;
1487
1488             if (sr->mediumSetElementSetNames)
1489             {
1490                 comp = (Z_RecordComposition *)
1491                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1492                 comp->which = Z_RecordComp_simple;
1493                 comp->u.simple = sr->mediumSetElementSetNames;
1494             }
1495
1496             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1497                                           sr->preferredRecordSyntax, comp))
1498             {
1499                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1500                          m_session_str);
1501                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1502                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1503                 new_apdu->u.searchResponse->resultCount =
1504                     &m_client->m_last_resultCount;
1505
1506                 new_apdu->u.searchResponse->numberOfRecordsReturned
1507                     = odr_intdup(odr_encode(), toget);
1508
1509                 new_apdu->u.searchResponse->presentStatus =
1510                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1511                 new_apdu->u.searchResponse->records = (Z_Records*)
1512                     odr_malloc(odr_encode(), sizeof(Z_Records));
1513                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1514                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1515                 new_apdu->u.searchResponse->nextResultSetPosition =
1516                     odr_intdup(odr_encode(), toget+1);
1517                 send_to_client(new_apdu);
1518                 return 0;
1519             }
1520             else
1521             {
1522                 // medium Set
1523                 // send present request (medium size)
1524                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1525                          m_session_str);
1526
1527                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1528                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1529                 pr->referenceId = sr->referenceId;
1530                 pr->resultSetId = sr->resultSetName;
1531                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1532                 *pr->numberOfRecordsRequested = toget;
1533                 pr->recordComposition = comp;
1534                 m_client->m_sr_transform = 1;
1535                 return new_apdu;
1536             }
1537         }
1538         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1539             m_client->m_last_resultCount <= 0)
1540         {
1541             // large set. Return pseudo-search response immediately
1542             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1543                      m_session_str);
1544             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1545             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1546             new_apdu->u.searchResponse->resultCount =
1547                 &m_client->m_last_resultCount;
1548             send_to_client(new_apdu);
1549             return 0;
1550         }
1551         else
1552         {
1553             Z_NamePlusRecordList *npr;
1554             int toget = m_client->m_last_resultCount;
1555             Z_RecordComposition *comp = 0;
1556             // small set
1557             // send a present request (small set)
1558
1559             if (sr->smallSetElementSetNames)
1560             {
1561                 comp = (Z_RecordComposition *)
1562                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1563                 comp->which = Z_RecordComp_simple;
1564                 comp->u.simple = sr->smallSetElementSetNames;
1565             }
1566
1567             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1568                                           sr->preferredRecordSyntax, comp))
1569             {
1570                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1571                          m_session_str);
1572                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1573                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1574                 new_apdu->u.searchResponse->resultCount =
1575                     &m_client->m_last_resultCount;
1576
1577                 new_apdu->u.searchResponse->numberOfRecordsReturned
1578                     = odr_intdup(odr_encode(), toget);
1579
1580                 new_apdu->u.searchResponse->presentStatus =
1581                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1582                 new_apdu->u.searchResponse->records = (Z_Records*)
1583                     odr_malloc(odr_encode(), sizeof(Z_Records));
1584                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1585                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1586                 new_apdu->u.searchResponse->nextResultSetPosition =
1587                     odr_intdup(odr_encode(), toget+1);
1588                 send_to_client(new_apdu);
1589                 return 0;
1590             }
1591             else
1592             {
1593                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1594                          m_session_str);
1595                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1596                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1597                 pr->referenceId = sr->referenceId;
1598                 pr->resultSetId = sr->resultSetName;
1599                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1600                 *pr->numberOfRecordsRequested = toget;
1601                 pr->recordComposition = comp;
1602                 m_client->m_sr_transform = 1;
1603                 return new_apdu;
1604             }
1605         }
1606     }
1607     else  // query doesn't match
1608     {
1609         delete m_client->m_last_query;
1610         m_client->m_last_query = this_query;
1611         m_client->m_last_ok = 0;
1612         m_client->m_cache.clear();
1613         m_client->m_resultSetStartPoint = 0;
1614
1615         xfree (m_client->m_last_resultSetId);
1616         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1617
1618         m_client->m_last_databases.set(sr->num_databaseNames,
1619                                        (const char **) sr->databaseNames);
1620     }
1621     return apdu;
1622 }
1623
1624
1625 void Yaz_Proxy::inc_request_no()
1626 {
1627     char *cp = strchr(m_session_str, ' ');
1628     m_request_no++;
1629     if (cp)
1630         sprintf(cp+1, "%d ", m_request_no);
1631 }
1632
1633 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1634 {
1635     inc_request_no();
1636
1637     m_bytes_recv += len;
1638
1639     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1640         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1641                  m_session_str, gdu_name(apdu), len);
1642
1643     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1644         shutdown();
1645
1646     m_bw_stat.add_bytes(len);
1647     m_pdu_stat.add_bytes(1);
1648
1649 #if HAVE_GETTIMEOFDAY
1650     gettimeofday((struct timeval *) m_time_tv, 0);
1651 #endif
1652
1653     int bw_total = m_bw_stat.get_total();
1654     int pdu_total = m_pdu_stat.get_total();
1655
1656     int reduce = 0;
1657     if (m_bw_max)
1658     {
1659         if (bw_total > m_bw_max)
1660         {
1661             reduce = (bw_total/m_bw_max);
1662         }
1663     }
1664     if (m_pdu_max)
1665     {
1666         if (pdu_total > m_pdu_max)
1667         {
1668             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1669             reduce = (reduce > nreduce) ? reduce : nreduce;
1670         }
1671     }
1672     m_http_version = 0;
1673     if (reduce)
1674     {
1675         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1676                 m_session_str, reduce, bw_total, pdu_total,
1677                 m_bw_max, m_pdu_max);
1678
1679         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1680         timeout(reduce);       // call us reduce seconds later
1681     }
1682     else if (apdu->which == Z_GDU_Z3950)
1683         handle_incoming_Z_PDU(apdu->u.z3950);
1684     else if (apdu->which == Z_GDU_HTTP_Request)
1685         handle_incoming_HTTP(apdu->u.HTTP_Request);
1686 }
1687
1688 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1689 {
1690     if (m_max_record_retrieve)
1691     {
1692         if (apdu->which == Z_APDU_presentRequest)
1693         {
1694             Z_PresentRequest *pr = apdu->u.presentRequest;
1695             if (pr->numberOfRecordsRequested &&
1696                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1697                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1698         }
1699     }
1700 }
1701 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1702 {
1703     if (apdu->which == Z_APDU_initRequest)
1704     {
1705         yaz_log(YLOG_LOG, "%shandle_charset_lang_negotiation",
1706                 m_session_str);
1707         if (m_initRequest_options &&
1708             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1709             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1710         {
1711             // There is no negotiation proposal from
1712             // client's side. OK. The proxy negotiation
1713             // in use, only.
1714             Z_InitRequest *initRequest = apdu->u.initRequest;
1715             Z_OtherInformation **otherInfo;
1716             Z_OtherInformationUnit *oi;
1717             get_otherInfoAPDU(apdu, &otherInfo);
1718             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1719             if (oi)
1720             {
1721                 ODR_MASK_SET(initRequest->options,
1722                     Z_Options_negotiationModel);
1723                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1724                 oi->information.externallyDefinedInfo =
1725                 yaz_set_proposal_charneg(odr_encode(),
1726                     (const char**)&m_proxy_negotiation_charset,
1727                     m_proxy_negotiation_charset ? 1:0,
1728                     (const char**)&m_proxy_negotiation_lang,
1729                     m_proxy_negotiation_lang ? 1:0,
1730                     1);
1731             }
1732         }
1733         else if (m_initRequest_options &&
1734                  ODR_MASK_GET(m_initRequest_options,
1735                               Z_Options_negotiationModel) &&
1736                  m_charset_converter->get_target_query_charset())
1737         {
1738             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1739                     m_session_str,
1740                     m_charset_converter->get_target_query_charset());
1741             Z_InitRequest *initRequest = apdu->u.initRequest;
1742             Z_CharSetandLanguageNegotiation *negotiation =
1743                 yaz_get_charneg_record (initRequest->otherInfo);
1744             if (negotiation &&
1745                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1746             {
1747                 NMEM nmem = nmem_create();
1748                 char **charsets = 0;
1749                 int num_charsets = 0;
1750                 char **langs = 0;
1751                 int num_langs = 0;
1752                 int selected = 0;
1753                 yaz_get_proposal_charneg (nmem, negotiation,
1754                                           &charsets, &num_charsets,
1755                                           &langs, &num_langs, &selected);
1756                 int i;
1757                 for (i = 0; i<num_charsets; i++)
1758                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
1759                             charsets[i]);
1760                 for (i = 0; i<num_langs; i++)
1761                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
1762                             langs[i]);
1763
1764                 const char *t_charset =
1765                     m_charset_converter->get_target_query_charset();
1766                 // sweep through charsets and pick the first supported
1767                 // conversion
1768                 for (i = 0; i<num_charsets; i++)
1769                 {
1770                     const char *c_charset = charsets[i];
1771                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
1772                         break;
1773                 }
1774                 if (i != num_charsets)
1775                 {
1776                     // got one .. set up ODR for reverse direction
1777                     const char *c_charset = charsets[i];
1778                     odr_set_charset(odr_encode(), c_charset, t_charset);
1779                     m_charset_converter->set_client_query_charset(c_charset);
1780                     m_charset_converter->set_client_charset_selected(selected);
1781                 }
1782                 nmem_destroy(nmem);
1783                 ODR_MASK_CLEAR(m_initRequest_options,
1784                                Z_Options_negotiationModel);
1785                 yaz_del_charneg_record(&initRequest->otherInfo);
1786             }
1787             else
1788             {
1789                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
1790                         m_session_str);
1791             }
1792         }
1793     }
1794     else if (apdu->which == Z_APDU_initResponse)
1795     {
1796         Z_InitResponse *initResponse = apdu->u.initResponse;
1797         Z_OtherInformation **otherInfo;
1798
1799         if (ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel))
1800         {
1801             char *charset = 0;
1802             char *lang = 0;
1803             int selected = 0;
1804
1805             get_otherInfoAPDU(apdu, &otherInfo);
1806
1807             if (!otherInfo && !(*otherInfo))
1808                 return;
1809
1810             Z_CharSetandLanguageNegotiation *charneg =
1811                 yaz_get_charneg_record(*otherInfo);
1812
1813             if (!charneg)
1814                 return;
1815
1816             yaz_get_response_charneg(m_referenceId_mem, charneg,
1817                 &charset, &lang, &selected);
1818
1819             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
1820                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
1821
1822             if (m_initRequest_options &&
1823                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
1824             {
1825                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
1826                     m_session_str);
1827             }
1828             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1829             {
1830                 // negotiation-charset, negotiation-lang
1831                 // elements of config file in use.
1832
1833                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
1834                     m_session_str);
1835
1836                 // clear negotiation option.
1837                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
1838
1839                 // Delete negotiation (charneg-3) entry.
1840                 yaz_del_charneg_record(otherInfo);
1841             }
1842         }
1843         else
1844         {
1845             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1846             {
1847                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
1848                         m_session_str);
1849             }
1850             else if (m_charset_converter->get_client_query_charset())
1851             {
1852                 Z_OtherInformation **otherInfo;
1853                 Z_OtherInformationUnit *oi;
1854                 get_otherInfoAPDU(apdu, &otherInfo);
1855                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1856                 if (oi)
1857                 {
1858                     ODR_MASK_SET(initResponse->options,
1859                                  Z_Options_negotiationModel);
1860                     ODR_MASK_SET(m_initRequest_options,
1861                                  Z_Options_negotiationModel);
1862
1863                     oi->which = Z_OtherInfo_externallyDefinedInfo;
1864                     oi->information.externallyDefinedInfo =
1865                         yaz_set_response_charneg(
1866                             odr_encode(),
1867                             m_charset_converter->get_client_query_charset(),
1868                             0 /* no lang */,
1869                             m_charset_converter->get_client_charset_selected());
1870                 }
1871             }
1872         }
1873     }
1874 }
1875
1876 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1877                                                      int error,
1878                                                      const char *addinfo)
1879 {
1880     Z_Records *rec = (Z_Records *)
1881         odr_malloc (odr, sizeof(*rec));
1882     int *err = (int *)
1883         odr_malloc (odr, sizeof(*err));
1884     Z_DiagRec *drec = (Z_DiagRec *)
1885         odr_malloc (odr, sizeof(*drec));
1886     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1887         odr_malloc (odr, sizeof(*dr));
1888     *err = error;
1889     rec->which = Z_Records_NSD;
1890     rec->u.nonSurrogateDiagnostic = dr;
1891     dr->diagnosticSetId =
1892         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1893     dr->condition = err;
1894     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1895     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1896     return rec;
1897 }
1898
1899 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1900 {
1901     if (apdu->which == Z_APDU_searchRequest &&
1902         apdu->u.searchRequest->query &&
1903         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1904         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1905     {
1906         Z_RPNQuery *rpnquery = 0;
1907         Z_SearchRequest *sr = apdu->u.searchRequest;
1908         char *addinfo = 0;
1909
1910         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
1911                 sr->query->u.type_104->u.cql);
1912
1913         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1914                                           &rpnquery, odr_encode(),
1915                                           &addinfo);
1916         if (r == -3)
1917             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
1918         else if (r)
1919         {
1920             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1921             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1922
1923             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1924             new_apdu->u.searchResponse->records =
1925                 create_nonSurrogateDiagnostics(odr_encode(),
1926                                                yaz_diag_srw_to_bib1(r),
1927                                                addinfo);
1928             *new_apdu->u.searchResponse->searchStatus = 0;
1929
1930             send_to_client(new_apdu);
1931
1932             return 0;
1933         }
1934         else
1935         {
1936             sr->query->which = Z_Query_type_1;
1937             sr->query->u.type_1 = rpnquery;
1938         }
1939         return apdu;
1940     }
1941     return apdu;
1942 }
1943
1944 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
1945 {
1946     if (apdu->which == Z_APDU_searchRequest &&
1947         apdu->u.searchRequest->query)
1948     {
1949         if (apdu->u.searchRequest->query->which == Z_Query_type_1
1950             || apdu->u.searchRequest->query->which == Z_Query_type_101)
1951         {
1952             if (m_http_version)
1953                 m_charset_converter->set_client_query_charset("UTF-8");
1954             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
1955             m_charset_converter->convert_type_1(rpnquery, odr_encode());
1956         }
1957     }
1958     return apdu;
1959 }
1960
1961
1962 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1963 {
1964     if (apdu->which == Z_APDU_searchRequest)
1965     {
1966         Z_SearchRequest *sr = apdu->u.searchRequest;
1967         int err = 0;
1968         char *addinfo = 0;
1969
1970         Yaz_ProxyConfig *cfg = check_reconfigure();
1971         if (cfg)
1972             err = cfg->check_query(odr_encode(), m_default_target,
1973                                    sr->query, &addinfo);
1974         if (err)
1975         {
1976             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1977
1978             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1979             new_apdu->u.searchResponse->records =
1980                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1981             *new_apdu->u.searchResponse->searchStatus = 0;
1982
1983             send_to_client(new_apdu);
1984
1985             return 0;
1986         }
1987     }
1988     return apdu;
1989 }
1990
1991 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
1992 {
1993     if (apdu->which != Z_APDU_initRequest)
1994         return 1;  // pass if no init request
1995     Z_InitRequest *req = apdu->u.initRequest;
1996
1997     Yaz_ProxyConfig *cfg = check_reconfigure();
1998     if (!cfg)
1999         return 1;  // pass if no config
2000
2001     int ret;
2002     if (req->idAuthentication == 0)
2003     {
2004         ret = cfg->client_authentication(m_default_target, 0, 0, 0);
2005     }
2006     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2007     {
2008         ret = cfg->client_authentication(m_default_target,
2009             req->idAuthentication->u.idPass->userId,
2010             req->idAuthentication->u.idPass->groupId,
2011             req->idAuthentication->u.idPass->password);
2012     }
2013     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2014     {
2015         char user[64], pass[64];
2016         *user = '\0';
2017         *pass = '\0';
2018         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2019         ret = cfg->client_authentication(m_default_target, user, 0, pass);
2020     }
2021     else
2022         ret = cfg->client_authentication(m_default_target, 0, 0, 0);
2023
2024     cfg->target_authentication(m_default_target, odr_encode(), req);
2025
2026     return ret;
2027 }
2028
2029 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2030 {
2031     m_marcxml_mode = none;
2032     if (apdu->which == Z_APDU_searchRequest)
2033     {
2034         Z_SearchRequest *sr = apdu->u.searchRequest;
2035         int err = 0;
2036         char *addinfo = 0;
2037         Yaz_ProxyConfig *cfg = check_reconfigure();
2038
2039         Z_RecordComposition rc_temp, *rc = 0;
2040         if (sr->smallSetElementSetNames)
2041         {
2042             rc_temp.which = Z_RecordComp_simple;
2043             rc_temp.u.simple = sr->smallSetElementSetNames;
2044             rc = &rc_temp;
2045         }
2046
2047         if (sr->preferredRecordSyntax)
2048         {
2049             struct oident *ent;
2050             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2051             m_frontend_type = ent->value;
2052         }
2053         else
2054             m_frontend_type = VAL_NONE;
2055
2056         char *stylesheet_name = 0;
2057         if (cfg)
2058             err = cfg->check_syntax(odr_encode(),
2059                                     m_default_target,
2060                                     sr->preferredRecordSyntax, rc,
2061                                     &addinfo, &stylesheet_name, &m_schema,
2062                                     &m_backend_type, &m_backend_charset,
2063                                     &m_usemarcon_ini_stage1,
2064                                     &m_usemarcon_ini_stage2);
2065         if (stylesheet_name)
2066         {
2067             m_parent->low_socket_close();
2068
2069 #if HAVE_XSLT
2070             if (m_stylesheet_xsp)
2071                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2072             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2073                                                        stylesheet_name);
2074 #endif
2075             m_stylesheet_offset = 0;
2076             xfree(stylesheet_name);
2077
2078             m_parent->low_socket_open();
2079         }
2080         if (err == -1)
2081         {
2082             sr->smallSetElementSetNames = 0;
2083             sr->mediumSetElementSetNames = 0;
2084             m_marcxml_mode = marcxml;
2085             if (m_backend_type)
2086             {
2087
2088                 sr->preferredRecordSyntax =
2089                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2090                                         m_backend_type);
2091             }
2092             else
2093                 sr->preferredRecordSyntax =
2094                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2095                                            VAL_USMARC);
2096         }
2097         else if (err)
2098         {
2099             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2100
2101             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2102             new_apdu->u.searchResponse->records =
2103                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2104             *new_apdu->u.searchResponse->searchStatus = 0;
2105
2106             send_to_client(new_apdu);
2107
2108             return 0;
2109         }
2110         else if (m_backend_type)
2111         {
2112             sr->preferredRecordSyntax =
2113                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2114         }
2115     }
2116     else if (apdu->which == Z_APDU_presentRequest)
2117     {
2118         Z_PresentRequest *pr = apdu->u.presentRequest;
2119         int err = 0;
2120         char *addinfo = 0;
2121         Yaz_ProxyConfig *cfg = check_reconfigure();
2122
2123         if (pr->preferredRecordSyntax)
2124         {
2125             struct oident *ent;
2126             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2127             m_frontend_type = ent->value;
2128         }
2129         else
2130             m_frontend_type = VAL_NONE;
2131
2132         char *stylesheet_name = 0;
2133         if (cfg)
2134             err = cfg->check_syntax(odr_encode(), m_default_target,
2135                                     pr->preferredRecordSyntax,
2136                                     pr->recordComposition,
2137                                     &addinfo, &stylesheet_name, &m_schema,
2138                                     &m_backend_type, &m_backend_charset,
2139                                     &m_usemarcon_ini_stage1,
2140                                     &m_usemarcon_ini_stage2
2141                                     );
2142         if (stylesheet_name)
2143         {
2144             m_parent->low_socket_close();
2145
2146 #if HAVE_XSLT
2147             if (m_stylesheet_xsp)
2148                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2149             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2150                                                        stylesheet_name);
2151 #endif
2152             m_stylesheet_offset = 0;
2153             xfree(stylesheet_name);
2154
2155             m_parent->low_socket_open();
2156         }
2157         if (err == -1)
2158         {
2159             pr->recordComposition = 0;
2160             m_marcxml_mode = marcxml;
2161             if (m_backend_type)
2162             {
2163
2164                 pr->preferredRecordSyntax =
2165                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2166                                         m_backend_type);
2167             }
2168             else
2169                 pr->preferredRecordSyntax =
2170                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2171                                            VAL_USMARC);
2172         }
2173         else if (err)
2174         {
2175             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2176
2177             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2178             new_apdu->u.presentResponse->records =
2179                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2180             *new_apdu->u.presentResponse->presentStatus =
2181                 Z_PresentStatus_failure;
2182
2183             send_to_client(new_apdu);
2184
2185             return 0;
2186         }
2187         else if (m_backend_type)
2188         {
2189             pr->preferredRecordSyntax =
2190                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2191         }
2192     }
2193     return apdu;
2194 }
2195
2196 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2197 {
2198     if (!schema)
2199         return 0;
2200     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2201         odr_malloc(o, sizeof(Z_ElementSetNames));
2202     esn->which = Z_ElementSetNames_generic;
2203     esn->u.generic = odr_strdup(o, schema);
2204     return esn;
2205 }
2206
2207 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2208 {
2209     const char *t = 0;
2210     Yaz_ProxyConfig *cfg = check_reconfigure();
2211     if (cfg)
2212         t = cfg->get_explain_name(db, backend_db);
2213
2214     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2215     {
2216         releaseClient();
2217     }
2218
2219     if (t)
2220     {
2221         xfree(m_default_target);
2222         m_default_target = xstrdup(t);
2223     }
2224 }
2225
2226 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2227 {
2228     struct stat sbuf;
2229     if (strcmp(hreq->method, "GET"))
2230         return 0;
2231     if (hreq->path[0] != '/')
2232     {
2233         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2234         return 0;
2235     }
2236     const char *cp = hreq->path;
2237     while (*cp)
2238     {
2239         if (*cp == '/' && strchr("/.", cp[1]))
2240         {
2241             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2242             return 0;
2243         }
2244         cp++;
2245     }
2246     const char *fname = hreq->path+1;
2247     if (stat(fname, &sbuf))
2248     {
2249         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2250         return 0;
2251     }
2252     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2253     {
2254         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2255         return 0;
2256     }
2257     if (sbuf.st_size > (off_t) 1000000)
2258     {
2259         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2260         return 0;
2261     }
2262
2263     ODR o = odr_encode();
2264     Yaz_ProxyConfig *cfg = check_reconfigure();
2265     const char *ctype = cfg->check_mime_type(fname);
2266     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2267     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2268     if (m_http_version)
2269         hres->version = odr_strdup(o, m_http_version);
2270     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2271     if (m_http_keepalive)
2272         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2273     else
2274         timeout(0);
2275
2276     hres->content_len = sbuf.st_size;
2277     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2278     FILE *f = fopen(fname, "rb");
2279     if (f)
2280     {
2281         fread(hres->content_buf, 1, hres->content_len, f);
2282         fclose(f);
2283     }
2284     else
2285     {
2286         return 0;
2287     }
2288     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2289     {
2290         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2291                  fname);
2292     }
2293     int len;
2294     send_GDU(gdu, &len);
2295     return 1;
2296 }
2297
2298 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2299 {
2300     if (m_s2z_odr_init)
2301     {
2302         odr_destroy(m_s2z_odr_init);
2303         m_s2z_odr_init = 0;
2304     }
2305     if (m_s2z_odr_search)
2306     {
2307         odr_destroy(m_s2z_odr_search);
2308         m_s2z_odr_search = 0;
2309     }
2310
2311     m_http_keepalive = 0;
2312     m_http_version = 0;
2313     if (!strcmp(hreq->version, "1.0"))
2314     {
2315         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2316         if (v && !strcmp(v, "Keep-Alive"))
2317             m_http_keepalive = 1;
2318         else
2319             m_http_keepalive = 0;
2320         m_http_version = "1.0";
2321     }
2322     else
2323     {
2324         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2325         if (v && !strcmp(v, "close"))
2326             m_http_keepalive = 0;
2327         else
2328             m_http_keepalive = 1;
2329         m_http_version = "1.1";
2330     }
2331
2332     Z_SRW_PDU *srw_pdu = 0;
2333     Z_SOAP *soap_package = 0;
2334     char *charset = 0;
2335     Z_SRW_diagnostic *diagnostic = 0;
2336     int num_diagnostic = 0;
2337
2338     if (file_access(hreq))
2339     {
2340         return;
2341     }
2342     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2343                             &charset) == 0
2344              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2345                                &charset, &diagnostic, &num_diagnostic) == 0)
2346     {
2347         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2348         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2349         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2350         m_s2z_init_apdu = 0;
2351         m_s2z_search_apdu = 0;
2352         m_s2z_present_apdu = 0;
2353
2354         m_s2z_stylesheet = 0;
2355
2356         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2357         {
2358             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2359
2360             const char *backend_db = srw_req->database;
2361             srw_get_client(srw_req->database, &backend_db);
2362
2363             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2364             // recordXPath unsupported.
2365             if (srw_req->recordXPath)
2366             {
2367                 yaz_add_srw_diagnostic(odr_decode(),
2368                                        &diagnostic, &num_diagnostic,
2369                                        72, 0);
2370             }
2371             // sort unsupported
2372             if (srw_req->sort_type != Z_SRW_sort_type_none)
2373             {
2374                 yaz_add_srw_diagnostic(odr_decode(),
2375                                        &diagnostic, &num_diagnostic,
2376                                        80, 0);
2377             }
2378             // save stylesheet
2379             if (srw_req->stylesheet)
2380                 m_s2z_stylesheet =
2381                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2382
2383             // set packing for response records ..
2384             if (srw_req->recordPacking &&
2385                 !strcmp(srw_req->recordPacking, "xml"))
2386                 m_s2z_packing = Z_SRW_recordPacking_XML;
2387             else
2388                 m_s2z_packing = Z_SRW_recordPacking_string;
2389
2390             if (num_diagnostic)
2391             {
2392                 Z_SRW_PDU *srw_pdu =
2393                     yaz_srw_get(odr_encode(),
2394                                 Z_SRW_searchRetrieve_response);
2395                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2396
2397                 srw_res->diagnostics = diagnostic;
2398                 srw_res->num_diagnostics = num_diagnostic;
2399                 send_srw_response(srw_pdu);
2400                 return;
2401             }
2402
2403             // prepare search PDU
2404             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2405                                           Z_APDU_searchRequest);
2406             Z_SearchRequest *z_searchRequest =
2407                 m_s2z_search_apdu->u.searchRequest;
2408
2409             z_searchRequest->num_databaseNames = 1;
2410             z_searchRequest->databaseNames = (char**)
2411                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2412             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2413                                                            backend_db);
2414
2415             // query transformation
2416             Z_Query *query = (Z_Query *)
2417                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2418             z_searchRequest->query = query;
2419
2420             if (srw_req->query_type == Z_SRW_query_type_cql)
2421             {
2422                 Z_External *ext = (Z_External *)
2423                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2424                 ext->direct_reference =
2425                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2426                 ext->indirect_reference = 0;
2427                 ext->descriptor = 0;
2428                 ext->which = Z_External_CQL;
2429                 ext->u.cql = srw_req->query.cql;
2430
2431                 query->which = Z_Query_type_104;
2432                 query->u.type_104 =  ext;
2433             }
2434             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2435             {
2436                 Z_RPNQuery *RPNquery;
2437                 YAZ_PQF_Parser pqf_parser;
2438
2439                 pqf_parser = yaz_pqf_create ();
2440
2441                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2442                                           srw_req->query.pqf);
2443                 if (!RPNquery)
2444                 {
2445                     const char *pqf_msg;
2446                     size_t off;
2447                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2448                     yaz_log(YLOG_LOG, "%*s^\n", off+4, "");
2449                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2450
2451                     send_to_srw_client_error(10, 0);
2452                     return;
2453                 }
2454                 query->which = Z_Query_type_1;
2455                 query->u.type_1 =  RPNquery;
2456
2457                 yaz_pqf_destroy (pqf_parser);
2458             }
2459             else
2460             {
2461                 send_to_srw_client_error(7, "query");
2462                 return;
2463             }
2464
2465             // present
2466             m_s2z_present_apdu = 0;
2467             int max = 0;
2468             if (srw_req->maximumRecords)
2469                 max = *srw_req->maximumRecords;
2470             int start = 1;
2471             if (srw_req->startRecord)
2472                 start = *srw_req->startRecord;
2473             if (max > 0)
2474             {
2475                 // Some backend, such as Voyager doesn't honor piggyback
2476                 // So we use present always (0 &&).
2477                 if (0 && start <= 1)  // Z39.50 piggyback
2478                 {
2479                     *z_searchRequest->smallSetUpperBound = max;
2480                     *z_searchRequest->mediumSetPresentNumber = max;
2481                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2482
2483                     z_searchRequest->preferredRecordSyntax =
2484                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2485                                                VAL_TEXT_XML);
2486                     if (srw_req->recordSchema)
2487                     {
2488                         z_searchRequest->smallSetElementSetNames =
2489                             z_searchRequest->mediumSetElementSetNames =
2490                             mk_esn_from_schema(m_s2z_odr_search,
2491                                                srw_req->recordSchema);
2492                     }
2493                 }
2494                 else   // Z39.50 present
2495                 {
2496                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2497                                                    Z_APDU_presentRequest);
2498                     Z_PresentRequest *z_presentRequest =
2499                         m_s2z_present_apdu->u.presentRequest;
2500                     *z_presentRequest->resultSetStartPoint = start;
2501                     *z_presentRequest->numberOfRecordsRequested = max;
2502                     z_presentRequest->preferredRecordSyntax =
2503                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2504                                                VAL_TEXT_XML);
2505                     if (srw_req->recordSchema)
2506                     {
2507                         z_presentRequest->recordComposition =
2508                             (Z_RecordComposition *)
2509                             odr_malloc(m_s2z_odr_search,
2510                                        sizeof(Z_RecordComposition));
2511                         z_presentRequest->recordComposition->which =
2512                             Z_RecordComp_simple;
2513                         z_presentRequest->recordComposition->u.simple =
2514                             mk_esn_from_schema(m_s2z_odr_search,
2515                                                srw_req->recordSchema);
2516                     }
2517                 }
2518             }
2519             if (!m_client)
2520             {
2521                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2522                                             Z_APDU_initRequest);
2523
2524                 // prevent m_initRequest_apdu memory from being grabbed
2525                 // in Yaz_Proxy::handle_incoming_Z_PDU
2526                 m_initRequest_apdu = m_s2z_init_apdu;
2527                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2528                 return;
2529             }
2530             else
2531             {
2532                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2533                 return;
2534             }
2535         }
2536         else if (srw_pdu->which == Z_SRW_explain_request)
2537         {
2538             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2539
2540             const char *backend_db = srw_req->database;
2541             srw_get_client(srw_req->database, &backend_db);
2542
2543             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2544
2545             // save stylesheet
2546             if (srw_req->stylesheet)
2547                 m_s2z_stylesheet =
2548                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2549
2550             if (srw_req->recordPacking &&
2551                 !strcmp(srw_req->recordPacking, "xml"))
2552                 m_s2z_packing = Z_SRW_recordPacking_XML;
2553             else
2554                 m_s2z_packing = Z_SRW_recordPacking_string;
2555
2556             if (num_diagnostic)
2557             {
2558                 send_srw_explain_response(diagnostic, num_diagnostic);
2559                 return;
2560             }
2561
2562             if (!m_client)
2563             {
2564                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2565                                             Z_APDU_initRequest);
2566
2567                 // prevent m_initRequest_apdu memory from being grabbed
2568                 // in Yaz_Proxy::handle_incoming_Z_PDU
2569                 m_initRequest_apdu = m_s2z_init_apdu;
2570                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2571             }
2572             else
2573                 send_srw_explain_response(0, 0);
2574             return;
2575         }
2576         else if (srw_pdu->which == Z_SRW_scan_request)
2577         {
2578             m_s2z_database = odr_strdup(m_s2z_odr_init,
2579                                         srw_pdu->u.scan_request->database);
2580
2581             yaz_add_srw_diagnostic(odr_decode(),
2582                                    &diagnostic, &num_diagnostic,
2583                                    4, "scan");
2584             Z_SRW_PDU *srw_pdu =
2585                 yaz_srw_get(odr_encode(),
2586                             Z_SRW_scan_response);
2587             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2588
2589             srw_res->diagnostics = diagnostic;
2590             srw_res->num_diagnostics = num_diagnostic;
2591             send_srw_response(srw_pdu);
2592             return;
2593         }
2594         else
2595         {
2596             m_s2z_database = 0;
2597
2598             send_to_srw_client_error(4, 0);
2599         }
2600     }
2601     send_http_response(400);
2602 }
2603
2604 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2605 {
2606     Z_ReferenceId **refid = get_referenceIdP(apdu);
2607     nmem_reset(m_referenceId_mem);
2608     if (refid && *refid)
2609     {
2610         m_referenceId = (Z_ReferenceId *)
2611             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2612         m_referenceId->len = m_referenceId->size = (*refid)->len;
2613         m_referenceId->buf = (unsigned char *)
2614             nmem_malloc(m_referenceId_mem, (*refid)->len);
2615         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2616     }
2617     else
2618         m_referenceId = 0;
2619
2620     if (!m_client && m_invalid_session)
2621     {
2622         m_apdu_invalid_session = apdu;
2623         m_mem_invalid_session = odr_extract_mem(odr_decode());
2624         apdu = m_initRequest_apdu;
2625     }
2626
2627     // Determine our client.
2628     Z_OtherInformation **oi;
2629     get_otherInfoAPDU(apdu, &oi);
2630     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2631     if (!m_client)
2632     {
2633         if (m_http_version)
2634         {
2635             send_http_response(404);
2636             return;
2637         }
2638         else
2639         {
2640             delete this;
2641             return;
2642         }
2643     }
2644
2645     m_client->m_server = this;
2646
2647     if (apdu->which == Z_APDU_initRequest)
2648     {
2649         if (apdu->u.initRequest->implementationId)
2650             yaz_log(YLOG_LOG, "%simplementationId: %s",
2651                     m_session_str, apdu->u.initRequest->implementationId);
2652         if (apdu->u.initRequest->implementationName)
2653             yaz_log(YLOG_LOG, "%simplementationName: %s",
2654                     m_session_str, apdu->u.initRequest->implementationName);
2655         if (apdu->u.initRequest->implementationVersion)
2656             yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2657                     m_session_str, apdu->u.initRequest->implementationVersion);
2658         if (m_initRequest_apdu == 0)
2659         {
2660             if (m_initRequest_mem)
2661                 nmem_destroy(m_initRequest_mem);
2662
2663             m_initRequest_apdu = apdu;
2664             m_initRequest_mem = odr_extract_mem(odr_decode());
2665
2666             m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2667                 preferredMessageSize;
2668             *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2669             m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2670                 maximumRecordSize;
2671             *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2672
2673             // Save proposal charsets and langs.
2674             if (ODR_MASK_GET(apdu->u.initRequest->options,
2675                 Z_Options_negotiationModel))
2676             {
2677                 Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2678                     yaz_get_charneg_record(*oi);
2679
2680                 yaz_get_proposal_charneg(m_referenceId_mem,
2681                     charSetandLangRecord,
2682                     &m_initRequest_oi_negotiation_charsets,
2683                     &m_initRequest_oi_negotiation_num_charsets,
2684                     &m_initRequest_oi_negotiation_langs,
2685                     &m_initRequest_oi_negotiation_num_langs,
2686                     &m_initRequest_oi_negotiation_selected);
2687
2688                 for (int i=0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2689                 {
2690                     yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2691                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2692                         m_initRequest_oi_negotiation_charsets[i]:"none");
2693                 }
2694                 for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2695                 {
2696                     yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2697                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2698                         m_initRequest_oi_negotiation_langs[i]:"none");
2699                 }
2700                 yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2701                     m_session_str, m_initRequest_oi_negotiation_selected);
2702             }
2703             // save init options for the response..
2704             m_initRequest_options = apdu->u.initRequest->options;
2705
2706             apdu->u.initRequest->options =
2707                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2708                                            sizeof(Odr_bitmask));
2709             ODR_MASK_ZERO(apdu->u.initRequest->options);
2710             int i;
2711             for (i = 0; i<= 24; i++)
2712                 ODR_MASK_SET(apdu->u.initRequest->options, i);
2713             // check negotiation option
2714             if (!ODR_MASK_GET(m_initRequest_options,
2715                 Z_Options_negotiationModel))
2716             {
2717                 ODR_MASK_CLEAR(apdu->u.initRequest->options,
2718                     Z_Options_negotiationModel);
2719             }
2720             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2721                 Z_Options_concurrentOperations);
2722             // make new version
2723             m_initRequest_version = apdu->u.initRequest->protocolVersion;
2724             apdu->u.initRequest->protocolVersion =
2725                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2726                                            sizeof(Odr_bitmask));
2727             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2728
2729             for (i = 0; i<= 8; i++)
2730                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2731         }
2732         if (m_client->m_init_flag)
2733         {
2734             if (handle_init_response_for_invalid_session(apdu))
2735                 return;
2736             if (m_client->m_initResponse)
2737             {
2738                 Z_APDU *apdu2 = m_client->m_initResponse;
2739                 apdu2->u.initResponse->otherInfo = 0;
2740                 if (m_client->m_cookie && *m_client->m_cookie)
2741                     set_otherInformationString(apdu2, VAL_COOKIE, 1,
2742                                                m_client->m_cookie);
2743                 apdu2->u.initResponse->referenceId =
2744                     apdu->u.initRequest->referenceId;
2745                 apdu2->u.initResponse->options = m_client->m_initResponse_options;
2746                 apdu2->u.initResponse->protocolVersion =
2747                     m_client->m_initResponse_version;
2748
2749                 send_to_client(apdu2);
2750                 return;
2751             }
2752         }
2753         m_client->m_init_flag = 1;
2754     }
2755
2756     if (!handle_authentication(apdu))
2757     {
2758         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
2759         *apdu_reject->u.initResponse->result = 0;
2760         send_to_client(apdu_reject);
2761
2762         shutdown();
2763         return;
2764     }
2765
2766     handle_max_record_retrieve(apdu);
2767
2768     if (apdu)
2769         apdu = handle_syntax_validation(apdu);
2770
2771     if (apdu)
2772         apdu = handle_query_transformation(apdu);
2773
2774     if (apdu)
2775         apdu = handle_target_charset_conversion(apdu);
2776
2777     if (apdu)
2778         apdu = handle_query_validation(apdu);
2779
2780     if (apdu)
2781         apdu = result_set_optimize(apdu);
2782
2783     if (!apdu)
2784     {
2785         m_client->timeout(m_target_idletime);  // mark it active even
2786         // though we didn't use it
2787         return;
2788     }
2789     // Add otherInformation entry in APDU if
2790     // negotiatoin in use.
2791     if (apdu)
2792         handle_charset_lang_negotiation(apdu);
2793
2794     // delete other info construct completely if 0 elements
2795     get_otherInfoAPDU(apdu, &oi);
2796     if (oi && *oi && (*oi)->num_elements == 0)
2797         *oi = 0;
2798
2799     if (apdu->which == Z_APDU_presentRequest &&
2800         m_client->m_resultSetStartPoint == 0)
2801     {
2802         Z_PresentRequest *pr = apdu->u.presentRequest;
2803         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2804         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2805     } else {
2806         m_client->m_resultSetStartPoint = 0;
2807     }
2808     if (m_client->send_to_target(apdu) < 0)
2809     {
2810         delete m_client;
2811         m_client = 0;
2812         delete this;
2813     }
2814     else
2815         m_client->m_waiting = 1;
2816 }
2817
2818 void Yaz_Proxy::connectNotify()
2819 {
2820 }
2821
2822 void Yaz_Proxy::releaseClient()
2823 {
2824     xfree(m_proxyTarget);
2825     m_proxyTarget = 0;
2826     m_invalid_session = 0;
2827     // only keep if keep_alive flag is set...
2828     if (m_client &&
2829         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2830         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2831         m_client->m_waiting == 0)
2832     {
2833         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2834                  m_session_str,
2835                  m_client->get_hostname());
2836         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2837                 m_session_str, m_client->m_pdu_recv,
2838                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2839                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2840         assert (m_client->m_waiting != 2);
2841         // Tell client (if any) that no server connection is there..
2842         m_client->m_server = 0;
2843         m_client = 0;
2844     }
2845     else if (m_client)
2846     {
2847         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
2848                  m_session_str,
2849                  m_client->get_hostname());
2850         assert (m_client->m_waiting != 2);
2851         delete m_client;
2852         m_client = 0;
2853     }
2854     else if (!m_parent)
2855     {
2856         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
2857                  m_session_str);
2858         assert (m_parent);
2859     }
2860     else
2861     {
2862         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
2863                  m_session_str);
2864     }
2865     if (m_parent)
2866         m_parent->pre_init();
2867 }
2868
2869 void Yaz_Proxy::shutdown()
2870 {
2871     releaseClient();
2872     delete this;
2873 }
2874
2875 const char *Yaz_ProxyClient::get_session_str()
2876 {
2877     if (!m_server)
2878         return "0 ";
2879     return m_server->get_session_str();
2880 }
2881
2882 void Yaz_ProxyClient::shutdown()
2883 {
2884     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2885              get_hostname());
2886     delete m_server;
2887     delete this;
2888 }
2889
2890 void Yaz_Proxy::failNotify()
2891 {
2892     inc_request_no();
2893     yaz_log (YLOG_LOG, "%sConnection closed by client",
2894              get_session_str());
2895     shutdown();
2896 }
2897
2898 void Yaz_ProxyClient::failNotify()
2899 {
2900     if (m_server)
2901         m_server->inc_request_no();
2902     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
2903              get_session_str(), get_hostname());
2904     shutdown();
2905 }
2906
2907 void Yaz_ProxyClient::connectNotify()
2908 {
2909     const char *s = get_session_str();
2910     const char *h = get_hostname();
2911     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2912              m_target_idletime);
2913     timeout(m_target_idletime);
2914     if (!m_server)
2915         pre_init_client();
2916 }
2917
2918 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2919                                                   *the_PDU_Observable, int fd)
2920 {
2921     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2922 }
2923
2924 Yaz_ProxyClient::~Yaz_ProxyClient()
2925 {
2926     if (m_prev)
2927         *m_prev = m_next;
2928     if (m_next)
2929         m_next->m_prev = m_prev;
2930     m_waiting = 2;     // for debugging purposes only.
2931     odr_destroy(m_init_odr);
2932     delete m_last_query;
2933     xfree (m_last_resultSetId);
2934     xfree (m_cookie);
2935 }
2936
2937 void Yaz_ProxyClient::pre_init_client()
2938 {
2939     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2940     Z_InitRequest *req = apdu->u.initRequest;
2941
2942     int i;
2943     for (i = 0; i<= 24; i++)
2944         ODR_MASK_SET(req->options, i);
2945     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2946                    Z_Options_negotiationModel);
2947     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2948                    Z_Options_concurrentOperations);
2949     for (i = 0; i<= 10; i++)
2950         ODR_MASK_SET(req->protocolVersion, i);
2951
2952     if (send_to_target(apdu) < 0)
2953     {
2954         delete this;
2955     }
2956     else
2957     {
2958         m_waiting = 1;
2959         m_init_flag = 1;
2960     }
2961 }
2962
2963 void Yaz_Proxy::pre_init()
2964 {
2965     int i;
2966     const char *name = 0;
2967     const char *zurl_in_use[MAX_ZURL_PLEX];
2968     int limit_bw, limit_pdu, limit_req;
2969     int target_idletime, client_idletime;
2970     int max_clients;
2971     int keepalive_limit_bw, keepalive_limit_pdu;
2972     int pre_init;
2973     const char *cql2rpn = 0;
2974     const char *authentication = 0;
2975     const char *negotiation_charset = 0;
2976     const char *negotiation_lang = 0;
2977
2978     Yaz_ProxyConfig *cfg = check_reconfigure();
2979
2980     zurl_in_use[0] = 0;
2981
2982     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2983         set_APDU_yazlog(1);
2984     else
2985         set_APDU_yazlog(0);
2986
2987     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2988                                           &limit_bw, &limit_pdu, &limit_req,
2989                                           &target_idletime, &client_idletime,
2990                                           &max_clients,
2991                                           &keepalive_limit_bw,
2992                                           &keepalive_limit_pdu,
2993                                           &pre_init,
2994                                           &cql2rpn,
2995                                           &authentication,
2996                                           &negotiation_charset,
2997                                           &negotiation_lang,
2998                                           0) ; i++)
2999     {
3000         if (pre_init)
3001         {
3002             int j;
3003             for (j = 0; zurl_in_use[j]; j++)
3004             {
3005                 Yaz_ProxyClient *c;
3006                 int spare = 0;
3007                 int spare_waiting = 0;
3008                 int in_use = 0;
3009                 int other = 0;
3010                 for (c = m_clientPool; c; c = c->m_next)
3011                 {
3012                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3013                     {
3014                         if (c->m_cookie == 0)
3015                         {
3016                             if (c->m_server == 0)
3017                                 if (c->m_waiting)
3018                                     spare_waiting++;
3019                                 else
3020                                     spare++;
3021                             else
3022                                 in_use++;
3023                         }
3024                         else
3025                             other++;
3026                     }
3027                 }
3028                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3029                         "sparew=%d preinit=%d",m_session_str,
3030                         name, zurl_in_use[j], in_use, other,
3031                         spare, spare_waiting, pre_init);
3032                 if (spare + spare_waiting < pre_init)
3033                 {
3034                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3035                     c->m_next = m_clientPool;
3036                     if (c->m_next)
3037                         c->m_next->m_prev = &c->m_next;
3038                     m_clientPool = c;
3039                     c->m_prev = &m_clientPool;
3040
3041                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3042                         c->set_APDU_yazlog(1);
3043                     else
3044                         c->set_APDU_yazlog(0);
3045
3046                     if (c->client(zurl_in_use[j]))
3047                     {
3048                         timeout(60);
3049                         delete c;
3050                         return;
3051                     }
3052                     c->timeout(30);
3053                     c->m_waiting = 1;
3054                     c->m_target_idletime = target_idletime;
3055                     c->m_seqno = m_seqno++;
3056                 }
3057             }
3058         }
3059     }
3060 }
3061
3062 void Yaz_Proxy::timeoutNotify()
3063 {
3064     if (m_parent)
3065     {
3066         if (m_bw_hold_PDU)
3067         {
3068             timeout(m_client_idletime);
3069             Z_GDU *apdu = m_bw_hold_PDU;
3070             m_bw_hold_PDU = 0;
3071
3072             if (apdu->which == Z_GDU_Z3950)
3073                 handle_incoming_Z_PDU(apdu->u.z3950);
3074             else if (apdu->which == Z_GDU_HTTP_Request)
3075                 handle_incoming_HTTP(apdu->u.HTTP_Request);
3076         }
3077         else if (m_stylesheet_nprl)
3078             convert_xsl_delay();
3079         else
3080         {
3081             inc_request_no();
3082
3083             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3084             shutdown();
3085         }
3086     }
3087     else
3088     {
3089         timeout(600);
3090         pre_init();
3091     }
3092 }
3093
3094 void Yaz_Proxy::markInvalid()
3095 {
3096     m_client = 0;
3097     m_invalid_session = 1;
3098 }
3099
3100 void Yaz_ProxyClient::timeoutNotify()
3101 {
3102     if (m_server)
3103         m_server->inc_request_no();
3104
3105     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3106              get_hostname());
3107     m_waiting = 1;
3108     m_root->pre_init();
3109     if (m_server && m_init_flag)
3110     {
3111         // target timed out in a session that was properly initialized
3112         // server object stay alive but we mark it as invalid so it
3113         // gets initialized again
3114         m_server->markInvalid();
3115         m_server = 0;
3116     }
3117     shutdown();
3118 }
3119
3120 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
3121                                  Yaz_Proxy *parent) :
3122     Yaz_Z_Assoc (the_PDU_Observable)
3123 {
3124     m_cookie = 0;
3125     m_next = 0;
3126     m_prev = 0;
3127     m_init_flag = 0;
3128     m_last_query = 0;
3129     m_last_resultSetId = 0;
3130     m_last_resultCount = 0;
3131     m_last_ok = 0;
3132     m_sr_transform = 0;
3133     m_waiting = 0;
3134     m_init_odr = odr_createmem (ODR_DECODE);
3135     m_initResponse = 0;
3136     m_initResponse_options = 0;
3137     m_initResponse_version = 0;
3138     m_initResponse_preferredMessageSize = 0;
3139     m_initResponse_maximumRecordSize = 0;
3140     m_resultSetStartPoint = 0;
3141     m_bytes_sent = m_bytes_recv = 0;
3142     m_pdu_recv = 0;
3143     m_server = 0;
3144     m_seqno = 0;
3145     m_target_idletime = 600;
3146     m_root = parent;
3147 }
3148
3149 const char *Yaz_Proxy::option(const char *name, const char *value)
3150 {
3151     if (!strcmp (name, "optimize")) {
3152         if (value) {
3153             xfree (m_optimize);
3154             m_optimize = xstrdup (value);
3155         }
3156         return m_optimize;
3157     }
3158     return 0;
3159 }
3160
3161 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
3162 {
3163
3164 }
3165
3166 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
3167 {
3168     if (apdu->which == Z_GDU_Z3950)
3169         recv_Z_PDU(apdu->u.z3950, len);
3170     else if (apdu->which == Z_GDU_HTTP_Response)
3171         recv_HTTP_response(apdu->u.HTTP_Response, len);
3172     else
3173         shutdown();
3174 }
3175
3176 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
3177 {
3178     if (!m_invalid_session)
3179         return 0;
3180     m_invalid_session = 0;
3181     handle_incoming_Z_PDU(m_apdu_invalid_session);
3182     assert (m_mem_invalid_session);
3183     nmem_destroy(m_mem_invalid_session);
3184     m_mem_invalid_session = 0;
3185     return 1;
3186 }
3187
3188 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
3189 {
3190     m_bytes_recv += len;
3191
3192     m_pdu_recv++;
3193     m_waiting = 0;
3194     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
3195         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
3196                  apdu_name(apdu), get_hostname(), len);
3197     if (apdu->which == Z_APDU_initResponse)
3198     {
3199         if (!m_server)  // if this is a pre init session , check for more
3200             m_root->pre_init();
3201         NMEM nmem = odr_extract_mem (odr_decode());
3202         odr_reset (m_init_odr);
3203         nmem_transfer (m_init_odr->mem, nmem);
3204         m_initResponse = apdu;
3205         m_initResponse_options = apdu->u.initResponse->options;
3206         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3207         m_initResponse_preferredMessageSize =
3208             *apdu->u.initResponse->preferredMessageSize;
3209         m_initResponse_maximumRecordSize =
3210             *apdu->u.initResponse->maximumRecordSize;
3211
3212         Z_InitResponse *ir = apdu->u.initResponse;
3213         char *im0 = ir->implementationName;
3214
3215         char *im1 = (char*)
3216             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3217         *im1 = '\0';
3218         if (im0)
3219         {
3220             strcat(im1, im0);
3221             strcat(im1, " ");
3222         }
3223         strcat(im1, "(YAZ Proxy)");
3224         ir->implementationName = im1;
3225
3226         nmem_destroy (nmem);
3227
3228         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3229             return;
3230     }
3231     if (apdu->which == Z_APDU_searchResponse)
3232     {
3233         Z_SearchResponse *sr = apdu->u.searchResponse;
3234         m_last_resultCount = *sr->resultCount;
3235         int status = *sr->searchStatus;
3236         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3237         {
3238             m_last_ok = 1;
3239
3240             if (sr->records && sr->records->which == Z_Records_DBOSD)
3241             {
3242                 m_cache.add(odr_decode(),
3243                             sr->records->u.databaseOrSurDiagnostics, 1,
3244                             *sr->resultCount);
3245             }
3246         }
3247     }
3248     if (apdu->which == Z_APDU_presentResponse)
3249     {
3250         Z_PresentResponse *pr = apdu->u.presentResponse;
3251         if (m_sr_transform)
3252         {
3253             m_sr_transform = 0;
3254             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3255             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3256             sr->referenceId = pr->referenceId;
3257             *sr->resultCount = m_last_resultCount;
3258             sr->records = pr->records;
3259             sr->nextResultSetPosition = pr->nextResultSetPosition;
3260             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3261             apdu = new_apdu;
3262         }
3263         if (pr->records &&
3264             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3265         {
3266             m_cache.add(odr_decode(),
3267                         pr->records->u.databaseOrSurDiagnostics,
3268                         m_resultSetStartPoint, -1);
3269             m_resultSetStartPoint = 0;
3270         }
3271     }
3272     if (m_cookie)
3273         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3274     if (m_server)
3275     {
3276         m_server->send_to_client(apdu);
3277     }
3278     if (apdu->which == Z_APDU_close)
3279     {
3280         shutdown();
3281     }
3282 }
3283
3284 void Yaz_Proxy::low_socket_close()
3285 {
3286 #if WIN32
3287 #else
3288     int i;
3289     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3290         if  (m_lo_fd[i] >= 0)
3291             ::close(m_lo_fd[i]);
3292 #endif
3293 }
3294
3295 void Yaz_Proxy::low_socket_open()
3296 {
3297 #if WIN32
3298 #else
3299     int i;
3300     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3301         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3302 #endif
3303 }
3304
3305 int Yaz_Proxy::server(const char *addr)
3306 {
3307     int r = Yaz_Z_Assoc::server(addr);
3308     if (!r)
3309     {
3310         yaz_log(YLOG_LOG, "%sStarted proxy "
3311 #ifdef VERSION
3312             VERSION
3313 #endif
3314             " on %s", m_session_str, addr);
3315         timeout(1);
3316     }
3317     return r;
3318 }
3319