src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.21 2005-02-10 19:17:44 adam Exp $
   2    Copyright (c) 1998-2005, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <time.h>
  42 #include <fcntl.h>
  43
  44 #include <yaz/srw.h>
  45 #include <yaz/marcdisp.h>
  46 #include <yaz/yaz-iconv.h>
  47 #include <yaz/log.h>
  48 #include <yaz/diagbib1.h>
  49 #include "proxyp.h"
  50 #include <yaz/pquery.h>
  51 #include <yaz/otherinfo.h>
  52 #include <yaz/charneg.h>
  53
  54
  55 static const char *apdu_name(Z_APDU *apdu)
  56 {
  57     switch (apdu->which)
  58     {
  59     case Z_APDU_initRequest:
  60         return "initRequest";
  61     case Z_APDU_initResponse:
  62         return "initResponse";
  63     case Z_APDU_searchRequest:
  64         return "searchRequest";
  65     case Z_APDU_searchResponse:
  66         return "searchResponse";
  67     case Z_APDU_presentRequest:
  68         return "presentRequest";
  69     case Z_APDU_presentResponse:
  70         return "presentResponse";
  71     case Z_APDU_deleteResultSetRequest:
  72         return "deleteResultSetRequest";
  73     case Z_APDU_deleteResultSetResponse:
  74         return "deleteResultSetResponse";
  75     case Z_APDU_scanRequest:
  76         return "scanRequest";
  77     case Z_APDU_scanResponse:
  78         return "scanResponse";
  79     case Z_APDU_sortRequest:
  80         return "sortRequest";
  81     case Z_APDU_sortResponse:
  82         return "sortResponse";
  83     case Z_APDU_extendedServicesRequest:
  84         return "extendedServicesRequest";
  85     case Z_APDU_extendedServicesResponse:
  86         return "extendedServicesResponse";
  87     case Z_APDU_close:
  88         return "close";
  89     }
  90     return "other";
  91 }
  92
  93 static const char *gdu_name(Z_GDU *gdu)
  94 {
  95     switch(gdu->which)
  96     {
  97     case Z_GDU_Z3950:
  98         return apdu_name(gdu->u.z3950);
  99     case Z_GDU_HTTP_Request:
 100         return "HTTP Request";
 101     case Z_GDU_HTTP_Response:
 102         return "HTTP Response";
 103     }
 104     return "Unknown request/response";
 105 }
 106 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
 107                      Yaz_Proxy *parent) :
 108     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 109 {
 110     m_PDU_Observable = the_PDU_Observable;
 111     m_client = 0;
 112     m_parent = parent;
 113     m_clientPool = 0;
 114     m_seqno = 1;
 115     m_keepalive_limit_bw = 500000;
 116     m_keepalive_limit_pdu = 1000;
 117     m_proxyTarget = 0;
 118     m_default_target = 0;
 119     m_proxy_authentication = 0;
 120     m_proxy_negotiation_charset = 0;
 121     m_proxy_negotiation_lang = 0;
 122     m_max_clients = 150;
 123     m_log_mask = 0;
 124     m_seed = time(0);
 125     m_client_idletime = 600;
 126     m_target_idletime = 600;
 127     m_optimize = xstrdup ("1");
 128     strcpy(m_session_str, "0 ");
 129     m_session_no=0;
 130     m_bytes_sent = 0;
 131     m_bytes_recv = 0;
 132     m_bw_hold_PDU = 0;
 133     m_bw_max = 0;
 134     m_pdu_max = 0;
 135     m_max_record_retrieve = 0;
 136     m_reconfig_flag = 0;
 137     m_config_fname = 0;
 138     m_request_no = 0;
 139     m_invalid_session = 0;
 140     m_referenceId = 0;
 141     m_referenceId_mem = nmem_create();
 142     m_config = 0;
 143     m_marcxml_flag = 0;
 144     m_stylesheet_xsp = 0;
 145     m_stylesheet_nprl = 0;
 146     m_s2z_stylesheet = 0;
 147     m_s2z_database = 0;
 148     m_schema = 0;
 149     m_backend_type = 0;
 150     m_backend_charset = 0;
 151     m_frontend_type = 0;
 152     m_initRequest_apdu = 0;
 153     m_initRequest_mem = 0;
 154     m_initRequest_preferredMessageSize = 0;
 155     m_initRequest_maximumRecordSize = 0;
 156     m_initRequest_options = 0;
 157     m_initRequest_version = 0;
 158     m_initRequest_oi_negotiation_charsets = 0;
 159     m_initRequest_oi_negotiation_num_charsets = 0;
 160     m_initRequest_oi_negotiation_langs = 0;
 161     m_initRequest_oi_negotiation_num_langs = 0;
 162     m_initRequest_oi_negotiation_selected = 0;
 163     m_apdu_invalid_session = 0;
 164     m_mem_invalid_session = 0;
 165     m_s2z_odr_init = 0;
 166     m_s2z_odr_search = 0;
 167     m_s2z_init_apdu = 0;
 168     m_s2z_search_apdu = 0;
 169     m_s2z_present_apdu = 0;
 170     m_http_keepalive = 0;
 171     m_http_version = 0;
 172     m_soap_ns = 0;
 173     m_s2z_packing = Z_SRW_recordPacking_string;
 174 #if HAVE_GETTIMEOFDAY
 175     m_time_tv = xmalloc(sizeof(struct timeval));
 176     struct timeval *tv = (struct timeval *) m_time_tv;
 177     tv->tv_sec = 0;
 178     tv->tv_usec = 0;
 179 #else
 180     m_time_tv = 0;
 181 #endif
 182     m_usemarcon_ini_stage1 = 0;
 183     m_usemarcon_ini_stage2 = 0;
 184     m_usemarcon = new Yaz_usemarcon();
 185     if (!m_parent)
 186         low_socket_open();
 187 }
 188
 189 Yaz_Proxy::~Yaz_Proxy()
 190 {
 191     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 192             m_bytes_sent, m_bytes_recv);
 193     nmem_destroy(m_initRequest_mem);
 194     nmem_destroy(m_mem_invalid_session);
 195     nmem_destroy(m_referenceId_mem);
 196
 197     xfree(m_proxyTarget);
 198     xfree(m_default_target);
 199     xfree(m_proxy_authentication);
 200     xfree(m_proxy_negotiation_charset);
 201     xfree(m_proxy_negotiation_lang);
 202     xfree(m_optimize);
 203
 204 #if HAVE_XSLT
 205     if (m_stylesheet_xsp)
 206         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 207 #endif
 208     xfree (m_time_tv);
 209
 210     xfree (m_schema);
 211     xfree (m_backend_type);
 212     xfree (m_backend_charset);
 213     xfree (m_usemarcon_ini_stage1);
 214     xfree (m_usemarcon_ini_stage2);
 215     delete m_usemarcon;
 216     if (m_s2z_odr_init)
 217         odr_destroy(m_s2z_odr_init);
 218     if (m_s2z_odr_search)
 219         odr_destroy(m_s2z_odr_search);
 220     if (!m_parent)
 221         low_socket_close();
 222     delete m_config;
 223 }
 224
 225 int Yaz_Proxy::set_config(const char *config)
 226 {
 227     delete m_config;
 228     m_config = new Yaz_ProxyConfig();
 229     xfree(m_config_fname);
 230     m_config_fname = xstrdup(config);
 231     int r = m_config->read_xml(config);
 232     if (!r)
 233         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 234     return r;
 235 }
 236
 237 void Yaz_Proxy::set_default_target(const char *target)
 238 {
 239     xfree (m_default_target);
 240     m_default_target = 0;
 241     if (target)
 242         m_default_target = (char *) xstrdup (target);
 243 }
 244
 245 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 246 {
 247     xfree (m_proxy_authentication);
 248     m_proxy_authentication = 0;
 249     if (auth)
 250         m_proxy_authentication = (char *) xstrdup (auth);
 251 }
 252 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang)
 253 {
 254     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
 255         "language to '%s'", m_session_str, charset?charset:"none",
 256         lang?lang:"none");
 257     xfree (m_proxy_negotiation_charset);
 258     xfree (m_proxy_negotiation_lang);
 259     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 260     if (charset)
 261         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 262     if (lang)
 263         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 264 }
 265 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 266 {
 267     if (m_parent)
 268         return m_parent->check_reconfigure();
 269
 270     Yaz_ProxyConfig *cfg = m_config;
 271     if (m_reconfig_flag)
 272     {
 273         yaz_log(YLOG_LOG, "reconfigure");
 274         yaz_log_reopen();
 275         if (m_config_fname && cfg)
 276         {
 277             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 278             int r = cfg->read_xml(m_config_fname);
 279             if (r)
 280                 yaz_log(YLOG_WARN, "reconfigure failed");
 281             else
 282             {
 283                 m_log_mask = 0;
 284                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 285             }
 286         }
 287         else
 288             yaz_log(YLOG_LOG, "reconfigure");
 289         m_reconfig_flag = 0;
 290     }
 291     return cfg;
 292 }
 293
 294 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 295                                             *the_PDU_Observable, int fd)
 296 {
 297     check_reconfigure();
 298     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 299     new_proxy->m_config = 0;
 300     new_proxy->m_config_fname = 0;
 301     new_proxy->timeout(m_client_idletime);
 302     new_proxy->m_target_idletime = m_target_idletime;
 303     new_proxy->set_default_target(m_default_target);
 304     new_proxy->m_max_clients = m_max_clients;
 305     new_proxy->m_log_mask = m_log_mask;
 306     new_proxy->set_APDU_log(get_APDU_log());
 307     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 308         new_proxy->set_APDU_yazlog(1);
 309     else
 310         new_proxy->set_APDU_yazlog(0);
 311     new_proxy->set_proxy_authentication(m_proxy_authentication);
 312     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 313         m_proxy_negotiation_lang);
 314     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 315     m_session_no++;
 316     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 317              the_PDU_Observable->getpeername());
 318     return new_proxy;
 319 }
 320
 321 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 322 {
 323     int oid[OID_SIZE];
 324     Z_OtherInformationUnit *oi;
 325     struct oident ent;
 326     ent.proto = PROTO_Z3950;
 327     ent.oclass = CLASS_USERINFO;
 328     ent.value = (oid_value) VAL_COOKIE;
 329     assert (oid_ent_to_oid (&ent, oid));
 330
 331     if (oid_ent_to_oid (&ent, oid) &&
 332         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 333         oi->which == Z_OtherInfo_characterInfo)
 334         return oi->information.characterInfo;
 335     return 0;
 336 }
 337 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 338 {
 339     int oid[OID_SIZE];
 340     Z_OtherInformationUnit *oi;
 341     struct oident ent;
 342     ent.proto = PROTO_Z3950;
 343     ent.oclass = CLASS_USERINFO;
 344     ent.value = (oid_value) VAL_PROXY;
 345     if (oid_ent_to_oid (&ent, oid) &&
 346         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 347         oi->which == Z_OtherInfo_characterInfo)
 348         return oi->information.characterInfo;
 349     return 0;
 350 }
 351 const char *Yaz_Proxy::load_balance(const char **url)
 352 {
 353     int zurl_in_use[MAX_ZURL_PLEX];
 354     int zurl_in_spare[MAX_ZURL_PLEX];
 355     Yaz_ProxyClient *c;
 356     int i;
 357
 358     for (i = 0; i<MAX_ZURL_PLEX; i++)
 359     {
 360         zurl_in_use[i] = 0;
 361         zurl_in_spare[i] = 0;
 362     }
 363     for (c = m_parent->m_clientPool; c; c = c->m_next)
 364     {
 365         for (i = 0; url[i]; i++)
 366             if (!strcmp(url[i], c->get_hostname()))
 367             {
 368                 zurl_in_use[i]++;
 369                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 370                     zurl_in_spare[i]++;
 371             }
 372     }
 373     int min_use = 100000;
 374     int spare_for_min = 0;
 375     int max_spare = 0;
 376     const char *ret_min = 0;
 377     const char *ret_spare = 0;
 378     for (i = 0; url[i]; i++)
 379     {
 380         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 381                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 382         if (min_use > zurl_in_use[i])
 383         {
 384             ret_min = url[i];
 385             min_use = zurl_in_use[i];
 386             spare_for_min = zurl_in_spare[i];
 387         }
 388         if (max_spare < zurl_in_spare[i])
 389         {
 390             ret_spare = url[i];
 391             max_spare = zurl_in_spare[i];
 392         }
 393     }
 394     // use the one with minimum connections if spare is > 3
 395     if (spare_for_min > 3)
 396         return ret_min;
 397     // use one with most spares (if any)
 398     if (max_spare > 0)
 399         return ret_spare;
 400     return ret_min;
 401 }
 402
 403 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 404                                        const char *proxy_host)
 405 {
 406     assert (m_parent);
 407     Yaz_Proxy *parent = m_parent;
 408     Yaz_ProxyClient *c = m_client;
 409
 410     if (!m_proxyTarget)
 411     {
 412         const char *url[MAX_ZURL_PLEX];
 413         Yaz_ProxyConfig *cfg = check_reconfigure();
 414         if (proxy_host)
 415         {
 416 #if 0
 417 /* only to be enabled for debugging... */
 418             if (!strcmp(proxy_host, "stop"))
 419                 exit(0);
 420 #endif
 421             xfree(m_default_target);
 422             m_default_target = xstrdup(proxy_host);
 423         }
 424         proxy_host = m_default_target;
 425         int client_idletime = -1;
 426         const char *cql2rpn_fname = 0;
 427         const char *authentication = 0;
 428         const char *negotiation_charset = 0;
 429         const char *negotiation_lang = 0;
 430         url[0] = m_default_target;
 431         url[1] = 0;
 432         if (cfg)
 433         {
 434             int pre_init = 0;
 435             cfg->get_target_info(proxy_host, url, &m_bw_max,
 436                                  &m_pdu_max, &m_max_record_retrieve,
 437                                  &m_target_idletime, &client_idletime,
 438                                  &parent->m_max_clients,
 439                                  &m_keepalive_limit_bw,
 440                                  &m_keepalive_limit_pdu,
 441                                  &pre_init,
 442                                  &cql2rpn_fname,
 443                                  &authentication,
 444                                  &negotiation_charset,
 445                                  &negotiation_lang);
 446         }
 447         if (client_idletime != -1)
 448         {
 449             m_client_idletime = client_idletime;
 450             timeout(m_client_idletime);
 451         }
 452         if (cql2rpn_fname)
 453             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 454         if (authentication)
 455             set_proxy_authentication(authentication);
 456         if (negotiation_charset || negotiation_lang)
 457             set_proxy_negotiation(negotiation_charset,
 458                 negotiation_lang);
 459         if (!url[0])
 460         {
 461             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 462             return 0;
 463         }
 464         // we don't handle multiplexing for cookie session, so we just
 465         // pick the first one in this case (anonymous users will be able
 466         // to use any backend)
 467         if (cookie && *cookie)
 468             m_proxyTarget = (char*) xstrdup(url[0]);
 469         else
 470             m_proxyTarget = (char*) xstrdup(load_balance(url));
 471     }
 472     if (cookie && *cookie)
 473     {   // search in sessions with a cookie
 474         for (c = parent->m_clientPool; c; c = c->m_next)
 475         {
 476             assert (c->m_prev);
 477             assert (*c->m_prev == c);
 478             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 479                 !strcmp(m_proxyTarget, c->get_hostname()))
 480             {
 481                 // Found it in cache
 482                 // The following handles "cancel"
 483                 // If connection is busy (waiting for PDU) and
 484                 // we have an initRequest we can safely do re-open
 485                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 486                 {
 487                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 488                              c->get_hostname());
 489                     c->close();
 490                     c->m_init_flag = 0;
 491
 492                     c->m_last_ok = 0;
 493                     c->m_cache.clear();
 494                     c->m_last_resultCount = 0;
 495                     c->m_sr_transform = 0;
 496                     c->m_waiting = 0;
 497                     c->m_resultSetStartPoint = 0;
 498                     c->m_target_idletime = m_target_idletime;
 499                     if (c->client(m_proxyTarget))
 500                     {
 501                         delete c;
 502                         return 0;
 503                     }
 504                     c->timeout(30);
 505                 }
 506                 c->m_seqno = parent->m_seqno;
 507                 if (c->m_server && c->m_server != this)
 508                     c->m_server->m_client = 0;
 509                 c->m_server = this;
 510                 (parent->m_seqno)++;
 511                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 512                 return c;
 513             }
 514         }
 515     }
 516     else if (!c &&
 517             apdu->which == Z_APDU_initRequest &&
 518             apdu->u.initRequest->idAuthentication == 0 &&
 519             !ODR_MASK_GET(apdu->u.initRequest->options, Z_Options_negotiationModel))
 520     {
 521         // anonymous sessions without cookie.
 522         // if authentication is set it is NOT anonymous se we can't share them.
 523         // If charset and lang negotiation is use it is NOT anonymous session too.
 524         for (c = parent->m_clientPool; c; c = c->m_next)
 525         {
 526             assert(c->m_prev);
 527             assert(*c->m_prev == c);
 528             if (c->m_server == 0 && c->m_cookie == 0 &&
 529                 c->m_waiting == 0 &&
 530                 !strcmp(m_proxyTarget, c->get_hostname()))
 531             {
 532                 // found it in cache
 533                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 534                          m_session_str, parent->m_seqno, c->get_hostname());
 535
 536                 c->m_seqno = parent->m_seqno;
 537                 assert(c->m_server == 0);
 538                 c->m_server = this;
 539
 540                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 541                     c->set_APDU_yazlog(1);
 542                 else
 543                     c->set_APDU_yazlog(0);
 544
 545                 (parent->m_seqno)++;
 546
 547                 parent->pre_init();
 548
 549                 return c;
 550             }
 551         }
 552     }
 553     if (!m_client)
 554     {
 555         if (apdu->which != Z_APDU_initRequest)
 556         {
 557             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 558             return 0;
 559         }
 560         Z_InitRequest *initRequest = apdu->u.initRequest;
 561
 562         if (!initRequest->idAuthentication)
 563         {
 564             if (m_proxy_authentication)
 565             {
 566                 initRequest->idAuthentication =
 567                     (Z_IdAuthentication *)
 568                     odr_malloc (odr_encode(),
 569                                 sizeof(*initRequest->idAuthentication));
 570                 initRequest->idAuthentication->which =
 571                     Z_IdAuthentication_open;
 572                 initRequest->idAuthentication->u.open =
 573                     odr_strdup (odr_encode(), m_proxy_authentication);
 574             }
 575         }
 576         else
 577         {
 578             // the client use authentication. We set the keepalive PDU
 579             // to 0 so we don't cache it in releaseClient
 580             m_keepalive_limit_pdu = 0;
 581         }
 582         // go through list of clients - and find the lowest/oldest one.
 583         Yaz_ProxyClient *c_min = 0;
 584         int min_seq = -1;
 585         int no_of_clients = 0;
 586         if (parent->m_clientPool)
 587             yaz_log (YLOG_DEBUG, "Existing sessions");
 588         for (c = parent->m_clientPool; c; c = c->m_next)
 589         {
 590             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 591                                c->m_waiting, c->get_hostname(),
 592                                c->m_cookie ? c->m_cookie : "");
 593             no_of_clients++;
 594             if (min_seq < 0 || c->m_seqno < min_seq)
 595             {
 596                 min_seq = c->m_seqno;
 597                 c_min = c;
 598             }
 599         }
 600         if (no_of_clients >= parent->m_max_clients)
 601         {
 602             c = c_min;
 603             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 604             {
 605                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 606                          m_session_str, parent->m_max_clients, c->m_seqno);
 607                 if (c->m_server && c->m_server != this)
 608                     delete c->m_server;
 609                 c->m_server = 0;
 610             }
 611             else
 612             {
 613                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 614                          m_session_str, parent->m_max_clients,
 615                          c->m_seqno, parent->m_seqno, c->get_hostname());
 616                 xfree (c->m_cookie);
 617                 c->m_cookie = 0;
 618                 if (cookie)
 619                     c->m_cookie = xstrdup(cookie);
 620                 c->m_seqno = parent->m_seqno;
 621                 if (c->m_server && c->m_server != this)
 622                 {
 623                     c->m_server->m_client = 0;
 624                     delete c->m_server;
 625                 }
 626                 (parent->m_seqno)++;
 627                 c->m_target_idletime = m_target_idletime;
 628                 c->timeout(m_target_idletime);
 629
 630                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 631                     c->set_APDU_yazlog(1);
 632                 else
 633                     c->set_APDU_yazlog(0);
 634
 635                 return c;
 636             }
 637         }
 638         else
 639         {
 640             yaz_log (YLOG_LOG, "%sNEW %d %s",
 641                      m_session_str, parent->m_seqno, m_proxyTarget);
 642             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 643             c->m_next = parent->m_clientPool;
 644             if (c->m_next)
 645                 c->m_next->m_prev = &c->m_next;
 646             parent->m_clientPool = c;
 647             c->m_prev = &parent->m_clientPool;
 648         }
 649
 650         xfree (c->m_cookie);
 651         c->m_cookie = 0;
 652         if (cookie)
 653             c->m_cookie = xstrdup(cookie);
 654
 655         c->m_seqno = parent->m_seqno;
 656         c->m_init_flag = 0;
 657         c->m_last_resultCount = 0;
 658         c->m_last_ok = 0;
 659         c->m_cache.clear();
 660         c->m_sr_transform = 0;
 661         c->m_waiting = 0;
 662         c->m_resultSetStartPoint = 0;
 663         (parent->m_seqno)++;
 664         if (c->client(m_proxyTarget))
 665         {
 666             delete c;
 667             return 0;
 668         }
 669         c->m_target_idletime = m_target_idletime;
 670         c->timeout(30);
 671
 672         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 673             c->set_APDU_yazlog(1);
 674         else
 675             c->set_APDU_yazlog(0);
 676     }
 677     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 678     return c;
 679 }
 680 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 681 {
 682     int i;
 683     for (i = 0; i<num; i++)
 684     {
 685         oident *ent;
 686         Z_DefaultDiagFormat *r;
 687         Z_DiagRec *p = pp[i];
 688         if (p->which != Z_DiagRec_defaultFormat)
 689         {
 690             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 691             return;
 692         }
 693         else
 694             r = p->u.defaultFormat;
 695         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 696             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 697             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 698         switch (r->which)
 699         {
 700         case Z_DefaultDiagFormat_v2Addinfo:
 701             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 702                     m_session_str,
 703                     *r->condition, diagbib1_str(*r->condition),
 704                     r->u.v2Addinfo);
 705             break;
 706         case Z_DefaultDiagFormat_v3Addinfo:
 707             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 708                     m_session_str,
 709                     *r->condition, diagbib1_str(*r->condition),
 710                     r->u.v3Addinfo);
 711             break;
 712         }
 713     }
 714 }
 715
 716 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 717 {
 718     if (!m_stylesheet_xsp || p->num_records <= 0)
 719         return 0;  /* no XSLT to be done ... */
 720
 721     m_stylesheet_offset = 0;
 722     m_stylesheet_nprl = p;
 723     m_stylesheet_apdu = apdu;
 724     timeout(0);
 725     return 1;
 726 }
 727
 728 void Yaz_Proxy::convert_xsl_delay()
 729 {
 730 #if HAVE_XSLT
 731     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 732     if (npr->which == Z_NamePlusRecord_databaseRecord)
 733     {
 734         Z_External *r = npr->u.databaseRecord;
 735         if (r->which == Z_External_octet)
 736         {
 737 #if 0
 738             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 739 #endif
 740             xmlDocPtr res, doc = xmlParseMemory(
 741                 (char*) r->u.octet_aligned->buf,
 742                 r->u.octet_aligned->len);
 743
 744
 745             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 746                     m_session_str, m_stylesheet_offset);
 747             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 748                                       doc, 0);
 749
 750             if (res)
 751             {
 752                 xmlChar *out_buf;
 753                 int out_len;
 754                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 755
 756                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 757                     u.databaseRecord =
 758                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 759                                  (char*) out_buf, out_len);
 760                 xmlFree(out_buf);
 761                 xmlFreeDoc(res);
 762             }
 763
 764             xmlFreeDoc(doc);
 765         }
 766     }
 767 #endif
 768     m_stylesheet_offset++;
 769     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 770     {
 771         m_stylesheet_nprl = 0;
 772 #if HAVE_XSLT
 773         if (m_stylesheet_xsp)
 774             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 775 #endif
 776         m_stylesheet_xsp = 0;
 777         timeout(m_client_idletime);
 778         send_PDU_convert(m_stylesheet_apdu);
 779     }
 780     else
 781         timeout(0);
 782 }
 783
 784 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 785 {
 786     if (m_frontend_type != VAL_NONE)
 787     {
 788         int i;
 789         for (i = 0; i < p->num_records; i++)
 790         {
 791             Z_NamePlusRecord *npr = p->records[i];
 792             if (npr->which == Z_NamePlusRecord_databaseRecord)
 793             {
 794                 Z_External *r = npr->u.databaseRecord;
 795                 if (r->which == Z_External_octet)
 796                 {
 797 #if HAVE_USEMARCON
 798                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 799                     {
 800                         if (!m_usemarcon->m_stage1)
 801                         {
 802                             m_usemarcon->m_stage1 = new CDetails();
 803                         }
 804                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 805                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 806                         int res = m_usemarcon->m_stage1->Start();
 807                         if (res == 0)
 808                         {
 809                             char *converted;
 810                             int convlen;
 811                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 812                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 813                             {
 814                                 if (!m_usemarcon->m_stage2)
 815                                 {
 816                                     m_usemarcon->m_stage2 = new CDetails();
 817                                 }
 818                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 819                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 820                                 res = m_usemarcon->m_stage2->Start();
 821                                 if (res == 0)
 822                                 {
 823                                     free(converted);
 824                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 825                                 }
 826                                 else
 827                                 {
 828                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 829                                 }
 830                             }
 831                             npr->u.databaseRecord =
 832                                 z_ext_record(odr_encode(),
 833                                              m_frontend_type,
 834                                              converted,
 835                                              strlen(converted));
 836                             free(converted);
 837                         }
 838                         else
 839                         {
 840                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 841                         }
 842                         continue;
 843                     }
 844 #endif
 845                     npr->u.databaseRecord =
 846                         z_ext_record(odr_encode(),
 847                                      m_frontend_type,
 848                                      (char*) r->u.octet_aligned->buf,
 849                                      r->u.octet_aligned->len);
 850                 }
 851             }
 852         }
 853     }
 854 }
 855
 856 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
 857                                    const char *backend_charset)
 858 {
 859     int i;
 860     if (!backend_charset)
 861         backend_charset = "MARC-8";
 862     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
 863     yaz_marc_t mt = yaz_marc_create();
 864     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 865     yaz_marc_iconv(mt, cd);
 866     for (i = 0; i < p->num_records; i++)
 867     {
 868         Z_NamePlusRecord *npr = p->records[i];
 869         if (npr->which == Z_NamePlusRecord_databaseRecord)
 870         {
 871             Z_External *r = npr->u.databaseRecord;
 872             if (r->which == Z_External_octet)
 873             {
 874                 int rlen;
 875                 char *result;
 876                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 877                                         r->u.octet_aligned->len,
 878                                         &result, &rlen))
 879                 {
 880                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 881                                                          VAL_TEXT_XML,
 882                                                          result, rlen);
 883                 }
 884             }
 885         }
 886     }
 887     if (cd)
 888         yaz_iconv_close(cd);
 889     yaz_marc_destroy(mt);
 890 }
 891
 892 void Yaz_Proxy::logtime()
 893 {
 894 #if HAVE_GETTIMEOFDAY
 895     struct timeval *tv = (struct timeval*) m_time_tv;
 896     if (tv->tv_sec)
 897     {
 898         struct timeval tv1;
 899         gettimeofday(&tv1, 0);
 900         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
 901             (tv1.tv_usec - tv->tv_usec);
 902         if (diff >= 0)
 903             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 904                     diff/1000000, (diff/1000)%1000);
 905     }
 906     tv->tv_sec = 0;
 907     tv->tv_usec = 0;
 908 #endif
 909 }
 910
 911 int Yaz_Proxy::send_http_response(int code)
 912 {
 913     ODR o = odr_encode();
 914     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 915     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 916     if (m_http_version)
 917         hres->version = odr_strdup(o, m_http_version);
 918     if (m_http_keepalive)
 919         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 920     else
 921         timeout(0);
 922
 923     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 924     {
 925         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
 926                  gdu_name(gdu));
 927     }
 928     int len;
 929     int r = send_GDU(gdu, &len);
 930     m_bytes_sent += len;
 931     m_bw_stat.add_bytes(len);
 932     logtime();
 933     return r;
 934 }
 935
 936 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 937 {
 938     ODR o = odr_encode();
 939     const char *ctype = "text/xml";
 940     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 941     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 942     if (m_http_version)
 943         hres->version = odr_strdup(o, m_http_version);
 944     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 945     if (m_http_keepalive)
 946         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 947     else
 948         timeout(0);
 949
 950     static Z_SOAP_Handler soap_handlers[2] = {
 951 #if HAVE_XSLT
 952         {"http://www.loc.gov/zing/srw/", 0,
 953          (Z_SOAP_fun) yaz_srw_codec},
 954 #endif
 955         {0, 0, 0}
 956     };
 957
 958     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 959     soap_package->which = Z_SOAP_generic;
 960     soap_package->u.generic =
 961         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 962     soap_package->u.generic->no = 0;
 963     soap_package->u.generic->ns = soap_handlers[0].ns;
 964     soap_package->u.generic->p = (void *) srw_pdu;
 965     soap_package->ns = m_soap_ns;
 966     z_soap_codec_enc_xsl(o, &soap_package,
 967                          &hres->content_buf, &hres->content_len,
 968                          soap_handlers, 0, m_s2z_stylesheet);
 969     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 970     {
 971         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
 972                  gdu_name(gdu));
 973     }
 974     int len;
 975     int r = send_GDU(gdu, &len);
 976     m_bytes_sent += len;
 977     m_bw_stat.add_bytes(len);
 978     logtime();
 979     return r;
 980 }
 981
 982 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
 983 {
 984     ODR o = odr_encode();
 985     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 986     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 987
 988     srw_res->num_diagnostics = 1;
 989     srw_res->diagnostics = (Z_SRW_diagnostic *)
 990         odr_malloc(o, sizeof(*srw_res->diagnostics));
 991     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
 992     return send_srw_response(srw_pdu);
 993 }
 994
 995 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 996                              Z_DefaultDiagFormat *ddf)
 997 {
 998     int bib1_code = *ddf->condition;
 999     if (bib1_code == 109)
1000         return 404;
1001     srw_res->num_diagnostics = 1;
1002     srw_res->diagnostics = (Z_SRW_diagnostic *)
1003         odr_malloc(o, sizeof(*srw_res->diagnostics));
1004     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1005                           yaz_diag_bib1_to_srw(*ddf->condition),
1006                           ddf->u.v2Addinfo);
1007     return 0;
1008 }
1009
1010 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1011 {
1012     ODR o = odr_encode();
1013     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1014     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1015
1016     srw_res->numberOfRecords = odr_intdup (o, hits);
1017     if (records && records->which == Z_Records_DBOSD)
1018     {
1019         srw_res->num_records =
1020             records->u.databaseOrSurDiagnostics->num_records;
1021         int i;
1022         srw_res->records = (Z_SRW_record *)
1023             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1024         for (i = 0; i < srw_res->num_records; i++)
1025         {
1026             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1027             if (npr->which != Z_NamePlusRecord_databaseRecord)
1028             {
1029                 srw_res->records[i].recordSchema = "diagnostic";
1030                 srw_res->records[i].recordPacking = m_s2z_packing;
1031                 srw_res->records[i].recordData_buf = "67";
1032                 srw_res->records[i].recordData_len = 2;
1033                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1034                 continue;
1035             }
1036             Z_External *r = npr->u.databaseRecord;
1037             oident *ent = oid_getentbyoid(r->direct_reference);
1038             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1039             {
1040                 srw_res->records[i].recordSchema = m_schema;
1041                 srw_res->records[i].recordPacking = m_s2z_packing;
1042                 srw_res->records[i].recordData_buf = (char*)
1043                     r->u.octet_aligned->buf;
1044                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1045                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1046             }
1047             else
1048             {
1049                 srw_res->records[i].recordSchema = "diagnostic";
1050                 srw_res->records[i].recordPacking = m_s2z_packing;
1051                 srw_res->records[i].recordData_buf = "67";
1052                 srw_res->records[i].recordData_len = 2;
1053                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1054             }
1055         }
1056     }
1057     if (records && records->which == Z_Records_NSD)
1058     {
1059         int http_code;
1060         http_code = z_to_srw_diag(odr_encode(), srw_res,
1061                                    records->u.nonSurrogateDiagnostic);
1062         if (http_code)
1063             return send_http_response(http_code);
1064     }
1065     return send_srw_response(srw_pdu);
1066
1067 }
1068
1069 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1070                                         int num_diagnostics)
1071 {
1072     Yaz_ProxyConfig *cfg = check_reconfigure();
1073     if (cfg)
1074     {
1075         int len;
1076         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1077                                        m_s2z_database, &len);
1078         if (b)
1079         {
1080             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1081             Z_SRW_explainResponse *er = res->u.explain_response;
1082
1083             er->record.recordData_buf = b;
1084             er->record.recordData_len = len;
1085             er->record.recordPacking = m_s2z_packing;
1086             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1087
1088             er->diagnostics = diagnostics;
1089             er->num_diagnostics = num_diagnostics;
1090             return send_srw_response(res);
1091         }
1092     }
1093     return send_http_response(404);
1094 }
1095
1096 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1097 {
1098     if (m_http_version)
1099     {
1100         if (apdu->which == Z_APDU_initResponse)
1101         {
1102             Z_InitResponse *res = apdu->u.initResponse;
1103             if (*res->result == 0)
1104             {
1105                 send_to_srw_client_error(3, 0);
1106             }
1107             else if (!m_s2z_search_apdu)
1108             {
1109                 send_srw_explain_response(0, 0);
1110             }
1111             else
1112             {
1113                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1114             }
1115         }
1116         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1117         {
1118             m_s2z_search_apdu = 0;
1119             Z_SearchResponse *res = apdu->u.searchResponse;
1120             m_s2z_hit_count = *res->resultCount;
1121             if (res->records && res->records->which == Z_Records_NSD)
1122             {
1123                 send_to_srw_client_ok(0, res->records, 1);
1124             }
1125             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1126             {
1127                 // adjust
1128                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1129
1130                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1131                 {
1132                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1133                         > m_s2z_hit_count)
1134                         *pr->numberOfRecordsRequested =
1135                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1136                 }
1137                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1138             }
1139             else
1140             {
1141                 m_s2z_present_apdu = 0;
1142                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1143             }
1144         }
1145         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1146         {
1147             int start =
1148                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1149
1150             m_s2z_present_apdu = 0;
1151             Z_PresentResponse *res = apdu->u.presentResponse;
1152             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1153         }
1154     }
1155     else
1156     {
1157         int len = 0;
1158         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1159             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1160                      apdu_name(apdu));
1161         int r = send_Z_PDU(apdu, &len);
1162         m_bytes_sent += len;
1163         m_bw_stat.add_bytes(len);
1164         logtime();
1165         return r;
1166     }
1167     return 0;
1168 }
1169
1170 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1171 {
1172     int kill_session = 0;
1173     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1174
1175     if (new_id)
1176         *new_id = m_referenceId;
1177
1178     if (apdu->which == Z_APDU_searchResponse)
1179     {
1180         Z_SearchResponse *sr = apdu->u.searchResponse;
1181         Z_Records *p = sr->records;
1182         if (p && p->which == Z_Records_NSD)
1183         {
1184             Z_DiagRec dr, *dr_p = &dr;
1185             dr.which = Z_DiagRec_defaultFormat;
1186             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1187
1188             *sr->searchStatus = 0;
1189             display_diagrecs(&dr_p, 1);
1190         }
1191         else
1192         {
1193             if (p && p->which == Z_Records_DBOSD)
1194             {
1195                 if (m_backend_type
1196 #if HAVE_USEMARCON
1197                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1198 #endif
1199                     )
1200                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1201                 if (m_marcxml_flag)
1202                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1203                                        m_backend_charset);
1204                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1205                     return 0;
1206
1207             }
1208             if (sr->resultCount)
1209             {
1210                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1211                         *sr->resultCount);
1212                 if (*sr->resultCount < 0)
1213                 {
1214                     m_invalid_session = 1;
1215                     kill_session = 1;
1216
1217                     *sr->searchStatus = 0;
1218                     sr->records =
1219                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1220                     *sr->resultCount = 0;
1221                 }
1222             }
1223         }
1224     }
1225     else if (apdu->which == Z_APDU_presentResponse)
1226     {
1227         Z_PresentResponse *sr = apdu->u.presentResponse;
1228         Z_Records *p = sr->records;
1229         if (p && p->which == Z_Records_NSD)
1230         {
1231             Z_DiagRec dr, *dr_p = &dr;
1232             dr.which = Z_DiagRec_defaultFormat;
1233             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1234             if (*sr->presentStatus == Z_PresentStatus_success)
1235                 *sr->presentStatus = Z_PresentStatus_failure;
1236             display_diagrecs(&dr_p, 1);
1237         }
1238         if (p && p->which == Z_Records_DBOSD)
1239         {
1240             if (m_backend_type
1241 #if HAVE_USEMARCON
1242                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1243 #endif
1244                 )
1245                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1246             if (m_marcxml_flag)
1247                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1248                                    m_backend_charset);
1249             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1250                 return 0;
1251         }
1252     }
1253     else if (apdu->which == Z_APDU_initResponse)
1254     {
1255         //Get and check negotiation record
1256         //from init response.
1257         handle_charset_lang_negotiation(apdu);
1258
1259         if (m_initRequest_options)
1260         {
1261             Z_Options *nopt =
1262                 (Odr_bitmask *)odr_malloc(odr_encode(),
1263                                           sizeof(Odr_bitmask));
1264             ODR_MASK_ZERO(nopt);
1265
1266             int i;
1267             for (i = 0; i<24; i++)
1268                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1269                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1270                     ODR_MASK_SET(nopt, i);
1271             apdu->u.initResponse->options = nopt;
1272         }
1273         if (m_initRequest_version)
1274         {
1275             Z_ProtocolVersion *nopt =
1276                 (Odr_bitmask *)odr_malloc(odr_encode(),
1277                                           sizeof(Odr_bitmask));
1278             ODR_MASK_ZERO(nopt);
1279
1280             int i;
1281             for (i = 0; i<8; i++)
1282                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1283                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1284                     ODR_MASK_SET(nopt, i);
1285             apdu->u.initResponse->protocolVersion = nopt;
1286         }
1287         apdu->u.initResponse->preferredMessageSize =
1288             odr_intdup(odr_encode(),
1289                        m_client->m_initResponse_preferredMessageSize >
1290                        m_initRequest_preferredMessageSize ?
1291                        m_initRequest_preferredMessageSize :
1292                        m_client->m_initResponse_preferredMessageSize);
1293         apdu->u.initResponse->maximumRecordSize =
1294             odr_intdup(odr_encode(),
1295                        m_client->m_initResponse_maximumRecordSize >
1296                        m_initRequest_maximumRecordSize ?
1297                        m_initRequest_maximumRecordSize :
1298                        m_client->m_initResponse_maximumRecordSize);
1299     }
1300
1301     int r = send_PDU_convert(apdu);
1302     if (r)
1303         return r;
1304     if (kill_session)
1305     {
1306         delete m_client;
1307         m_client = 0;
1308         m_parent->pre_init();
1309     }
1310     return r;
1311 }
1312
1313 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1314 {
1315     int len = 0;
1316     const char *apdu_name_tmp = apdu_name(apdu);
1317     int r = send_Z_PDU(apdu, &len);
1318     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1319         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1320                  get_session_str(),
1321                  apdu_name_tmp, get_hostname(), len);
1322     m_bytes_sent += len;
1323     return r;
1324 }
1325
1326 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1327 {
1328     if (apdu->which == Z_APDU_presentRequest)
1329     {
1330         Z_PresentRequest *pr = apdu->u.presentRequest;
1331         int toget = *pr->numberOfRecordsRequested;
1332         int start = *pr->resultSetStartPoint;
1333
1334         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1335                 pr->resultSetId, start, toget);
1336
1337         if (*m_parent->m_optimize == '0')
1338             return apdu;
1339
1340         if (!m_client->m_last_resultSetId)
1341         {
1342             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1343             new_apdu->u.presentResponse->records =
1344                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1345                                                pr->resultSetId);
1346             send_to_client(new_apdu);
1347             return 0;
1348         }
1349         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1350         {
1351             if (start+toget-1 > m_client->m_last_resultCount)
1352             {
1353                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1354                 new_apdu->u.presentResponse->records =
1355                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1356                 send_to_client(new_apdu);
1357                 return 0;
1358             }
1359             Z_NamePlusRecordList *npr;
1360 #if 0
1361             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1362                     m_session_str, start, toget, yaz_z3950oid_to_str(
1363                         pr->preferredRecordSyntax, &oclass));
1364 #endif
1365             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1366                                           pr->preferredRecordSyntax,
1367                                           pr->recordComposition))
1368             {
1369                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1370                          m_session_str);
1371                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1372                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1373
1374                 new_apdu->u.presentResponse->numberOfRecordsReturned
1375                     = odr_intdup(odr_encode(), toget);
1376
1377                 new_apdu->u.presentResponse->records = (Z_Records*)
1378                     odr_malloc(odr_encode(), sizeof(Z_Records));
1379                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1380                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1381                 new_apdu->u.presentResponse->nextResultSetPosition =
1382                     odr_intdup(odr_encode(), start+toget);
1383
1384                 send_to_client(new_apdu);
1385                 return 0;
1386             }
1387         }
1388     }
1389
1390     if (apdu->which != Z_APDU_searchRequest)
1391         return apdu;
1392     Z_SearchRequest *sr = apdu->u.searchRequest;
1393     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1394     Yaz_Z_Databases this_databases;
1395
1396     this_databases.set(sr->num_databaseNames, (const char **)
1397                        sr->databaseNames);
1398
1399     this_query->set_Z_Query(sr->query);
1400
1401     char query_str[120];
1402     this_query->print(query_str, sizeof(query_str)-1);
1403     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1404
1405     if (*m_parent->m_optimize != '0' &&
1406         m_client->m_last_ok && m_client->m_last_query &&
1407         m_client->m_last_query->match(this_query) &&
1408         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1409         m_client->m_last_databases.match(this_databases))
1410     {
1411         delete this_query;
1412         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1413             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1414         {
1415             Z_NamePlusRecordList *npr;
1416             int toget = *sr->mediumSetPresentNumber;
1417             Z_RecordComposition *comp = 0;
1418
1419             if (toget > m_client->m_last_resultCount)
1420                 toget = m_client->m_last_resultCount;
1421
1422             if (sr->mediumSetElementSetNames)
1423             {
1424                 comp = (Z_RecordComposition *)
1425                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1426                 comp->which = Z_RecordComp_simple;
1427                 comp->u.simple = sr->mediumSetElementSetNames;
1428             }
1429
1430             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1431                                           sr->preferredRecordSyntax, comp))
1432             {
1433                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1434                          m_session_str);
1435                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1436                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1437                 new_apdu->u.searchResponse->resultCount =
1438                     &m_client->m_last_resultCount;
1439
1440                 new_apdu->u.searchResponse->numberOfRecordsReturned
1441                     = odr_intdup(odr_encode(), toget);
1442
1443                 new_apdu->u.searchResponse->presentStatus =
1444                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1445                 new_apdu->u.searchResponse->records = (Z_Records*)
1446                     odr_malloc(odr_encode(), sizeof(Z_Records));
1447                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1448                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1449                 new_apdu->u.searchResponse->nextResultSetPosition =
1450                     odr_intdup(odr_encode(), toget+1);
1451                 send_to_client(new_apdu);
1452                 return 0;
1453             }
1454             else
1455             {
1456                 // medium Set
1457                 // send present request (medium size)
1458                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1459                          m_session_str);
1460
1461                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1462                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1463                 pr->referenceId = sr->referenceId;
1464                 pr->resultSetId = sr->resultSetName;
1465                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1466                 *pr->numberOfRecordsRequested = toget;
1467                 pr->recordComposition = comp;
1468                 m_client->m_sr_transform = 1;
1469                 return new_apdu;
1470             }
1471         }
1472         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1473             m_client->m_last_resultCount <= 0)
1474         {
1475             // large set. Return pseudo-search response immediately
1476             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1477                      m_session_str);
1478             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1479             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1480             new_apdu->u.searchResponse->resultCount =
1481                 &m_client->m_last_resultCount;
1482             send_to_client(new_apdu);
1483             return 0;
1484         }
1485         else
1486         {
1487             Z_NamePlusRecordList *npr;
1488             int toget = m_client->m_last_resultCount;
1489             Z_RecordComposition *comp = 0;
1490             // small set
1491             // send a present request (small set)
1492
1493             if (sr->smallSetElementSetNames)
1494             {
1495                 comp = (Z_RecordComposition *)
1496                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1497                 comp->which = Z_RecordComp_simple;
1498                 comp->u.simple = sr->smallSetElementSetNames;
1499             }
1500
1501             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1502                                           sr->preferredRecordSyntax, comp))
1503             {
1504                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1505                          m_session_str);
1506                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1507                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1508                 new_apdu->u.searchResponse->resultCount =
1509                     &m_client->m_last_resultCount;
1510
1511                 new_apdu->u.searchResponse->numberOfRecordsReturned
1512                     = odr_intdup(odr_encode(), toget);
1513
1514                 new_apdu->u.searchResponse->presentStatus =
1515                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1516                 new_apdu->u.searchResponse->records = (Z_Records*)
1517                     odr_malloc(odr_encode(), sizeof(Z_Records));
1518                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1519                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1520                 new_apdu->u.searchResponse->nextResultSetPosition =
1521                     odr_intdup(odr_encode(), toget+1);
1522                 send_to_client(new_apdu);
1523                 return 0;
1524             }
1525             else
1526             {
1527                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1528                          m_session_str);
1529                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1530                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1531                 pr->referenceId = sr->referenceId;
1532                 pr->resultSetId = sr->resultSetName;
1533                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1534                 *pr->numberOfRecordsRequested = toget;
1535                 pr->recordComposition = comp;
1536                 m_client->m_sr_transform = 1;
1537                 return new_apdu;
1538             }
1539         }
1540     }
1541     else  // query doesn't match
1542     {
1543         delete m_client->m_last_query;
1544         m_client->m_last_query = this_query;
1545         m_client->m_last_ok = 0;
1546         m_client->m_cache.clear();
1547         m_client->m_resultSetStartPoint = 0;
1548
1549         xfree (m_client->m_last_resultSetId);
1550         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1551
1552         m_client->m_last_databases.set(sr->num_databaseNames,
1553                                        (const char **) sr->databaseNames);
1554     }
1555     return apdu;
1556 }
1557
1558
1559 void Yaz_Proxy::inc_request_no()
1560 {
1561     char *cp = strchr(m_session_str, ' ');
1562     m_request_no++;
1563     if (cp)
1564         sprintf(cp+1, "%d ", m_request_no);
1565 }
1566
1567 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1568 {
1569     inc_request_no();
1570
1571     m_bytes_recv += len;
1572
1573     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1574         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1575                  m_session_str, gdu_name(apdu), len);
1576
1577     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1578         shutdown();
1579
1580     m_bw_stat.add_bytes(len);
1581     m_pdu_stat.add_bytes(1);
1582
1583 #if HAVE_GETTIMEOFDAY
1584     gettimeofday((struct timeval *) m_time_tv, 0);
1585 #endif
1586
1587     int bw_total = m_bw_stat.get_total();
1588     int pdu_total = m_pdu_stat.get_total();
1589
1590     int reduce = 0;
1591     if (m_bw_max)
1592     {
1593         if (bw_total > m_bw_max)
1594         {
1595             reduce = (bw_total/m_bw_max);
1596         }
1597     }
1598     if (m_pdu_max)
1599     {
1600         if (pdu_total > m_pdu_max)
1601         {
1602             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1603             reduce = (reduce > nreduce) ? reduce : nreduce;
1604         }
1605     }
1606     m_http_version = 0;
1607     if (reduce)
1608     {
1609         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1610                 m_session_str, reduce, bw_total, pdu_total,
1611                 m_bw_max, m_pdu_max);
1612
1613         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1614         timeout(reduce);       // call us reduce seconds later
1615     }
1616     else if (apdu->which == Z_GDU_Z3950)
1617         handle_incoming_Z_PDU(apdu->u.z3950);
1618     else if (apdu->which == Z_GDU_HTTP_Request)
1619         handle_incoming_HTTP(apdu->u.HTTP_Request);
1620 }
1621
1622 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1623 {
1624     if (m_max_record_retrieve)
1625     {
1626         if (apdu->which == Z_APDU_presentRequest)
1627         {
1628             Z_PresentRequest *pr = apdu->u.presentRequest;
1629             if (pr->numberOfRecordsRequested &&
1630                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1631                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1632         }
1633     }
1634 }
1635 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1636 {
1637     if (apdu->which == Z_APDU_initRequest)
1638     {
1639         if (m_initRequest_options &&
1640             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1641             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1642         {
1643             // There is not negotiation proposal from
1644             // client's side. OK. The proxy negotiation
1645             // in use, only.
1646             Z_InitRequest *initRequest = apdu->u.initRequest;
1647             Z_OtherInformation **otherInfo;
1648             Z_OtherInformationUnit *oi;
1649             get_otherInfoAPDU(apdu, &otherInfo);
1650             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1651             if (oi)
1652             {
1653                 ODR_MASK_SET(initRequest->options,
1654                     Z_Options_negotiationModel);
1655                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1656                 oi->information.externallyDefinedInfo =
1657                 yaz_set_proposal_charneg(odr_encode(),
1658                     (const char**)&m_proxy_negotiation_charset,
1659                     m_proxy_negotiation_charset ? 1:0,
1660                     (const char**)&m_proxy_negotiation_lang,
1661                     m_proxy_negotiation_lang ? 1:0,
1662                     1);
1663             }
1664         }
1665     }
1666     else if (apdu->which == Z_APDU_initResponse)
1667     {
1668         Z_InitResponse *initResponse = apdu->u.initResponse;
1669         Z_OtherInformation **otherInfo;
1670
1671         if (ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel))
1672         {
1673             char *charset = 0;
1674             char *lang = 0;
1675             int selected = 0;
1676
1677             get_otherInfoAPDU(apdu, &otherInfo);
1678
1679             if (!otherInfo && !(*otherInfo))
1680                 return;
1681
1682             Z_CharSetandLanguageNegotiation *charneg =
1683                 yaz_get_charneg_record(*otherInfo);
1684
1685             if (!charneg)
1686                 return;
1687
1688             yaz_get_response_charneg(m_referenceId_mem, charneg,
1689                 &charset, &lang, &selected);
1690
1691             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
1692                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
1693
1694             if (m_initRequest_options &&
1695                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
1696             {
1697                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
1698                     m_session_str);
1699             }
1700             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1701             {
1702                 // negotiation-charset, negotiation-lang
1703                 // elements of config file in use.
1704
1705                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
1706                     m_session_str);
1707
1708                 // clear negotiation option.
1709                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
1710
1711                 // Delete negotiation (charneg-3) entry.
1712                 Z_OtherInformation *p = *otherInfo;
1713                 for (int i=0; i<p->num_elements; i++)
1714                 {
1715                     if (p->list[i]->which == Z_OtherInfo_externallyDefinedInfo)
1716                     {
1717                         Z_External *pext =
1718                             p->list[i]->information.externallyDefinedInfo;
1719                         struct oident *e = oid_getentbyoid(pext->direct_reference);
1720
1721                         if (e && e->value == VAL_CHARNEG3 && e->oclass == CLASS_NEGOT &&
1722                             pext->which == Z_External_charSetandLanguageNegotiation)
1723                         {
1724                             (p->num_elements)--;
1725                             if(p->num_elements == 0)
1726                             {
1727                                 *otherInfo = 0;
1728                             }
1729                             else
1730                             {
1731                                 for (int j=i; j<p->num_elements;j++)
1732                                     p->list[j] = p->list[j+1];
1733                             }
1734                         }
1735                     }
1736                 }
1737             }
1738         }
1739     }
1740 }
1741 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1742                                                      int error,
1743                                                      const char *addinfo)
1744 {
1745     Z_Records *rec = (Z_Records *)
1746         odr_malloc (odr, sizeof(*rec));
1747     int *err = (int *)
1748         odr_malloc (odr, sizeof(*err));
1749     Z_DiagRec *drec = (Z_DiagRec *)
1750         odr_malloc (odr, sizeof(*drec));
1751     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1752         odr_malloc (odr, sizeof(*dr));
1753     *err = error;
1754     rec->which = Z_Records_NSD;
1755     rec->u.nonSurrogateDiagnostic = dr;
1756     dr->diagnosticSetId =
1757         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1758     dr->condition = err;
1759     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1760     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1761     return rec;
1762 }
1763
1764 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1765 {
1766     if (apdu->which == Z_APDU_searchRequest &&
1767         apdu->u.searchRequest->query &&
1768         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1769         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1770     {
1771         Z_RPNQuery *rpnquery = 0;
1772         Z_SearchRequest *sr = apdu->u.searchRequest;
1773         char *addinfo = 0;
1774
1775         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
1776                 sr->query->u.type_104->u.cql);
1777
1778         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1779                                           &rpnquery, odr_encode(),
1780                                           &addinfo);
1781         if (r == -3)
1782             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
1783         else if (r)
1784         {
1785             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1786             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1787
1788             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1789             new_apdu->u.searchResponse->records =
1790                 create_nonSurrogateDiagnostics(odr_encode(),
1791                                                yaz_diag_srw_to_bib1(r),
1792                                                addinfo);
1793             *new_apdu->u.searchResponse->searchStatus = 0;
1794
1795             send_to_client(new_apdu);
1796
1797             return 0;
1798         }
1799         else
1800         {
1801             sr->query->which = Z_Query_type_1;
1802             sr->query->u.type_1 = rpnquery;
1803         }
1804         return apdu;
1805     }
1806     return apdu;
1807 }
1808
1809 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1810 {
1811     if (apdu->which == Z_APDU_searchRequest)
1812     {
1813         Z_SearchRequest *sr = apdu->u.searchRequest;
1814         int err = 0;
1815         char *addinfo = 0;
1816
1817         Yaz_ProxyConfig *cfg = check_reconfigure();
1818         if (cfg)
1819             err = cfg->check_query(odr_encode(), m_default_target,
1820                                    sr->query, &addinfo);
1821         if (err)
1822         {
1823             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1824
1825             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1826             new_apdu->u.searchResponse->records =
1827                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1828             *new_apdu->u.searchResponse->searchStatus = 0;
1829
1830             send_to_client(new_apdu);
1831
1832             return 0;
1833         }
1834     }
1835     return apdu;
1836 }
1837
1838 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1839 {
1840     m_marcxml_flag = 0;
1841     if (apdu->which == Z_APDU_searchRequest)
1842     {
1843         Z_SearchRequest *sr = apdu->u.searchRequest;
1844         int err = 0;
1845         char *addinfo = 0;
1846         Yaz_ProxyConfig *cfg = check_reconfigure();
1847
1848         Z_RecordComposition rc_temp, *rc = 0;
1849         if (sr->smallSetElementSetNames)
1850         {
1851             rc_temp.which = Z_RecordComp_simple;
1852             rc_temp.u.simple = sr->smallSetElementSetNames;
1853             rc = &rc_temp;
1854         }
1855
1856         if (sr->preferredRecordSyntax)
1857         {
1858             struct oident *ent;
1859             ent = oid_getentbyoid(sr->preferredRecordSyntax);
1860             m_frontend_type = ent->value;
1861         }
1862         else
1863             m_frontend_type = VAL_NONE;
1864
1865         char *stylesheet_name = 0;
1866         if (cfg)
1867             err = cfg->check_syntax(odr_encode(),
1868                                     m_default_target,
1869                                     sr->preferredRecordSyntax, rc,
1870                                     &addinfo, &stylesheet_name, &m_schema,
1871                                     &m_backend_type, &m_backend_charset,
1872                                     &m_usemarcon_ini_stage1,
1873                                     &m_usemarcon_ini_stage2);
1874         if (stylesheet_name)
1875         {
1876             m_parent->low_socket_close();
1877
1878 #if HAVE_XSLT
1879             if (m_stylesheet_xsp)
1880                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
1881             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1882                                                        stylesheet_name);
1883 #endif
1884             m_stylesheet_offset = 0;
1885             xfree(stylesheet_name);
1886
1887             m_parent->low_socket_open();
1888         }
1889         if (err == -1)
1890         {
1891             sr->smallSetElementSetNames = 0;
1892             sr->mediumSetElementSetNames = 0;
1893             if (m_backend_type)
1894             {
1895
1896                 sr->preferredRecordSyntax =
1897                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
1898                                         m_backend_type);
1899             }
1900             else
1901                 sr->preferredRecordSyntax =
1902                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
1903                                            VAL_USMARC);
1904             m_marcxml_flag = 1;
1905         }
1906         else if (err)
1907         {
1908             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1909
1910             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1911             new_apdu->u.searchResponse->records =
1912                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1913             *new_apdu->u.searchResponse->searchStatus = 0;
1914
1915             send_to_client(new_apdu);
1916
1917             return 0;
1918         }
1919         else if (m_backend_type)
1920         {
1921             sr->preferredRecordSyntax =
1922                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
1923         }
1924     }
1925     else if (apdu->which == Z_APDU_presentRequest)
1926     {
1927         Z_PresentRequest *pr = apdu->u.presentRequest;
1928         int err = 0;
1929         char *addinfo = 0;
1930         Yaz_ProxyConfig *cfg = check_reconfigure();
1931
1932         if (pr->preferredRecordSyntax)
1933         {
1934             struct oident *ent;
1935             ent = oid_getentbyoid(pr->preferredRecordSyntax);
1936             m_frontend_type = ent->value;
1937         }
1938         else
1939             m_frontend_type = VAL_NONE;
1940
1941         char *stylesheet_name = 0;
1942         if (cfg)
1943             err = cfg->check_syntax(odr_encode(), m_default_target,
1944                                     pr->preferredRecordSyntax,
1945                                     pr->recordComposition,
1946                                     &addinfo, &stylesheet_name, &m_schema,
1947                                     &m_backend_type, &m_backend_charset,
1948                                     &m_usemarcon_ini_stage1,
1949                                     &m_usemarcon_ini_stage2
1950                                     );
1951         if (stylesheet_name)
1952         {
1953             m_parent->low_socket_close();
1954
1955 #if HAVE_XSLT
1956             if (m_stylesheet_xsp)
1957                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
1958             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1959                                                        stylesheet_name);
1960 #endif
1961             m_stylesheet_offset = 0;
1962             xfree(stylesheet_name);
1963
1964             m_parent->low_socket_open();
1965         }
1966         if (err == -1)
1967         {
1968             pr->recordComposition = 0;
1969             if (m_backend_type)
1970             {
1971
1972                 pr->preferredRecordSyntax =
1973                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
1974                                         m_backend_type);
1975             }
1976             else
1977                 pr->preferredRecordSyntax =
1978                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
1979                                            VAL_USMARC);
1980             m_marcxml_flag = 1;
1981         }
1982         else if (err)
1983         {
1984             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1985
1986             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1987             new_apdu->u.presentResponse->records =
1988                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1989             *new_apdu->u.presentResponse->presentStatus =
1990                 Z_PresentStatus_failure;
1991
1992             send_to_client(new_apdu);
1993
1994             return 0;
1995         }
1996         else if (m_backend_type)
1997         {
1998             pr->preferredRecordSyntax =
1999                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2000         }
2001     }
2002     return apdu;
2003 }
2004
2005 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2006 {
2007     if (!schema)
2008         return 0;
2009     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2010         odr_malloc(o, sizeof(Z_ElementSetNames));
2011     esn->which = Z_ElementSetNames_generic;
2012     esn->u.generic = odr_strdup(o, schema);
2013     return esn;
2014 }
2015
2016 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2017 {
2018     const char *t = 0;
2019     Yaz_ProxyConfig *cfg = check_reconfigure();
2020     if (cfg)
2021         t = cfg->get_explain_name(db, backend_db);
2022
2023     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2024     {
2025         releaseClient();
2026     }
2027
2028     if (t)
2029     {
2030         xfree(m_default_target);
2031         m_default_target = xstrdup(t);
2032     }
2033 }
2034
2035 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2036 {
2037     struct stat sbuf;
2038     yaz_log(YLOG_LOG, "file_access");
2039     if (strcmp(hreq->method, "GET"))
2040         return 0;
2041     if (hreq->path[0] != '/')
2042     {
2043         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2044         return 0;
2045     }
2046     const char *cp = hreq->path;
2047     while (*cp)
2048     {
2049         if (*cp == '/' && strchr("/.", cp[1]))
2050         {
2051             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2052             return 0;
2053         }
2054         cp++;
2055     }
2056     const char *fname = hreq->path+1;
2057     if (stat(fname, &sbuf))
2058     {
2059         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2060         return 0;
2061     }
2062     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2063     {
2064         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2065         return 0;
2066     }
2067     if (sbuf.st_size > (off_t) 1000000)
2068     {
2069         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2070         return 0;
2071     }
2072
2073     ODR o = odr_encode();
2074     Yaz_ProxyConfig *cfg = check_reconfigure();
2075     const char *ctype = cfg->check_mime_type(fname);
2076     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2077     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2078     if (m_http_version)
2079         hres->version = odr_strdup(o, m_http_version);
2080     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2081     if (m_http_keepalive)
2082         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2083     else
2084         timeout(0);
2085
2086     hres->content_len = sbuf.st_size;
2087     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2088     FILE *f = fopen(fname, "rb");
2089     if (f)
2090     {
2091         fread(hres->content_buf, 1, hres->content_len, f);
2092         fclose(f);
2093     }
2094     else
2095     {
2096         return 0;
2097     }
2098     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2099     {
2100         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2101                  fname);
2102     }
2103     int len;
2104     send_GDU(gdu, &len);
2105     return 1;
2106 }
2107
2108 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2109 {
2110     if (m_s2z_odr_init)
2111     {
2112         odr_destroy(m_s2z_odr_init);
2113         m_s2z_odr_init = 0;
2114     }
2115     if (m_s2z_odr_search)
2116     {
2117         odr_destroy(m_s2z_odr_search);
2118         m_s2z_odr_search = 0;
2119     }
2120
2121     m_http_keepalive = 0;
2122     m_http_version = 0;
2123     if (!strcmp(hreq->version, "1.0"))
2124     {
2125         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2126         if (v && !strcmp(v, "Keep-Alive"))
2127             m_http_keepalive = 1;
2128         else
2129             m_http_keepalive = 0;
2130         m_http_version = "1.0";
2131     }
2132     else
2133     {
2134         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2135         if (v && !strcmp(v, "close"))
2136             m_http_keepalive = 0;
2137         else
2138             m_http_keepalive = 1;
2139         m_http_version = "1.1";
2140     }
2141
2142     Z_SRW_PDU *srw_pdu = 0;
2143     Z_SOAP *soap_package = 0;
2144     char *charset = 0;
2145     Z_SRW_diagnostic *diagnostic = 0;
2146     int num_diagnostic = 0;
2147
2148     if (file_access(hreq))
2149     {
2150         return;
2151     }
2152     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2153                             &charset) == 0
2154              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2155                                &charset, &diagnostic, &num_diagnostic) == 0)
2156     {
2157         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2158         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2159         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2160         m_s2z_init_apdu = 0;
2161         m_s2z_search_apdu = 0;
2162         m_s2z_present_apdu = 0;
2163
2164         m_s2z_stylesheet = 0;
2165
2166         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2167         {
2168             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2169
2170             const char *backend_db = srw_req->database;
2171             srw_get_client(srw_req->database, &backend_db);
2172
2173             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2174             // recordXPath unsupported.
2175             if (srw_req->recordXPath)
2176             {
2177                 yaz_add_srw_diagnostic(odr_decode(),
2178                                        &diagnostic, &num_diagnostic,
2179                                        72, 0);
2180             }
2181             // sort unsupported
2182             if (srw_req->sort_type != Z_SRW_sort_type_none)
2183             {
2184                 yaz_add_srw_diagnostic(odr_decode(),
2185                                        &diagnostic, &num_diagnostic,
2186                                        80, 0);
2187             }
2188             // save stylesheet
2189             if (srw_req->stylesheet)
2190                 m_s2z_stylesheet =
2191                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2192
2193             // set packing for response records ..
2194             if (srw_req->recordPacking &&
2195                 !strcmp(srw_req->recordPacking, "xml"))
2196                 m_s2z_packing = Z_SRW_recordPacking_XML;
2197             else
2198                 m_s2z_packing = Z_SRW_recordPacking_string;
2199
2200             if (num_diagnostic)
2201             {
2202                 Z_SRW_PDU *srw_pdu =
2203                     yaz_srw_get(odr_encode(),
2204                                 Z_SRW_searchRetrieve_response);
2205                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2206
2207                 srw_res->diagnostics = diagnostic;
2208                 srw_res->num_diagnostics = num_diagnostic;
2209                 send_srw_response(srw_pdu);
2210                 return;
2211             }
2212
2213             // prepare search PDU
2214             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2215                                           Z_APDU_searchRequest);
2216             Z_SearchRequest *z_searchRequest =
2217                 m_s2z_search_apdu->u.searchRequest;
2218
2219             z_searchRequest->num_databaseNames = 1;
2220             z_searchRequest->databaseNames = (char**)
2221                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2222             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2223                                                            backend_db);
2224
2225             // query transformation
2226             Z_Query *query = (Z_Query *)
2227                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2228             z_searchRequest->query = query;
2229
2230             if (srw_req->query_type == Z_SRW_query_type_cql)
2231             {
2232                 Z_External *ext = (Z_External *)
2233                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2234                 ext->direct_reference =
2235                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2236                 ext->indirect_reference = 0;
2237                 ext->descriptor = 0;
2238                 ext->which = Z_External_CQL;
2239                 ext->u.cql = srw_req->query.cql;
2240
2241                 query->which = Z_Query_type_104;
2242                 query->u.type_104 =  ext;
2243             }
2244             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2245             {
2246                 Z_RPNQuery *RPNquery;
2247                 YAZ_PQF_Parser pqf_parser;
2248
2249                 pqf_parser = yaz_pqf_create ();
2250
2251                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2252                                           srw_req->query.pqf);
2253                 if (!RPNquery)
2254                 {
2255                     const char *pqf_msg;
2256                     size_t off;
2257                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2258                     yaz_log(YLOG_LOG, "%*s^\n", off+4, "");
2259                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2260
2261                     send_to_srw_client_error(10, 0);
2262                     return;
2263                 }
2264                 query->which = Z_Query_type_1;
2265                 query->u.type_1 =  RPNquery;
2266
2267                 yaz_pqf_destroy (pqf_parser);
2268             }
2269             else
2270             {
2271                 send_to_srw_client_error(7, "query");
2272                 return;
2273             }
2274
2275             // present
2276             m_s2z_present_apdu = 0;
2277             int max = 0;
2278             if (srw_req->maximumRecords)
2279                 max = *srw_req->maximumRecords;
2280             int start = 1;
2281             if (srw_req->startRecord)
2282                 start = *srw_req->startRecord;
2283             if (max > 0)
2284             {
2285                 // Some backend, such as Voyager doesn't honor piggyback
2286                 // So we use present always (0 &&).
2287                 if (0 && start <= 1)  // Z39.50 piggyback
2288                 {
2289                     *z_searchRequest->smallSetUpperBound = max;
2290                     *z_searchRequest->mediumSetPresentNumber = max;
2291                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2292
2293                     z_searchRequest->preferredRecordSyntax =
2294                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2295                                                VAL_TEXT_XML);
2296                     if (srw_req->recordSchema)
2297                     {
2298                         z_searchRequest->smallSetElementSetNames =
2299                             z_searchRequest->mediumSetElementSetNames =
2300                             mk_esn_from_schema(m_s2z_odr_search,
2301                                                srw_req->recordSchema);
2302                     }
2303                 }
2304                 else   // Z39.50 present
2305                 {
2306                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2307                                                    Z_APDU_presentRequest);
2308                     Z_PresentRequest *z_presentRequest =
2309                         m_s2z_present_apdu->u.presentRequest;
2310                     *z_presentRequest->resultSetStartPoint = start;
2311                     *z_presentRequest->numberOfRecordsRequested = max;
2312                     z_presentRequest->preferredRecordSyntax =
2313                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2314                                                VAL_TEXT_XML);
2315                     if (srw_req->recordSchema)
2316                     {
2317                         z_presentRequest->recordComposition =
2318                             (Z_RecordComposition *)
2319                             odr_malloc(m_s2z_odr_search,
2320                                        sizeof(Z_RecordComposition));
2321                         z_presentRequest->recordComposition->which =
2322                             Z_RecordComp_simple;
2323                         z_presentRequest->recordComposition->u.simple =
2324                             mk_esn_from_schema(m_s2z_odr_search,
2325                                                srw_req->recordSchema);
2326                     }
2327                 }
2328             }
2329             if (!m_client)
2330             {
2331                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2332                                             Z_APDU_initRequest);
2333
2334                 // prevent m_initRequest_apdu memory from being grabbed
2335                 // in Yaz_Proxy::handle_incoming_Z_PDU
2336                 m_initRequest_apdu = m_s2z_init_apdu;
2337                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2338                 return;
2339             }
2340             else
2341             {
2342                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2343                 return;
2344             }
2345         }
2346         else if (srw_pdu->which == Z_SRW_explain_request)
2347         {
2348             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2349
2350             const char *backend_db = srw_req->database;
2351             srw_get_client(srw_req->database, &backend_db);
2352
2353             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2354
2355             // save stylesheet
2356             if (srw_req->stylesheet)
2357                 m_s2z_stylesheet =
2358                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2359
2360             if (srw_req->recordPacking &&
2361                 !strcmp(srw_req->recordPacking, "xml"))
2362                 m_s2z_packing = Z_SRW_recordPacking_XML;
2363             else
2364                 m_s2z_packing = Z_SRW_recordPacking_string;
2365
2366             if (num_diagnostic)
2367             {
2368                 send_srw_explain_response(diagnostic, num_diagnostic);
2369                 return;
2370             }
2371
2372             if (!m_client)
2373             {
2374                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2375                                             Z_APDU_initRequest);
2376
2377                 // prevent m_initRequest_apdu memory from being grabbed
2378                 // in Yaz_Proxy::handle_incoming_Z_PDU
2379                 m_initRequest_apdu = m_s2z_init_apdu;
2380                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2381             }
2382             else
2383                 send_srw_explain_response(0, 0);
2384             return;
2385         }
2386         else if (srw_pdu->which == Z_SRW_scan_request)
2387         {
2388             m_s2z_database = odr_strdup(m_s2z_odr_init,
2389                                         srw_pdu->u.scan_request->database);
2390
2391             yaz_add_srw_diagnostic(odr_decode(),
2392                                    &diagnostic, &num_diagnostic,
2393                                    4, "scan");
2394             Z_SRW_PDU *srw_pdu =
2395                 yaz_srw_get(odr_encode(),
2396                             Z_SRW_scan_response);
2397             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2398
2399             srw_res->diagnostics = diagnostic;
2400             srw_res->num_diagnostics = num_diagnostic;
2401             send_srw_response(srw_pdu);
2402             return;
2403         }
2404         else
2405         {
2406             m_s2z_database = 0;
2407
2408             send_to_srw_client_error(4, 0);
2409         }
2410     }
2411     send_http_response(400);
2412 }
2413
2414 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2415 {
2416     Z_ReferenceId **refid = get_referenceIdP(apdu);
2417     nmem_reset(m_referenceId_mem);
2418     if (refid && *refid)
2419     {
2420         m_referenceId = (Z_ReferenceId *)
2421             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2422         m_referenceId->len = m_referenceId->size = (*refid)->len;
2423         m_referenceId->buf = (unsigned char *)
2424             nmem_malloc(m_referenceId_mem, (*refid)->len);
2425         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2426     }
2427     else
2428         m_referenceId = 0;
2429
2430     if (!m_client && m_invalid_session)
2431     {
2432         m_apdu_invalid_session = apdu;
2433         m_mem_invalid_session = odr_extract_mem(odr_decode());
2434         apdu = m_initRequest_apdu;
2435     }
2436
2437     // Determine our client.
2438     Z_OtherInformation **oi;
2439     get_otherInfoAPDU(apdu, &oi);
2440     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2441     if (!m_client)
2442     {
2443         if (m_http_version)
2444         {
2445             send_http_response(404);
2446             return;
2447         }
2448         else
2449         {
2450             delete this;
2451             return;
2452         }
2453     }
2454
2455     m_client->m_server = this;
2456
2457     if (apdu->which == Z_APDU_initRequest)
2458     {
2459         if (apdu->u.initRequest->implementationId)
2460             yaz_log(YLOG_LOG, "%simplementationId: %s",
2461                     m_session_str, apdu->u.initRequest->implementationId);
2462         if (apdu->u.initRequest->implementationName)
2463             yaz_log(YLOG_LOG, "%simplementationName: %s",
2464                     m_session_str, apdu->u.initRequest->implementationName);
2465         if (apdu->u.initRequest->implementationVersion)
2466             yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2467                     m_session_str, apdu->u.initRequest->implementationVersion);
2468         if (m_initRequest_apdu == 0)
2469         {
2470             if (m_initRequest_mem)
2471                 nmem_destroy(m_initRequest_mem);
2472
2473             m_initRequest_apdu = apdu;
2474             m_initRequest_mem = odr_extract_mem(odr_decode());
2475
2476             m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2477                 preferredMessageSize;
2478             *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2479             m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2480                 maximumRecordSize;
2481             *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2482
2483             // Save proposal charsets and langs.
2484             if (ODR_MASK_GET(apdu->u.initRequest->options,
2485                 Z_Options_negotiationModel))
2486             {
2487                 Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2488                     yaz_get_charneg_record(*oi);
2489
2490                 yaz_get_proposal_charneg(m_referenceId_mem,
2491                     charSetandLangRecord,
2492                     &m_initRequest_oi_negotiation_charsets,
2493                     &m_initRequest_oi_negotiation_num_charsets,
2494                     &m_initRequest_oi_negotiation_langs,
2495                     &m_initRequest_oi_negotiation_num_langs,
2496                     &m_initRequest_oi_negotiation_selected);
2497
2498                 for (int i=0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2499                 {
2500                     yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2501                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2502                         m_initRequest_oi_negotiation_charsets[i]:"none");
2503                 }
2504                 for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2505                 {
2506                     yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2507                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2508                         m_initRequest_oi_negotiation_langs[i]:"none");
2509                 }
2510                 yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2511                     m_session_str, m_initRequest_oi_negotiation_selected);
2512             }
2513             // save init options for the response..
2514             m_initRequest_options = apdu->u.initRequest->options;
2515
2516             apdu->u.initRequest->options =
2517                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2518                                            sizeof(Odr_bitmask));
2519             ODR_MASK_ZERO(apdu->u.initRequest->options);
2520             int i;
2521             for (i = 0; i<= 24; i++)
2522                 ODR_MASK_SET(apdu->u.initRequest->options, i);
2523             // check negotiation option
2524             if (!ODR_MASK_GET(m_initRequest_options,
2525                 Z_Options_negotiationModel))
2526             {
2527                 ODR_MASK_CLEAR(apdu->u.initRequest->options,
2528                     Z_Options_negotiationModel);
2529             }
2530             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2531                 Z_Options_concurrentOperations);
2532             // make new version
2533             m_initRequest_version = apdu->u.initRequest->protocolVersion;
2534             apdu->u.initRequest->protocolVersion =
2535                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2536                                            sizeof(Odr_bitmask));
2537             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2538
2539             for (i = 0; i<= 8; i++)
2540                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2541         }
2542         if (m_client->m_init_flag)
2543         {
2544             if (handle_init_response_for_invalid_session(apdu))
2545                 return;
2546             if (m_client->m_initResponse)
2547             {
2548                 Z_APDU *apdu2 = m_client->m_initResponse;
2549                 apdu2->u.initResponse->otherInfo = 0;
2550                 if (m_client->m_cookie && *m_client->m_cookie)
2551                     set_otherInformationString(apdu2, VAL_COOKIE, 1,
2552                                                m_client->m_cookie);
2553                 apdu2->u.initResponse->referenceId =
2554                     apdu->u.initRequest->referenceId;
2555                 apdu2->u.initResponse->options = m_client->m_initResponse_options;
2556                 apdu2->u.initResponse->protocolVersion =
2557                     m_client->m_initResponse_version;
2558
2559                 send_to_client(apdu2);
2560                 return;
2561             }
2562         }
2563         m_client->m_init_flag = 1;
2564     }
2565     handle_max_record_retrieve(apdu);
2566
2567     if (apdu)
2568         apdu = handle_syntax_validation(apdu);
2569
2570     if (apdu)
2571         apdu = handle_query_transformation(apdu);
2572
2573     if (apdu)
2574         apdu = handle_query_validation(apdu);
2575
2576     if (apdu)
2577         apdu = result_set_optimize(apdu);
2578     if (!apdu)
2579     {
2580         m_client->timeout(m_target_idletime);  // mark it active even
2581         // though we didn't use it
2582         return;
2583     }
2584     // Add otherInformation entry in APDU if
2585     // negotiatoin in use.
2586     if (apdu)
2587         handle_charset_lang_negotiation(apdu);
2588
2589     // delete other info construct completely if 0 elements
2590     get_otherInfoAPDU(apdu, &oi);
2591     if (oi && *oi && (*oi)->num_elements == 0)
2592         *oi = 0;
2593
2594     if (apdu->which == Z_APDU_presentRequest &&
2595         m_client->m_resultSetStartPoint == 0)
2596     {
2597         Z_PresentRequest *pr = apdu->u.presentRequest;
2598         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2599         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2600     } else {
2601         m_client->m_resultSetStartPoint = 0;
2602     }
2603     if (m_client->send_to_target(apdu) < 0)
2604     {
2605         delete m_client;
2606         m_client = 0;
2607         delete this;
2608     }
2609     else
2610         m_client->m_waiting = 1;
2611 }
2612
2613 void Yaz_Proxy::connectNotify()
2614 {
2615 }
2616
2617 void Yaz_Proxy::releaseClient()
2618 {
2619     xfree(m_proxyTarget);
2620     m_proxyTarget = 0;
2621     m_invalid_session = 0;
2622     // only keep if keep_alive flag is set...
2623     if (m_client &&
2624         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2625         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2626         m_client->m_waiting == 0)
2627     {
2628         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2629                  m_session_str,
2630                  m_client->get_hostname());
2631         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2632                 m_session_str, m_client->m_pdu_recv,
2633                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2634                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2635         assert (m_client->m_waiting != 2);
2636         // Tell client (if any) that no server connection is there..
2637         m_client->m_server = 0;
2638         m_client = 0;
2639     }
2640     else if (m_client)
2641     {
2642         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
2643                  m_session_str,
2644                  m_client->get_hostname());
2645         assert (m_client->m_waiting != 2);
2646         delete m_client;
2647         m_client = 0;
2648     }
2649     else if (!m_parent)
2650     {
2651         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
2652                  m_session_str);
2653         assert (m_parent);
2654     }
2655     else
2656     {
2657         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
2658                  m_session_str);
2659     }
2660     if (m_parent)
2661         m_parent->pre_init();
2662 }
2663
2664 void Yaz_Proxy::shutdown()
2665 {
2666     releaseClient();
2667     delete this;
2668 }
2669
2670 const char *Yaz_ProxyClient::get_session_str()
2671 {
2672     if (!m_server)
2673         return "0 ";
2674     return m_server->get_session_str();
2675 }
2676
2677 void Yaz_ProxyClient::shutdown()
2678 {
2679     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2680              get_hostname());
2681     delete m_server;
2682     delete this;
2683 }
2684
2685 void Yaz_Proxy::failNotify()
2686 {
2687     inc_request_no();
2688     yaz_log (YLOG_LOG, "%sConnection closed by client",
2689              get_session_str());
2690     shutdown();
2691 }
2692
2693 void Yaz_ProxyClient::failNotify()
2694 {
2695     if (m_server)
2696         m_server->inc_request_no();
2697     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
2698              get_session_str(), get_hostname());
2699     shutdown();
2700 }
2701
2702 void Yaz_ProxyClient::connectNotify()
2703 {
2704     const char *s = get_session_str();
2705     const char *h = get_hostname();
2706     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2707              m_target_idletime);
2708     timeout(m_target_idletime);
2709     if (!m_server)
2710         pre_init_client();
2711 }
2712
2713 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2714                                                   *the_PDU_Observable, int fd)
2715 {
2716     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2717 }
2718
2719 Yaz_ProxyClient::~Yaz_ProxyClient()
2720 {
2721     if (m_prev)
2722         *m_prev = m_next;
2723     if (m_next)
2724         m_next->m_prev = m_prev;
2725     m_waiting = 2;     // for debugging purposes only.
2726     odr_destroy(m_init_odr);
2727     delete m_last_query;
2728     xfree (m_last_resultSetId);
2729     xfree (m_cookie);
2730 }
2731
2732 void Yaz_ProxyClient::pre_init_client()
2733 {
2734     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2735     Z_InitRequest *req = apdu->u.initRequest;
2736
2737     int i;
2738     for (i = 0; i<= 24; i++)
2739         ODR_MASK_SET(req->options, i);
2740     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2741                    Z_Options_negotiationModel);
2742     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2743                    Z_Options_concurrentOperations);
2744     for (i = 0; i<= 10; i++)
2745         ODR_MASK_SET(req->protocolVersion, i);
2746
2747     if (send_to_target(apdu) < 0)
2748     {
2749         delete this;
2750     }
2751     else
2752     {
2753         m_waiting = 1;
2754         m_init_flag = 1;
2755     }
2756 }
2757
2758 void Yaz_Proxy::pre_init()
2759 {
2760     int i;
2761     const char *name = 0;
2762     const char *zurl_in_use[MAX_ZURL_PLEX];
2763     int limit_bw, limit_pdu, limit_req;
2764     int target_idletime, client_idletime;
2765     int max_clients;
2766     int keepalive_limit_bw, keepalive_limit_pdu;
2767     int pre_init;
2768     const char *cql2rpn = 0;
2769     const char *authentication = 0;
2770     const char *negotiation_charset = 0;
2771     const char *negotiation_lang = 0;
2772
2773     Yaz_ProxyConfig *cfg = check_reconfigure();
2774
2775     zurl_in_use[0] = 0;
2776
2777     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2778         set_APDU_yazlog(1);
2779     else
2780         set_APDU_yazlog(0);
2781
2782     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2783                                           &limit_bw, &limit_pdu, &limit_req,
2784                                           &target_idletime, &client_idletime,
2785                                           &max_clients,
2786                                           &keepalive_limit_bw,
2787                                           &keepalive_limit_pdu,
2788                                           &pre_init,
2789                                           &cql2rpn,
2790                                           &authentication,
2791                                           &negotiation_charset,
2792                                           &negotiation_lang) ; i++)
2793     {
2794         if (pre_init)
2795         {
2796             int j;
2797             for (j = 0; zurl_in_use[j]; j++)
2798             {
2799                 Yaz_ProxyClient *c;
2800                 int spare = 0;
2801                 int spare_waiting = 0;
2802                 int in_use = 0;
2803                 int other = 0;
2804                 for (c = m_clientPool; c; c = c->m_next)
2805                 {
2806                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
2807                     {
2808                         if (c->m_cookie == 0)
2809                         {
2810                             if (c->m_server == 0)
2811                                 if (c->m_waiting)
2812                                     spare_waiting++;
2813                                 else
2814                                     spare++;
2815                             else
2816                                 in_use++;
2817                         }
2818                         else
2819                             other++;
2820                     }
2821                 }
2822                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2823                         "sparew=%d preinit=%d",m_session_str,
2824                         name, zurl_in_use[j], in_use, other,
2825                         spare, spare_waiting, pre_init);
2826                 if (spare + spare_waiting < pre_init)
2827                 {
2828                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2829                     c->m_next = m_clientPool;
2830                     if (c->m_next)
2831                         c->m_next->m_prev = &c->m_next;
2832                     m_clientPool = c;
2833                     c->m_prev = &m_clientPool;
2834
2835                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
2836                         c->set_APDU_yazlog(1);
2837                     else
2838                         c->set_APDU_yazlog(0);
2839
2840                     if (c->client(zurl_in_use[j]))
2841                     {
2842                         timeout(60);
2843                         delete c;
2844                         return;
2845                     }
2846                     c->timeout(30);
2847                     c->m_waiting = 1;
2848                     c->m_target_idletime = target_idletime;
2849                     c->m_seqno = m_seqno++;
2850                 }
2851             }
2852         }
2853     }
2854 }
2855
2856 void Yaz_Proxy::timeoutNotify()
2857 {
2858     if (m_parent)
2859     {
2860         if (m_bw_hold_PDU)
2861         {
2862             timeout(m_client_idletime);
2863             Z_GDU *apdu = m_bw_hold_PDU;
2864             m_bw_hold_PDU = 0;
2865
2866             if (apdu->which == Z_GDU_Z3950)
2867                 handle_incoming_Z_PDU(apdu->u.z3950);
2868             else if (apdu->which == Z_GDU_HTTP_Request)
2869                 handle_incoming_HTTP(apdu->u.HTTP_Request);
2870         }
2871         else if (m_stylesheet_nprl)
2872             convert_xsl_delay();
2873         else
2874         {
2875             inc_request_no();
2876
2877             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2878             shutdown();
2879         }
2880     }
2881     else
2882     {
2883         timeout(600);
2884         pre_init();
2885     }
2886 }
2887
2888 void Yaz_Proxy::markInvalid()
2889 {
2890     m_client = 0;
2891     m_invalid_session = 1;
2892 }
2893
2894 void Yaz_ProxyClient::timeoutNotify()
2895 {
2896     if (m_server)
2897         m_server->inc_request_no();
2898
2899     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2900              get_hostname());
2901     m_waiting = 1;
2902     m_root->pre_init();
2903     if (m_server && m_init_flag)
2904     {
2905         // target timed out in a session that was properly initialized
2906         // server object stay alive but we mark it as invalid so it
2907         // gets initialized again
2908         m_server->markInvalid();
2909         m_server = 0;
2910     }
2911     shutdown();
2912 }
2913
2914 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2915                                  Yaz_Proxy *parent) :
2916     Yaz_Z_Assoc (the_PDU_Observable)
2917 {
2918     m_cookie = 0;
2919     m_next = 0;
2920     m_prev = 0;
2921     m_init_flag = 0;
2922     m_last_query = 0;
2923     m_last_resultSetId = 0;
2924     m_last_resultCount = 0;
2925     m_last_ok = 0;
2926     m_sr_transform = 0;
2927     m_waiting = 0;
2928     m_init_odr = odr_createmem (ODR_DECODE);
2929     m_initResponse = 0;
2930     m_initResponse_options = 0;
2931     m_initResponse_version = 0;
2932     m_initResponse_preferredMessageSize = 0;
2933     m_initResponse_maximumRecordSize = 0;
2934     m_resultSetStartPoint = 0;
2935     m_bytes_sent = m_bytes_recv = 0;
2936     m_pdu_recv = 0;
2937     m_server = 0;
2938     m_seqno = 0;
2939     m_target_idletime = 600;
2940     m_root = parent;
2941 }
2942
2943 const char *Yaz_Proxy::option(const char *name, const char *value)
2944 {
2945     if (!strcmp (name, "optimize")) {
2946         if (value) {
2947             xfree (m_optimize);
2948             m_optimize = xstrdup (value);
2949         }
2950         return m_optimize;
2951     }
2952     return 0;
2953 }
2954
2955 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2956 {
2957
2958 }
2959
2960 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2961 {
2962     if (apdu->which == Z_GDU_Z3950)
2963         recv_Z_PDU(apdu->u.z3950, len);
2964     else if (apdu->which == Z_GDU_HTTP_Response)
2965         recv_HTTP_response(apdu->u.HTTP_Response, len);
2966     else
2967         shutdown();
2968 }
2969
2970 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2971 {
2972     if (!m_invalid_session)
2973         return 0;
2974     m_invalid_session = 0;
2975     handle_incoming_Z_PDU(m_apdu_invalid_session);
2976     assert (m_mem_invalid_session);
2977     nmem_destroy(m_mem_invalid_session);
2978     m_mem_invalid_session = 0;
2979     return 1;
2980 }
2981
2982 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2983 {
2984     m_bytes_recv += len;
2985
2986     m_pdu_recv++;
2987     m_waiting = 0;
2988     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2989         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2990                  apdu_name(apdu), get_hostname(), len);
2991     if (apdu->which == Z_APDU_initResponse)
2992     {
2993         if (!m_server)  // if this is a pre init session , check for more
2994             m_root->pre_init();
2995         NMEM nmem = odr_extract_mem (odr_decode());
2996         odr_reset (m_init_odr);
2997         nmem_transfer (m_init_odr->mem, nmem);
2998         m_initResponse = apdu;
2999         m_initResponse_options = apdu->u.initResponse->options;
3000         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3001         m_initResponse_preferredMessageSize =
3002             *apdu->u.initResponse->preferredMessageSize;
3003         m_initResponse_maximumRecordSize =
3004             *apdu->u.initResponse->maximumRecordSize;
3005
3006         Z_InitResponse *ir = apdu->u.initResponse;
3007         char *im0 = ir->implementationName;
3008
3009         char *im1 = (char*)
3010             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3011         *im1 = '\0';
3012         if (im0)
3013         {
3014             strcat(im1, im0);
3015             strcat(im1, " ");
3016         }
3017         strcat(im1, "(YAZ Proxy)");
3018         ir->implementationName = im1;
3019
3020         nmem_destroy (nmem);
3021
3022         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3023             return;
3024     }
3025     if (apdu->which == Z_APDU_searchResponse)
3026     {
3027         Z_SearchResponse *sr = apdu->u.searchResponse;
3028         m_last_resultCount = *sr->resultCount;
3029         int status = *sr->searchStatus;
3030         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3031         {
3032             m_last_ok = 1;
3033
3034             if (sr->records && sr->records->which == Z_Records_DBOSD)
3035             {
3036                 m_cache.add(odr_decode(),
3037                             sr->records->u.databaseOrSurDiagnostics, 1,
3038                             *sr->resultCount);
3039             }
3040         }
3041     }
3042     if (apdu->which == Z_APDU_presentResponse)
3043     {
3044         Z_PresentResponse *pr = apdu->u.presentResponse;
3045         if (m_sr_transform)
3046         {
3047             m_sr_transform = 0;
3048             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3049             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3050             sr->referenceId = pr->referenceId;
3051             *sr->resultCount = m_last_resultCount;
3052             sr->records = pr->records;
3053             sr->nextResultSetPosition = pr->nextResultSetPosition;
3054             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3055             apdu = new_apdu;
3056         }
3057         if (pr->records &&
3058             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3059         {
3060             m_cache.add(odr_decode(),
3061                         pr->records->u.databaseOrSurDiagnostics,
3062                         m_resultSetStartPoint, -1);
3063             m_resultSetStartPoint = 0;
3064         }
3065     }
3066     if (m_cookie)
3067         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3068     if (m_server)
3069     {
3070         m_server->send_to_client(apdu);
3071     }
3072     if (apdu->which == Z_APDU_close)
3073     {
3074         shutdown();
3075     }
3076 }
3077
3078 void Yaz_Proxy::low_socket_close()
3079 {
3080 #if WIN32
3081 #else
3082     int i;
3083     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3084         if  (m_lo_fd[i] >= 0)
3085             ::close(m_lo_fd[i]);
3086 #endif
3087 }
3088
3089 void Yaz_Proxy::low_socket_open()
3090 {
3091 #if WIN32
3092 #else
3093     int i;
3094     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3095         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3096 #endif
3097 }
3098
3099 int Yaz_Proxy::server(const char *addr)
3100 {
3101     int r = Yaz_Z_Assoc::server(addr);
3102     if (!r)
3103     {
3104         yaz_log(YLOG_LOG, "%sStarted proxy "
3105 #ifdef VERSION
3106             VERSION
3107 #endif
3108             " on %s", m_session_str, addr);
3109         timeout(1);
3110     }
3111     return r;
3112 }
3113