src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.20 2005-02-10 08:09:42 oleg Exp $
   2    Copyright (c) 1998-2005, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <time.h>
  42 #include <fcntl.h>
  43
  44 #include <yaz/srw.h>
  45 #include <yaz/marcdisp.h>
  46 #include <yaz/yaz-iconv.h>
  47 #include <yaz/log.h>
  48 #include <yaz/diagbib1.h>
  49 #include "proxyp.h"
  50 #include <yaz/pquery.h>
  51 #include <yaz/otherinfo.h>
  52 #include <yaz/charneg.h>
  53
  54
  55 static const char *apdu_name(Z_APDU *apdu)
  56 {
  57     switch (apdu->which)
  58     {
  59     case Z_APDU_initRequest:
  60         return "initRequest";
  61     case Z_APDU_initResponse:
  62         return "initResponse";
  63     case Z_APDU_searchRequest:
  64         return "searchRequest";
  65     case Z_APDU_searchResponse:
  66         return "searchResponse";
  67     case Z_APDU_presentRequest:
  68         return "presentRequest";
  69     case Z_APDU_presentResponse:
  70         return "presentResponse";
  71     case Z_APDU_deleteResultSetRequest:
  72         return "deleteResultSetRequest";
  73     case Z_APDU_deleteResultSetResponse:
  74         return "deleteResultSetResponse";
  75     case Z_APDU_scanRequest:
  76         return "scanRequest";
  77     case Z_APDU_scanResponse:
  78         return "scanResponse";
  79     case Z_APDU_sortRequest:
  80         return "sortRequest";
  81     case Z_APDU_sortResponse:
  82         return "sortResponse";
  83     case Z_APDU_extendedServicesRequest:
  84         return "extendedServicesRequest";
  85     case Z_APDU_extendedServicesResponse:
  86         return "extendedServicesResponse";
  87     case Z_APDU_close:
  88         return "close";
  89     }
  90     return "other";
  91 }
  92
  93 static const char *gdu_name(Z_GDU *gdu)
  94 {
  95     switch(gdu->which)
  96     {
  97     case Z_GDU_Z3950:
  98         return apdu_name(gdu->u.z3950);
  99     case Z_GDU_HTTP_Request:
 100         return "HTTP Request";
 101     case Z_GDU_HTTP_Response:
 102         return "HTTP Response";
 103     }
 104     return "Unknown request/response";
 105 }
 106 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
 107                      Yaz_Proxy *parent) :
 108     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 109 {
 110     m_PDU_Observable = the_PDU_Observable;
 111     m_client = 0;
 112     m_parent = parent;
 113     m_clientPool = 0;
 114     m_seqno = 1;
 115     m_keepalive_limit_bw = 500000;
 116     m_keepalive_limit_pdu = 1000;
 117     m_proxyTarget = 0;
 118     m_default_target = 0;
 119     m_proxy_authentication = 0;
 120     m_proxy_negotiation_charset = 0;
 121     m_proxy_negotiation_lang = 0;
 122     m_max_clients = 150;
 123     m_log_mask = 0;
 124     m_seed = time(0);
 125     m_client_idletime = 600;
 126     m_target_idletime = 600;
 127     m_optimize = xstrdup ("1");
 128     strcpy(m_session_str, "0 ");
 129     m_session_no=0;
 130     m_bytes_sent = 0;
 131     m_bytes_recv = 0;
 132     m_bw_hold_PDU = 0;
 133     m_bw_max = 0;
 134     m_pdu_max = 0;
 135     m_max_record_retrieve = 0;
 136     m_reconfig_flag = 0;
 137     m_config_fname = 0;
 138     m_request_no = 0;
 139     m_invalid_session = 0;
 140     m_referenceId = 0;
 141     m_referenceId_mem = nmem_create();
 142     m_config = 0;
 143     m_marcxml_flag = 0;
 144     m_stylesheet_xsp = 0;
 145     m_stylesheet_nprl = 0;
 146     m_s2z_stylesheet = 0;
 147     m_s2z_database = 0;
 148     m_schema = 0;
 149     m_backend_type = 0;
 150     m_backend_charset = 0;
 151     m_frontend_type = 0;
 152     m_initRequest_apdu = 0;
 153     m_initRequest_mem = 0;
 154     m_initRequest_preferredMessageSize = 0;
 155     m_initRequest_maximumRecordSize = 0;
 156     m_initRequest_options = 0;
 157     m_initRequest_version = 0;
 158     m_initRequest_oi_negotiation_charsets = 0;
 159     m_initRequest_oi_negotiation_num_charsets = 0;
 160     m_initRequest_oi_negotiation_langs = 0;
 161     m_initRequest_oi_negotiation_num_langs = 0;
 162     m_initRequest_oi_negotiation_selected = 0;
 163     m_apdu_invalid_session = 0;
 164     m_mem_invalid_session = 0;
 165     m_s2z_odr_init = 0;
 166     m_s2z_odr_search = 0;
 167     m_s2z_init_apdu = 0;
 168     m_s2z_search_apdu = 0;
 169     m_s2z_present_apdu = 0;
 170     m_http_keepalive = 0;
 171     m_http_version = 0;
 172     m_soap_ns = 0;
 173     m_s2z_packing = Z_SRW_recordPacking_string;
 174 #if HAVE_GETTIMEOFDAY
 175     m_time_tv = xmalloc(sizeof(struct timeval));
 176     struct timeval *tv = (struct timeval *) m_time_tv;
 177     tv->tv_sec = 0;
 178     tv->tv_usec = 0;
 179 #else
 180     m_time_tv = 0;
 181 #endif
 182     m_usemarcon_ini_stage1 = 0;
 183     m_usemarcon_ini_stage2 = 0;
 184     m_usemarcon = new Yaz_usemarcon();
 185     if (!m_parent)
 186         low_socket_open();
 187 }
 188
 189 Yaz_Proxy::~Yaz_Proxy()
 190 {
 191     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 192             m_bytes_sent, m_bytes_recv);
 193     nmem_destroy(m_initRequest_mem);
 194     nmem_destroy(m_mem_invalid_session);
 195     nmem_destroy(m_referenceId_mem);
 196
 197     xfree(m_proxyTarget);
 198     xfree(m_default_target);
 199     xfree(m_proxy_authentication);
 200     xfree(m_proxy_negotiation_charset);
 201     xfree(m_proxy_negotiation_lang);
 202     xfree(m_optimize);
 203
 204 #if HAVE_XSLT
 205     if (m_stylesheet_xsp)
 206         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 207 #endif
 208     xfree (m_time_tv);
 209
 210     xfree (m_schema);
 211     xfree (m_backend_type);
 212     xfree (m_backend_charset);
 213     xfree (m_usemarcon_ini_stage1);
 214     xfree (m_usemarcon_ini_stage2);
 215     delete m_usemarcon;
 216     if (m_s2z_odr_init)
 217         odr_destroy(m_s2z_odr_init);
 218     if (m_s2z_odr_search)
 219         odr_destroy(m_s2z_odr_search);
 220     if (!m_parent)
 221         low_socket_close();
 222     delete m_config;
 223 }
 224
 225 int Yaz_Proxy::set_config(const char *config)
 226 {
 227     delete m_config;
 228     m_config = new Yaz_ProxyConfig();
 229     xfree(m_config_fname);
 230     m_config_fname = xstrdup(config);
 231     int r = m_config->read_xml(config);
 232     if (!r)
 233         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 234     return r;
 235 }
 236
 237 void Yaz_Proxy::set_default_target(const char *target)
 238 {
 239     xfree (m_default_target);
 240     m_default_target = 0;
 241     if (target)
 242         m_default_target = (char *) xstrdup (target);
 243 }
 244
 245 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 246 {
 247     xfree (m_proxy_authentication);
 248     m_proxy_authentication = 0;
 249     if (auth)
 250         m_proxy_authentication = (char *) xstrdup (auth);
 251 }
 252 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang)
 253 {
 254     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s',
 255         language to '%s'", m_session_str, (charset)?charset:"none",
 256         (lang)?lang:"none");
 257     xfree (m_proxy_negotiation_charset);
 258     xfree (m_proxy_negotiation_lang);
 259     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 260     if (charset)
 261         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 262     if (lang)
 263         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 264 }
 265 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 266 {
 267     if (m_parent)
 268         return m_parent->check_reconfigure();
 269
 270     Yaz_ProxyConfig *cfg = m_config;
 271     if (m_reconfig_flag)
 272     {
 273         yaz_log(YLOG_LOG, "reconfigure");
 274         yaz_log_reopen();
 275         if (m_config_fname && cfg)
 276         {
 277             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 278             int r = cfg->read_xml(m_config_fname);
 279             if (r)
 280                 yaz_log(YLOG_WARN, "reconfigure failed");
 281             else
 282             {
 283                 m_log_mask = 0;
 284                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 285             }
 286         }
 287         else
 288             yaz_log(YLOG_LOG, "reconfigure");
 289         m_reconfig_flag = 0;
 290     }
 291     return cfg;
 292 }
 293
 294 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 295                                             *the_PDU_Observable, int fd)
 296 {
 297     check_reconfigure();
 298     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 299     new_proxy->m_config = 0;
 300     new_proxy->m_config_fname = 0;
 301     new_proxy->timeout(m_client_idletime);
 302     new_proxy->m_target_idletime = m_target_idletime;
 303     new_proxy->set_default_target(m_default_target);
 304     new_proxy->m_max_clients = m_max_clients;
 305     new_proxy->m_log_mask = m_log_mask;
 306     new_proxy->set_APDU_log(get_APDU_log());
 307     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 308         new_proxy->set_APDU_yazlog(1);
 309     else
 310         new_proxy->set_APDU_yazlog(0);
 311     new_proxy->set_proxy_authentication(m_proxy_authentication);
 312     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 313         m_proxy_negotiation_lang);
 314     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 315     m_session_no++;
 316     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 317              the_PDU_Observable->getpeername());
 318     return new_proxy;
 319 }
 320
 321 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 322 {
 323     int oid[OID_SIZE];
 324     Z_OtherInformationUnit *oi;
 325     struct oident ent;
 326     ent.proto = PROTO_Z3950;
 327     ent.oclass = CLASS_USERINFO;
 328     ent.value = (oid_value) VAL_COOKIE;
 329     assert (oid_ent_to_oid (&ent, oid));
 330
 331     if (oid_ent_to_oid (&ent, oid) &&
 332         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 333         oi->which == Z_OtherInfo_characterInfo)
 334         return oi->information.characterInfo;
 335     return 0;
 336 }
 337 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 338 {
 339     int oid[OID_SIZE];
 340     Z_OtherInformationUnit *oi;
 341     struct oident ent;
 342     ent.proto = PROTO_Z3950;
 343     ent.oclass = CLASS_USERINFO;
 344     ent.value = (oid_value) VAL_PROXY;
 345     if (oid_ent_to_oid (&ent, oid) &&
 346         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 347         oi->which == Z_OtherInfo_characterInfo)
 348         return oi->information.characterInfo;
 349     return 0;
 350 }
 351 const char *Yaz_Proxy::load_balance(const char **url)
 352 {
 353     int zurl_in_use[MAX_ZURL_PLEX];
 354     int zurl_in_spare[MAX_ZURL_PLEX];
 355     Yaz_ProxyClient *c;
 356     int i;
 357
 358     for (i = 0; i<MAX_ZURL_PLEX; i++)
 359     {
 360         zurl_in_use[i] = 0;
 361         zurl_in_spare[i] = 0;
 362     }
 363     for (c = m_parent->m_clientPool; c; c = c->m_next)
 364     {
 365         for (i = 0; url[i]; i++)
 366             if (!strcmp(url[i], c->get_hostname()))
 367             {
 368                 zurl_in_use[i]++;
 369                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 370                     zurl_in_spare[i]++;
 371             }
 372     }
 373     int min_use = 100000;
 374     int spare_for_min = 0;
 375     int max_spare = 0;
 376     const char *ret_min = 0;
 377     const char *ret_spare = 0;
 378     for (i = 0; url[i]; i++)
 379     {
 380         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 381                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 382         if (min_use > zurl_in_use[i])
 383         {
 384             ret_min = url[i];
 385             min_use = zurl_in_use[i];
 386             spare_for_min = zurl_in_spare[i];
 387         }
 388         if (max_spare < zurl_in_spare[i])
 389         {
 390             ret_spare = url[i];
 391             max_spare = zurl_in_spare[i];
 392         }
 393     }
 394     // use the one with minimum connections if spare is > 3
 395     if (spare_for_min > 3)
 396         return ret_min;
 397     // use one with most spares (if any)
 398     if (max_spare > 0)
 399         return ret_spare;
 400     return ret_min;
 401 }
 402
 403 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 404                                        const char *proxy_host)
 405 {
 406     assert (m_parent);
 407     Yaz_Proxy *parent = m_parent;
 408     Yaz_ProxyClient *c = m_client;
 409
 410     if (!m_proxyTarget)
 411     {
 412         const char *url[MAX_ZURL_PLEX];
 413         Yaz_ProxyConfig *cfg = check_reconfigure();
 414         if (proxy_host)
 415         {
 416 #if 0
 417 /* only to be enabled for debugging... */
 418             if (!strcmp(proxy_host, "stop"))
 419                 exit(0);
 420 #endif
 421             xfree(m_default_target);
 422             m_default_target = xstrdup(proxy_host);
 423         }
 424         proxy_host = m_default_target;
 425         int client_idletime = -1;
 426         const char *cql2rpn_fname = 0;
 427         const char *authentication = 0;
 428         const char *negotiation_charset = 0;
 429         const char *negotiation_lang = 0;
 430         url[0] = m_default_target;
 431         url[1] = 0;
 432         if (cfg)
 433         {
 434             int pre_init = 0;
 435             cfg->get_target_info(proxy_host, url, &m_bw_max,
 436                                  &m_pdu_max, &m_max_record_retrieve,
 437                                  &m_target_idletime, &client_idletime,
 438                                  &parent->m_max_clients,
 439                                  &m_keepalive_limit_bw,
 440                                  &m_keepalive_limit_pdu,
 441                                  &pre_init,
 442                                  &cql2rpn_fname,
 443                                  &authentication,
 444                                  &negotiation_charset,
 445                                  &negotiation_lang);
 446         }
 447         if (client_idletime != -1)
 448         {
 449             m_client_idletime = client_idletime;
 450             timeout(m_client_idletime);
 451         }
 452         if (cql2rpn_fname)
 453             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 454         if (authentication)
 455             set_proxy_authentication(authentication);
 456         if (negotiation_charset || negotiation_lang)
 457             set_proxy_negotiation(negotiation_charset,
 458                 negotiation_lang);
 459         if (!url[0])
 460         {
 461             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 462             return 0;
 463         }
 464         // we don't handle multiplexing for cookie session, so we just
 465         // pick the first one in this case (anonymous users will be able
 466         // to use any backend)
 467         if (cookie && *cookie)
 468             m_proxyTarget = (char*) xstrdup(url[0]);
 469         else
 470             m_proxyTarget = (char*) xstrdup(load_balance(url));
 471     }
 472     if (cookie && *cookie)
 473     {   // search in sessions with a cookie
 474         for (c = parent->m_clientPool; c; c = c->m_next)
 475         {
 476             assert (c->m_prev);
 477             assert (*c->m_prev == c);
 478             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 479                 !strcmp(m_proxyTarget, c->get_hostname()))
 480             {
 481                 // Found it in cache
 482                 // The following handles "cancel"
 483                 // If connection is busy (waiting for PDU) and
 484                 // we have an initRequest we can safely do re-open
 485                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 486                 {
 487                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 488                              c->get_hostname());
 489                     c->close();
 490                     c->m_init_flag = 0;
 491
 492                     c->m_last_ok = 0;
 493                     c->m_cache.clear();
 494                     c->m_last_resultCount = 0;
 495                     c->m_sr_transform = 0;
 496                     c->m_waiting = 0;
 497                     c->m_resultSetStartPoint = 0;
 498                     c->m_target_idletime = m_target_idletime;
 499                     if (c->client(m_proxyTarget))
 500                     {
 501                         delete c;
 502                         return 0;
 503                     }
 504                     c->timeout(30);
 505                 }
 506                 c->m_seqno = parent->m_seqno;
 507                 if (c->m_server && c->m_server != this)
 508                     c->m_server->m_client = 0;
 509                 c->m_server = this;
 510                 (parent->m_seqno)++;
 511                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 512                 return c;
 513             }
 514         }
 515     }
 516     else if (!c &&
 517             apdu->which == Z_APDU_initRequest &&
 518             apdu->u.initRequest->idAuthentication == 0 &&
 519             !ODR_MASK_GET(apdu->u.initRequest->options, Z_Options_negotiationModel))
 520     {
 521         // anonymous sessions without cookie.
 522         // if authentication is set it is NOT anonymous se we can't share them.
 523         // If charset and lang negotiation is use it is NOT anonymous session too.
 524         for (c = parent->m_clientPool; c; c = c->m_next)
 525         {
 526             assert(c->m_prev);
 527             assert(*c->m_prev == c);
 528             if (c->m_server == 0 && c->m_cookie == 0 &&
 529                 c->m_waiting == 0 &&
 530                 !strcmp(m_proxyTarget, c->get_hostname()))
 531             {
 532                 // found it in cache
 533                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 534                          m_session_str, parent->m_seqno, c->get_hostname());
 535
 536                 c->m_seqno = parent->m_seqno;
 537                 assert(c->m_server == 0);
 538                 c->m_server = this;
 539
 540                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 541                     c->set_APDU_yazlog(1);
 542                 else
 543                     c->set_APDU_yazlog(0);
 544
 545                 (parent->m_seqno)++;
 546
 547                 parent->pre_init();
 548
 549                 return c;
 550             }
 551         }
 552     }
 553     if (!m_client)
 554     {
 555         if (apdu->which != Z_APDU_initRequest)
 556         {
 557             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 558             return 0;
 559         }
 560         Z_InitRequest *initRequest = apdu->u.initRequest;
 561
 562         if (!initRequest->idAuthentication)
 563         {
 564             if (m_proxy_authentication)
 565             {
 566                 initRequest->idAuthentication =
 567                     (Z_IdAuthentication *)
 568                     odr_malloc (odr_encode(),
 569                                 sizeof(*initRequest->idAuthentication));
 570                 initRequest->idAuthentication->which =
 571                     Z_IdAuthentication_open;
 572                 initRequest->idAuthentication->u.open =
 573                     odr_strdup (odr_encode(), m_proxy_authentication);
 574             }
 575         }
 576         else
 577         {
 578             // the client use authentication. We set the keepalive PDU
 579             // to 0 so we don't cache it in releaseClient
 580             m_keepalive_limit_pdu = 0;
 581         }
 582         // go through list of clients - and find the lowest/oldest one.
 583         Yaz_ProxyClient *c_min = 0;
 584         int min_seq = -1;
 585         int no_of_clients = 0;
 586         if (parent->m_clientPool)
 587             yaz_log (YLOG_DEBUG, "Existing sessions");
 588         for (c = parent->m_clientPool; c; c = c->m_next)
 589         {
 590             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 591                                c->m_waiting, c->get_hostname(),
 592                                c->m_cookie ? c->m_cookie : "");
 593             no_of_clients++;
 594             if (min_seq < 0 || c->m_seqno < min_seq)
 595             {
 596                 min_seq = c->m_seqno;
 597                 c_min = c;
 598             }
 599         }
 600         if (no_of_clients >= parent->m_max_clients)
 601         {
 602             c = c_min;
 603             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 604             {
 605                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 606                          m_session_str, parent->m_max_clients, c->m_seqno);
 607                 if (c->m_server && c->m_server != this)
 608                     delete c->m_server;
 609                 c->m_server = 0;
 610             }
 611             else
 612             {
 613                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 614                          m_session_str, parent->m_max_clients,
 615                          c->m_seqno, parent->m_seqno, c->get_hostname());
 616                 xfree (c->m_cookie);
 617                 c->m_cookie = 0;
 618                 if (cookie)
 619                     c->m_cookie = xstrdup(cookie);
 620                 c->m_seqno = parent->m_seqno;
 621                 if (c->m_server && c->m_server != this)
 622                 {
 623                     c->m_server->m_client = 0;
 624                     delete c->m_server;
 625                 }
 626                 (parent->m_seqno)++;
 627                 c->m_target_idletime = m_target_idletime;
 628                 c->timeout(m_target_idletime);
 629
 630                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 631                     c->set_APDU_yazlog(1);
 632                 else
 633                     c->set_APDU_yazlog(0);
 634
 635                 return c;
 636             }
 637         }
 638         else
 639         {
 640             yaz_log (YLOG_LOG, "%sNEW %d %s",
 641                      m_session_str, parent->m_seqno, m_proxyTarget);
 642             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 643             c->m_next = parent->m_clientPool;
 644             if (c->m_next)
 645                 c->m_next->m_prev = &c->m_next;
 646             parent->m_clientPool = c;
 647             c->m_prev = &parent->m_clientPool;
 648         }
 649
 650         xfree (c->m_cookie);
 651         c->m_cookie = 0;
 652         if (cookie)
 653             c->m_cookie = xstrdup(cookie);
 654
 655         c->m_seqno = parent->m_seqno;
 656         c->m_init_flag = 0;
 657         c->m_last_resultCount = 0;
 658         c->m_last_ok = 0;
 659         c->m_cache.clear();
 660         c->m_sr_transform = 0;
 661         c->m_waiting = 0;
 662         c->m_resultSetStartPoint = 0;
 663         (parent->m_seqno)++;
 664         if (c->client(m_proxyTarget))
 665         {
 666             delete c;
 667             return 0;
 668         }
 669         c->m_target_idletime = m_target_idletime;
 670         c->timeout(30);
 671
 672         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 673             c->set_APDU_yazlog(1);
 674         else
 675             c->set_APDU_yazlog(0);
 676     }
 677     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 678     return c;
 679 }
 680 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 681 {
 682     int i;
 683     for (i = 0; i<num; i++)
 684     {
 685         oident *ent;
 686         Z_DefaultDiagFormat *r;
 687         Z_DiagRec *p = pp[i];
 688         if (p->which != Z_DiagRec_defaultFormat)
 689         {
 690             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 691             return;
 692         }
 693         else
 694             r = p->u.defaultFormat;
 695         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 696             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 697             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 698         switch (r->which)
 699         {
 700         case Z_DefaultDiagFormat_v2Addinfo:
 701             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 702                     m_session_str,
 703                     *r->condition, diagbib1_str(*r->condition),
 704                     r->u.v2Addinfo);
 705             break;
 706         case Z_DefaultDiagFormat_v3Addinfo:
 707             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 708                     m_session_str,
 709                     *r->condition, diagbib1_str(*r->condition),
 710                     r->u.v3Addinfo);
 711             break;
 712         }
 713     }
 714 }
 715
 716 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 717 {
 718     if (!m_stylesheet_xsp || p->num_records <= 0)
 719         return 0;  /* no XSLT to be done ... */
 720
 721     m_stylesheet_offset = 0;
 722     m_stylesheet_nprl = p;
 723     m_stylesheet_apdu = apdu;
 724     timeout(0);
 725     return 1;
 726 }
 727
 728 void Yaz_Proxy::convert_xsl_delay()
 729 {
 730 #if HAVE_XSLT
 731     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 732     if (npr->which == Z_NamePlusRecord_databaseRecord)
 733     {
 734         Z_External *r = npr->u.databaseRecord;
 735         if (r->which == Z_External_octet)
 736         {
 737 #if 0
 738             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 739 #endif
 740             xmlDocPtr res, doc = xmlParseMemory(
 741                 (char*) r->u.octet_aligned->buf,
 742                 r->u.octet_aligned->len);
 743
 744
 745             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 746                     m_session_str, m_stylesheet_offset);
 747             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 748                                       doc, 0);
 749
 750             if (res)
 751             {
 752                 xmlChar *out_buf;
 753                 int out_len;
 754                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 755
 756                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 757                     u.databaseRecord =
 758                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 759                                  (char*) out_buf, out_len);
 760                 xmlFree(out_buf);
 761                 xmlFreeDoc(res);
 762             }
 763
 764             xmlFreeDoc(doc);
 765         }
 766     }
 767 #endif
 768     m_stylesheet_offset++;
 769     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 770     {
 771         m_stylesheet_nprl = 0;
 772 #if HAVE_XSLT
 773         if (m_stylesheet_xsp)
 774             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 775 #endif
 776         m_stylesheet_xsp = 0;
 777         timeout(m_client_idletime);
 778         send_PDU_convert(m_stylesheet_apdu);
 779     }
 780     else
 781         timeout(0);
 782 }
 783
 784 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 785 {
 786     if (m_frontend_type != VAL_NONE)
 787     {
 788         int i;
 789         for (i = 0; i < p->num_records; i++)
 790         {
 791             Z_NamePlusRecord *npr = p->records[i];
 792             if (npr->which == Z_NamePlusRecord_databaseRecord)
 793             {
 794                 Z_External *r = npr->u.databaseRecord;
 795                 if (r->which == Z_External_octet)
 796                 {
 797 #if HAVE_USEMARCON
 798                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 799                     {
 800                         if (!m_usemarcon->m_stage1)
 801                         {
 802                             m_usemarcon->m_stage1 = new CDetails();
 803                         }
 804                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 805                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 806                         int res = m_usemarcon->m_stage1->Start();
 807                         if (res == 0)
 808                         {
 809                             char *converted;
 810                             int convlen;
 811                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 812                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 813                             {
 814                                 if (!m_usemarcon->m_stage2)
 815                                 {
 816                                     m_usemarcon->m_stage2 = new CDetails();
 817                                 }
 818                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 819                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 820                                 res = m_usemarcon->m_stage2->Start();
 821                                 if (res == 0)
 822                                 {
 823                                     free(converted);
 824                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 825                                 }
 826                                 else
 827                                 {
 828                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 829                                 }
 830                             }
 831                             npr->u.databaseRecord =
 832                                 z_ext_record(odr_encode(),
 833                                              m_frontend_type,
 834                                              converted,
 835                                              strlen(converted));
 836                             free(converted);
 837                         }
 838                         else
 839                         {
 840                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 841                         }
 842                         continue;
 843                     }
 844 #endif
 845                     npr->u.databaseRecord =
 846                         z_ext_record(odr_encode(),
 847                                      m_frontend_type,
 848                                      (char*) r->u.octet_aligned->buf,
 849                                      r->u.octet_aligned->len);
 850                 }
 851             }
 852         }
 853     }
 854 }
 855
 856 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
 857                                    const char *backend_charset)
 858 {
 859     int i;
 860     if (!backend_charset)
 861         backend_charset = "MARC-8";
 862     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
 863     yaz_marc_t mt = yaz_marc_create();
 864     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 865     yaz_marc_iconv(mt, cd);
 866     for (i = 0; i < p->num_records; i++)
 867     {
 868         Z_NamePlusRecord *npr = p->records[i];
 869         if (npr->which == Z_NamePlusRecord_databaseRecord)
 870         {
 871             Z_External *r = npr->u.databaseRecord;
 872             if (r->which == Z_External_octet)
 873             {
 874                 int rlen;
 875                 char *result;
 876                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 877                                         r->u.octet_aligned->len,
 878                                         &result, &rlen))
 879                 {
 880                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 881                                                          VAL_TEXT_XML,
 882                                                          result, rlen);
 883                 }
 884             }
 885         }
 886     }
 887     if (cd)
 888         yaz_iconv_close(cd);
 889     yaz_marc_destroy(mt);
 890 }
 891
 892 void Yaz_Proxy::logtime()
 893 {
 894 #if HAVE_GETTIMEOFDAY
 895     struct timeval *tv = (struct timeval*) m_time_tv;
 896     if (tv->tv_sec)
 897     {
 898         struct timeval tv1;
 899         gettimeofday(&tv1, 0);
 900         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
 901             (tv1.tv_usec - tv->tv_usec);
 902         if (diff >= 0)
 903             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 904                     diff/1000000, (diff/1000)%1000);
 905     }
 906     tv->tv_sec = 0;
 907     tv->tv_usec = 0;
 908 #endif
 909 }
 910
 911 int Yaz_Proxy::send_http_response(int code)
 912 {
 913     ODR o = odr_encode();
 914     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 915     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 916     if (m_http_version)
 917         hres->version = odr_strdup(o, m_http_version);
 918     if (m_http_keepalive)
 919         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 920     else
 921         timeout(0);
 922
 923     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 924     {
 925         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
 926                  gdu_name(gdu));
 927     }
 928     int len;
 929     int r = send_GDU(gdu, &len);
 930     m_bytes_sent += len;
 931     m_bw_stat.add_bytes(len);
 932     logtime();
 933     return r;
 934 }
 935
 936 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 937 {
 938     ODR o = odr_encode();
 939     const char *ctype = "text/xml";
 940     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 941     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 942     if (m_http_version)
 943         hres->version = odr_strdup(o, m_http_version);
 944     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 945     if (m_http_keepalive)
 946         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 947     else
 948         timeout(0);
 949
 950     static Z_SOAP_Handler soap_handlers[2] = {
 951 #if HAVE_XSLT
 952         {"http://www.loc.gov/zing/srw/", 0,
 953          (Z_SOAP_fun) yaz_srw_codec},
 954 #endif
 955         {0, 0, 0}
 956     };
 957
 958     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 959     soap_package->which = Z_SOAP_generic;
 960     soap_package->u.generic =
 961         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 962     soap_package->u.generic->no = 0;
 963     soap_package->u.generic->ns = soap_handlers[0].ns;
 964     soap_package->u.generic->p = (void *) srw_pdu;
 965     soap_package->ns = m_soap_ns;
 966     z_soap_codec_enc_xsl(o, &soap_package,
 967                          &hres->content_buf, &hres->content_len,
 968                          soap_handlers, 0, m_s2z_stylesheet);
 969     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 970     {
 971         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
 972                  gdu_name(gdu));
 973     }
 974     int len;
 975     int r = send_GDU(gdu, &len);
 976     m_bytes_sent += len;
 977     m_bw_stat.add_bytes(len);
 978     logtime();
 979     return r;
 980 }
 981
 982 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
 983 {
 984     ODR o = odr_encode();
 985     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 986     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 987
 988     srw_res->num_diagnostics = 1;
 989     srw_res->diagnostics = (Z_SRW_diagnostic *)
 990         odr_malloc(o, sizeof(*srw_res->diagnostics));
 991     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
 992     return send_srw_response(srw_pdu);
 993 }
 994
 995 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 996                              Z_DefaultDiagFormat *ddf)
 997 {
 998     int bib1_code = *ddf->condition;
 999     if (bib1_code == 109)
1000         return 404;
1001     srw_res->num_diagnostics = 1;
1002     srw_res->diagnostics = (Z_SRW_diagnostic *)
1003         odr_malloc(o, sizeof(*srw_res->diagnostics));
1004     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1005                           yaz_diag_bib1_to_srw(*ddf->condition),
1006                           ddf->u.v2Addinfo);
1007     return 0;
1008 }
1009
1010 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1011 {
1012     ODR o = odr_encode();
1013     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1014     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1015
1016     srw_res->numberOfRecords = odr_intdup (o, hits);
1017     if (records && records->which == Z_Records_DBOSD)
1018     {
1019         srw_res->num_records =
1020             records->u.databaseOrSurDiagnostics->num_records;
1021         int i;
1022         srw_res->records = (Z_SRW_record *)
1023             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1024         for (i = 0; i < srw_res->num_records; i++)
1025         {
1026             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1027             if (npr->which != Z_NamePlusRecord_databaseRecord)
1028             {
1029                 srw_res->records[i].recordSchema = "diagnostic";
1030                 srw_res->records[i].recordPacking = m_s2z_packing;
1031                 srw_res->records[i].recordData_buf = "67";
1032                 srw_res->records[i].recordData_len = 2;
1033                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1034                 continue;
1035             }
1036             Z_External *r = npr->u.databaseRecord;
1037             oident *ent = oid_getentbyoid(r->direct_reference);
1038             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1039             {
1040                 srw_res->records[i].recordSchema = m_schema;
1041                 srw_res->records[i].recordPacking = m_s2z_packing;
1042                 srw_res->records[i].recordData_buf = (char*)
1043                     r->u.octet_aligned->buf;
1044                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1045                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1046             }
1047             else
1048             {
1049                 srw_res->records[i].recordSchema = "diagnostic";
1050                 srw_res->records[i].recordPacking = m_s2z_packing;
1051                 srw_res->records[i].recordData_buf = "67";
1052                 srw_res->records[i].recordData_len = 2;
1053                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1054             }
1055         }
1056     }
1057     if (records && records->which == Z_Records_NSD)
1058     {
1059         int http_code;
1060         http_code = z_to_srw_diag(odr_encode(), srw_res,
1061                                    records->u.nonSurrogateDiagnostic);
1062         if (http_code)
1063             return send_http_response(http_code);
1064     }
1065     return send_srw_response(srw_pdu);
1066
1067 }
1068
1069 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1070                                         int num_diagnostics)
1071 {
1072     Yaz_ProxyConfig *cfg = check_reconfigure();
1073     if (cfg)
1074     {
1075         int len;
1076         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1077                                        m_s2z_database, &len);
1078         if (b)
1079         {
1080             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1081             Z_SRW_explainResponse *er = res->u.explain_response;
1082
1083             er->record.recordData_buf = b;
1084             er->record.recordData_len = len;
1085             er->record.recordPacking = m_s2z_packing;
1086             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1087
1088             er->diagnostics = diagnostics;
1089             er->num_diagnostics = num_diagnostics;
1090             return send_srw_response(res);
1091         }
1092     }
1093     return send_http_response(404);
1094 }
1095
1096 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1097 {
1098     if (m_http_version)
1099     {
1100         if (apdu->which == Z_APDU_initResponse)
1101         {
1102             Z_InitResponse *res = apdu->u.initResponse;
1103             if (*res->result == 0)
1104             {
1105                 send_to_srw_client_error(3, 0);
1106             }
1107             else if (!m_s2z_search_apdu)
1108             {
1109                 send_srw_explain_response(0, 0);
1110             }
1111             else
1112             {
1113                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1114             }
1115         }
1116         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1117         {
1118             m_s2z_search_apdu = 0;
1119             Z_SearchResponse *res = apdu->u.searchResponse;
1120             m_s2z_hit_count = *res->resultCount;
1121             if (res->records && res->records->which == Z_Records_NSD)
1122             {
1123                 send_to_srw_client_ok(0, res->records, 1);
1124             }
1125             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1126             {
1127                 // adjust
1128                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1129
1130                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1131                 {
1132                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1133                         > m_s2z_hit_count)
1134                         *pr->numberOfRecordsRequested =
1135                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1136                 }
1137                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1138             }
1139             else
1140             {
1141                 m_s2z_present_apdu = 0;
1142                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1143             }
1144         }
1145         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1146         {
1147             int start =
1148                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1149
1150             m_s2z_present_apdu = 0;
1151             Z_PresentResponse *res = apdu->u.presentResponse;
1152             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1153         }
1154     }
1155     else
1156     {
1157         int len = 0;
1158         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1159             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1160                      apdu_name(apdu));
1161         int r = send_Z_PDU(apdu, &len);
1162         m_bytes_sent += len;
1163         m_bw_stat.add_bytes(len);
1164         logtime();
1165         return r;
1166     }
1167     return 0;
1168 }
1169
1170 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1171 {
1172     int kill_session = 0;
1173     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1174
1175     if (new_id)
1176         *new_id = m_referenceId;
1177
1178     if (apdu->which == Z_APDU_searchResponse)
1179     {
1180         Z_SearchResponse *sr = apdu->u.searchResponse;
1181         Z_Records *p = sr->records;
1182         if (p && p->which == Z_Records_NSD)
1183         {
1184             Z_DiagRec dr, *dr_p = &dr;
1185             dr.which = Z_DiagRec_defaultFormat;
1186             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1187
1188             *sr->searchStatus = 0;
1189             display_diagrecs(&dr_p, 1);
1190         }
1191         else
1192         {
1193             if (p && p->which == Z_Records_DBOSD)
1194             {
1195                 if (m_backend_type
1196 #if HAVE_USEMARCON
1197                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1198 #endif
1199                     )
1200                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1201                 if (m_marcxml_flag)
1202                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1203                                        m_backend_charset);
1204                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1205                     return 0;
1206
1207             }
1208             if (sr->resultCount)
1209             {
1210                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1211                         *sr->resultCount);
1212                 if (*sr->resultCount < 0)
1213                 {
1214                     m_invalid_session = 1;
1215                     kill_session = 1;
1216
1217                     *sr->searchStatus = 0;
1218                     sr->records =
1219                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1220                     *sr->resultCount = 0;
1221                 }
1222             }
1223         }
1224     }
1225     else if (apdu->which == Z_APDU_presentResponse)
1226     {
1227         Z_PresentResponse *sr = apdu->u.presentResponse;
1228         Z_Records *p = sr->records;
1229         if (p && p->which == Z_Records_NSD)
1230         {
1231             Z_DiagRec dr, *dr_p = &dr;
1232             dr.which = Z_DiagRec_defaultFormat;
1233             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1234             if (*sr->presentStatus == Z_PresentStatus_success)
1235                 *sr->presentStatus = Z_PresentStatus_failure;
1236             display_diagrecs(&dr_p, 1);
1237         }
1238         if (p && p->which == Z_Records_DBOSD)
1239         {
1240             if (m_backend_type
1241 #if HAVE_USEMARCON
1242                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1243 #endif
1244                 )
1245                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1246             if (m_marcxml_flag)
1247                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1248                                    m_backend_charset);
1249             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1250                 return 0;
1251         }
1252     }
1253     else if (apdu->which == Z_APDU_initResponse)
1254     {
1255         //Get and check negotiation record
1256         //from init response.
1257         handle_charset_lang_negotiation(apdu);
1258
1259         if (m_initRequest_options)
1260         {
1261             Z_Options *nopt =
1262                 (Odr_bitmask *)odr_malloc(odr_encode(),
1263                                           sizeof(Odr_bitmask));
1264             ODR_MASK_ZERO(nopt);
1265
1266             int i;
1267             for (i = 0; i<24; i++)
1268                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1269                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1270                     ODR_MASK_SET(nopt, i);
1271             apdu->u.initResponse->options = nopt;
1272         }
1273         if (m_initRequest_version)
1274         {
1275             Z_ProtocolVersion *nopt =
1276                 (Odr_bitmask *)odr_malloc(odr_encode(),
1277                                           sizeof(Odr_bitmask));
1278             ODR_MASK_ZERO(nopt);
1279
1280             int i;
1281             for (i = 0; i<8; i++)
1282                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1283                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1284                     ODR_MASK_SET(nopt, i);
1285             apdu->u.initResponse->protocolVersion = nopt;
1286         }
1287         apdu->u.initResponse->preferredMessageSize =
1288             odr_intdup(odr_encode(),
1289                        m_client->m_initResponse_preferredMessageSize >
1290                        m_initRequest_preferredMessageSize ?
1291                        m_initRequest_preferredMessageSize :
1292                        m_client->m_initResponse_preferredMessageSize);
1293         apdu->u.initResponse->maximumRecordSize =
1294             odr_intdup(odr_encode(),
1295                        m_client->m_initResponse_maximumRecordSize >
1296                        m_initRequest_maximumRecordSize ?
1297                        m_initRequest_maximumRecordSize :
1298                        m_client->m_initResponse_maximumRecordSize);
1299     }
1300
1301     int r = send_PDU_convert(apdu);
1302     if (r)
1303         return r;
1304     if (kill_session)
1305     {
1306         delete m_client;
1307         m_client = 0;
1308         m_parent->pre_init();
1309     }
1310     return r;
1311 }
1312
1313 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1314 {
1315     int len = 0;
1316     const char *apdu_name_tmp = apdu_name(apdu);
1317     int r = send_Z_PDU(apdu, &len);
1318     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1319         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1320                  get_session_str(),
1321                  apdu_name_tmp, get_hostname(), len);
1322     m_bytes_sent += len;
1323     return r;
1324 }
1325
1326 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1327 {
1328     if (apdu->which == Z_APDU_presentRequest)
1329     {
1330         Z_PresentRequest *pr = apdu->u.presentRequest;
1331         int toget = *pr->numberOfRecordsRequested;
1332         int start = *pr->resultSetStartPoint;
1333
1334         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1335                 pr->resultSetId, start, toget);
1336
1337         if (*m_parent->m_optimize == '0')
1338             return apdu;
1339
1340         if (!m_client->m_last_resultSetId)
1341         {
1342             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1343             new_apdu->u.presentResponse->records =
1344                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1345                                                pr->resultSetId);
1346             send_to_client(new_apdu);
1347             return 0;
1348         }
1349         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1350         {
1351             if (start+toget-1 > m_client->m_last_resultCount)
1352             {
1353                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1354                 new_apdu->u.presentResponse->records =
1355                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1356                 send_to_client(new_apdu);
1357                 return 0;
1358             }
1359             Z_NamePlusRecordList *npr;
1360 #if 0
1361             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1362                     m_session_str, start, toget, yaz_z3950oid_to_str(
1363                         pr->preferredRecordSyntax, &oclass));
1364 #endif
1365             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1366                                           pr->preferredRecordSyntax,
1367                                           pr->recordComposition))
1368             {
1369                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1370                          m_session_str);
1371                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1372                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1373
1374                 new_apdu->u.presentResponse->numberOfRecordsReturned
1375                     = odr_intdup(odr_encode(), toget);
1376
1377                 new_apdu->u.presentResponse->records = (Z_Records*)
1378                     odr_malloc(odr_encode(), sizeof(Z_Records));
1379                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1380                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1381                 new_apdu->u.presentResponse->nextResultSetPosition =
1382                     odr_intdup(odr_encode(), start+toget);
1383
1384                 send_to_client(new_apdu);
1385                 return 0;
1386             }
1387         }
1388     }
1389
1390     if (apdu->which != Z_APDU_searchRequest)
1391         return apdu;
1392     Z_SearchRequest *sr = apdu->u.searchRequest;
1393     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1394     Yaz_Z_Databases this_databases;
1395
1396     this_databases.set(sr->num_databaseNames, (const char **)
1397                        sr->databaseNames);
1398
1399     this_query->set_Z_Query(sr->query);
1400
1401     char query_str[120];
1402     this_query->print(query_str, sizeof(query_str)-1);
1403     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1404
1405     if (*m_parent->m_optimize != '0' &&
1406         m_client->m_last_ok && m_client->m_last_query &&
1407         m_client->m_last_query->match(this_query) &&
1408         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1409         m_client->m_last_databases.match(this_databases))
1410     {
1411         delete this_query;
1412         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1413             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1414         {
1415             Z_NamePlusRecordList *npr;
1416             int toget = *sr->mediumSetPresentNumber;
1417             Z_RecordComposition *comp = 0;
1418
1419             if (toget > m_client->m_last_resultCount)
1420                 toget = m_client->m_last_resultCount;
1421
1422             if (sr->mediumSetElementSetNames)
1423             {
1424                 comp = (Z_RecordComposition *)
1425                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1426                 comp->which = Z_RecordComp_simple;
1427                 comp->u.simple = sr->mediumSetElementSetNames;
1428             }
1429
1430             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1431                                           sr->preferredRecordSyntax, comp))
1432             {
1433                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1434                          m_session_str);
1435                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1436                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1437                 new_apdu->u.searchResponse->resultCount =
1438                     &m_client->m_last_resultCount;
1439
1440                 new_apdu->u.searchResponse->numberOfRecordsReturned
1441                     = odr_intdup(odr_encode(), toget);
1442
1443                 new_apdu->u.searchResponse->presentStatus =
1444                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1445                 new_apdu->u.searchResponse->records = (Z_Records*)
1446                     odr_malloc(odr_encode(), sizeof(Z_Records));
1447                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1448                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1449                 new_apdu->u.searchResponse->nextResultSetPosition =
1450                     odr_intdup(odr_encode(), toget+1);
1451                 send_to_client(new_apdu);
1452                 return 0;
1453             }
1454             else
1455             {
1456                 // medium Set
1457                 // send present request (medium size)
1458                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1459                          m_session_str);
1460
1461                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1462                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1463                 pr->referenceId = sr->referenceId;
1464                 pr->resultSetId = sr->resultSetName;
1465                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1466                 *pr->numberOfRecordsRequested = toget;
1467                 pr->recordComposition = comp;
1468                 m_client->m_sr_transform = 1;
1469                 return new_apdu;
1470             }
1471         }
1472         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1473             m_client->m_last_resultCount <= 0)
1474         {
1475             // large set. Return pseudo-search response immediately
1476             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1477                      m_session_str);
1478             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1479             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1480             new_apdu->u.searchResponse->resultCount =
1481                 &m_client->m_last_resultCount;
1482             send_to_client(new_apdu);
1483             return 0;
1484         }
1485         else
1486         {
1487             Z_NamePlusRecordList *npr;
1488             int toget = m_client->m_last_resultCount;
1489             Z_RecordComposition *comp = 0;
1490             // small set
1491             // send a present request (small set)
1492
1493             if (sr->smallSetElementSetNames)
1494             {
1495                 comp = (Z_RecordComposition *)
1496                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1497                 comp->which = Z_RecordComp_simple;
1498                 comp->u.simple = sr->smallSetElementSetNames;
1499             }
1500
1501             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1502                                           sr->preferredRecordSyntax, comp))
1503             {
1504                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1505                          m_session_str);
1506                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1507                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1508                 new_apdu->u.searchResponse->resultCount =
1509                     &m_client->m_last_resultCount;
1510
1511                 new_apdu->u.searchResponse->numberOfRecordsReturned
1512                     = odr_intdup(odr_encode(), toget);
1513
1514                 new_apdu->u.searchResponse->presentStatus =
1515                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1516                 new_apdu->u.searchResponse->records = (Z_Records*)
1517                     odr_malloc(odr_encode(), sizeof(Z_Records));
1518                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1519                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1520                 new_apdu->u.searchResponse->nextResultSetPosition =
1521                     odr_intdup(odr_encode(), toget+1);
1522                 send_to_client(new_apdu);
1523                 return 0;
1524             }
1525             else
1526             {
1527                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1528                          m_session_str);
1529                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1530                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1531                 pr->referenceId = sr->referenceId;
1532                 pr->resultSetId = sr->resultSetName;
1533                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1534                 *pr->numberOfRecordsRequested = toget;
1535                 pr->recordComposition = comp;
1536                 m_client->m_sr_transform = 1;
1537                 return new_apdu;
1538             }
1539         }
1540     }
1541     else  // query doesn't match
1542     {
1543         delete m_client->m_last_query;
1544         m_client->m_last_query = this_query;
1545         m_client->m_last_ok = 0;
1546         m_client->m_cache.clear();
1547         m_client->m_resultSetStartPoint = 0;
1548
1549         xfree (m_client->m_last_resultSetId);
1550         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1551
1552         m_client->m_last_databases.set(sr->num_databaseNames,
1553                                        (const char **) sr->databaseNames);
1554     }
1555     return apdu;
1556 }
1557
1558
1559 void Yaz_Proxy::inc_request_no()
1560 {
1561     char *cp = strchr(m_session_str, ' ');
1562     m_request_no++;
1563     if (cp)
1564         sprintf(cp+1, "%d ", m_request_no);
1565 }
1566
1567 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1568 {
1569     inc_request_no();
1570
1571     m_bytes_recv += len;
1572
1573     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1574         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1575                  m_session_str, gdu_name(apdu), len);
1576
1577     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1578         shutdown();
1579
1580     m_bw_stat.add_bytes(len);
1581     m_pdu_stat.add_bytes(1);
1582
1583 #if HAVE_GETTIMEOFDAY
1584     gettimeofday((struct timeval *) m_time_tv, 0);
1585 #endif
1586
1587     int bw_total = m_bw_stat.get_total();
1588     int pdu_total = m_pdu_stat.get_total();
1589
1590     int reduce = 0;
1591     if (m_bw_max)
1592     {
1593         if (bw_total > m_bw_max)
1594         {
1595             reduce = (bw_total/m_bw_max);
1596         }
1597     }
1598     if (m_pdu_max)
1599     {
1600         if (pdu_total > m_pdu_max)
1601         {
1602             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1603             reduce = (reduce > nreduce) ? reduce : nreduce;
1604         }
1605     }
1606     m_http_version = 0;
1607     if (reduce)
1608     {
1609         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1610                 m_session_str, reduce, bw_total, pdu_total,
1611                 m_bw_max, m_pdu_max);
1612
1613         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1614         timeout(reduce);       // call us reduce seconds later
1615     }
1616     else if (apdu->which == Z_GDU_Z3950)
1617         handle_incoming_Z_PDU(apdu->u.z3950);
1618     else if (apdu->which == Z_GDU_HTTP_Request)
1619         handle_incoming_HTTP(apdu->u.HTTP_Request);
1620 }
1621
1622 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1623 {
1624     if (m_max_record_retrieve)
1625     {
1626         if (apdu->which == Z_APDU_presentRequest)
1627         {
1628             Z_PresentRequest *pr = apdu->u.presentRequest;
1629             if (pr->numberOfRecordsRequested &&
1630                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1631                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1632         }
1633     }
1634 }
1635 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1636 {
1637     if (apdu->which == Z_APDU_initRequest)
1638     {
1639         if (m_initRequest_options &&
1640             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1641             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1642         {
1643             // There is not negotiation proposal from
1644             // client's side. OK. The proxy negotiation
1645             // in use, only.
1646             Z_InitRequest *initRequest = apdu->u.initRequest;
1647             Z_OtherInformation **otherInfo;
1648             Z_OtherInformationUnit *oi;
1649             get_otherInfoAPDU(apdu, &otherInfo);
1650             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1651             if (oi)
1652             {
1653                 ODR_MASK_SET(initRequest->options,
1654                     Z_Options_negotiationModel);
1655                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1656                 oi->information.externallyDefinedInfo =
1657                 yaz_set_proposal_charneg(odr_encode(),
1658                     (const char**)&m_proxy_negotiation_charset,
1659                     m_proxy_negotiation_charset ? 1:0,
1660                     (const char**)&m_proxy_negotiation_lang,
1661                     m_proxy_negotiation_lang ? 1:0,
1662                     1);
1663             }
1664         }
1665     }
1666     else if (apdu->which == Z_APDU_initResponse)
1667     {
1668         Z_InitResponse *initResponse = apdu->u.initResponse;
1669         Z_OtherInformation **otherInfo;
1670         Z_OtherInformationUnit *oi;
1671
1672         if (ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel))
1673         {
1674             char *charset = 0;
1675             char *lang = 0;
1676             int selected = 0;
1677
1678             get_otherInfoAPDU(apdu, &otherInfo);
1679
1680             if (!otherInfo && !(*otherInfo))
1681                 return;
1682
1683             Z_CharSetandLanguageNegotiation *charneg =
1684                 yaz_get_charneg_record(*otherInfo);
1685
1686             if (!charneg)
1687                 return;
1688
1689             yaz_get_response_charneg(m_referenceId_mem, charneg,
1690                 &charset, &lang, &selected);
1691
1692             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
1693                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
1694
1695             if (m_initRequest_options &&
1696                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
1697             {
1698                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
1699                     m_session_str);
1700             }
1701             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1702             {
1703                 // negotiation-charset, negotiation-lang
1704                 // elements of config file in use.
1705
1706                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
1707                     m_session_str);
1708
1709                 // clear negotiation option.
1710                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
1711
1712                 // Delete negotiation (charneg-3) entry.
1713                 Z_OtherInformation *p = *otherInfo;
1714                 for (int i=0; i<p->num_elements; i++)
1715                 {
1716                     if (p->list[i]->which == Z_OtherInfo_externallyDefinedInfo)
1717                     {
1718                         Z_External *pext =
1719                             p->list[i]->information.externallyDefinedInfo;
1720                         struct oident *e = oid_getentbyoid(pext->direct_reference);
1721
1722                         if (e && e->value == VAL_CHARNEG3 && e->oclass == CLASS_NEGOT &&
1723                             pext->which == Z_External_charSetandLanguageNegotiation)
1724                         {
1725                             (p->num_elements)--;
1726                             if(p->num_elements == 0)
1727                             {
1728                                 *otherInfo = 0;
1729                             }
1730                             else
1731                             {
1732                                 for (int j=i; j<p->num_elements;j++)
1733                                     p->list[j] = p->list[j+1];
1734                             }
1735                         }
1736                     }
1737                 }
1738             }
1739         }
1740     }
1741 }
1742 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1743                                                      int error,
1744                                                      const char *addinfo)
1745 {
1746     Z_Records *rec = (Z_Records *)
1747         odr_malloc (odr, sizeof(*rec));
1748     int *err = (int *)
1749         odr_malloc (odr, sizeof(*err));
1750     Z_DiagRec *drec = (Z_DiagRec *)
1751         odr_malloc (odr, sizeof(*drec));
1752     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1753         odr_malloc (odr, sizeof(*dr));
1754     *err = error;
1755     rec->which = Z_Records_NSD;
1756     rec->u.nonSurrogateDiagnostic = dr;
1757     dr->diagnosticSetId =
1758         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1759     dr->condition = err;
1760     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1761     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1762     return rec;
1763 }
1764
1765 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1766 {
1767     if (apdu->which == Z_APDU_searchRequest &&
1768         apdu->u.searchRequest->query &&
1769         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1770         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1771     {
1772         Z_RPNQuery *rpnquery = 0;
1773         Z_SearchRequest *sr = apdu->u.searchRequest;
1774         char *addinfo = 0;
1775
1776         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
1777                 sr->query->u.type_104->u.cql);
1778
1779         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1780                                           &rpnquery, odr_encode(),
1781                                           &addinfo);
1782         if (r == -3)
1783             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
1784         else if (r)
1785         {
1786             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1787             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1788
1789             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1790             new_apdu->u.searchResponse->records =
1791                 create_nonSurrogateDiagnostics(odr_encode(),
1792                                                yaz_diag_srw_to_bib1(r),
1793                                                addinfo);
1794             *new_apdu->u.searchResponse->searchStatus = 0;
1795
1796             send_to_client(new_apdu);
1797
1798             return 0;
1799         }
1800         else
1801         {
1802             sr->query->which = Z_Query_type_1;
1803             sr->query->u.type_1 = rpnquery;
1804         }
1805         return apdu;
1806     }
1807     return apdu;
1808 }
1809
1810 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1811 {
1812     if (apdu->which == Z_APDU_searchRequest)
1813     {
1814         Z_SearchRequest *sr = apdu->u.searchRequest;
1815         int err = 0;
1816         char *addinfo = 0;
1817
1818         Yaz_ProxyConfig *cfg = check_reconfigure();
1819         if (cfg)
1820             err = cfg->check_query(odr_encode(), m_default_target,
1821                                    sr->query, &addinfo);
1822         if (err)
1823         {
1824             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1825
1826             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1827             new_apdu->u.searchResponse->records =
1828                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1829             *new_apdu->u.searchResponse->searchStatus = 0;
1830
1831             send_to_client(new_apdu);
1832
1833             return 0;
1834         }
1835     }
1836     return apdu;
1837 }
1838
1839 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1840 {
1841     m_marcxml_flag = 0;
1842     if (apdu->which == Z_APDU_searchRequest)
1843     {
1844         Z_SearchRequest *sr = apdu->u.searchRequest;
1845         int err = 0;
1846         char *addinfo = 0;
1847         Yaz_ProxyConfig *cfg = check_reconfigure();
1848
1849         Z_RecordComposition rc_temp, *rc = 0;
1850         if (sr->smallSetElementSetNames)
1851         {
1852             rc_temp.which = Z_RecordComp_simple;
1853             rc_temp.u.simple = sr->smallSetElementSetNames;
1854             rc = &rc_temp;
1855         }
1856
1857         if (sr->preferredRecordSyntax)
1858         {
1859             struct oident *ent;
1860             ent = oid_getentbyoid(sr->preferredRecordSyntax);
1861             m_frontend_type = ent->value;
1862         }
1863         else
1864             m_frontend_type = VAL_NONE;
1865
1866         char *stylesheet_name = 0;
1867         if (cfg)
1868             err = cfg->check_syntax(odr_encode(),
1869                                     m_default_target,
1870                                     sr->preferredRecordSyntax, rc,
1871                                     &addinfo, &stylesheet_name, &m_schema,
1872                                     &m_backend_type, &m_backend_charset,
1873                                     &m_usemarcon_ini_stage1,
1874                                     &m_usemarcon_ini_stage2);
1875         if (stylesheet_name)
1876         {
1877             m_parent->low_socket_close();
1878
1879 #if HAVE_XSLT
1880             if (m_stylesheet_xsp)
1881                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
1882             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1883                                                        stylesheet_name);
1884 #endif
1885             m_stylesheet_offset = 0;
1886             xfree(stylesheet_name);
1887
1888             m_parent->low_socket_open();
1889         }
1890         if (err == -1)
1891         {
1892             sr->smallSetElementSetNames = 0;
1893             sr->mediumSetElementSetNames = 0;
1894             if (m_backend_type)
1895             {
1896
1897                 sr->preferredRecordSyntax =
1898                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
1899                                         m_backend_type);
1900             }
1901             else
1902                 sr->preferredRecordSyntax =
1903                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
1904                                            VAL_USMARC);
1905             m_marcxml_flag = 1;
1906         }
1907         else if (err)
1908         {
1909             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1910
1911             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1912             new_apdu->u.searchResponse->records =
1913                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1914             *new_apdu->u.searchResponse->searchStatus = 0;
1915
1916             send_to_client(new_apdu);
1917
1918             return 0;
1919         }
1920         else if (m_backend_type)
1921         {
1922             sr->preferredRecordSyntax =
1923                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
1924         }
1925     }
1926     else if (apdu->which == Z_APDU_presentRequest)
1927     {
1928         Z_PresentRequest *pr = apdu->u.presentRequest;
1929         int err = 0;
1930         char *addinfo = 0;
1931         Yaz_ProxyConfig *cfg = check_reconfigure();
1932
1933         if (pr->preferredRecordSyntax)
1934         {
1935             struct oident *ent;
1936             ent = oid_getentbyoid(pr->preferredRecordSyntax);
1937             m_frontend_type = ent->value;
1938         }
1939         else
1940             m_frontend_type = VAL_NONE;
1941
1942         char *stylesheet_name = 0;
1943         if (cfg)
1944             err = cfg->check_syntax(odr_encode(), m_default_target,
1945                                     pr->preferredRecordSyntax,
1946                                     pr->recordComposition,
1947                                     &addinfo, &stylesheet_name, &m_schema,
1948                                     &m_backend_type, &m_backend_charset,
1949                                     &m_usemarcon_ini_stage1,
1950                                     &m_usemarcon_ini_stage2
1951                                     );
1952         if (stylesheet_name)
1953         {
1954             m_parent->low_socket_close();
1955
1956 #if HAVE_XSLT
1957             if (m_stylesheet_xsp)
1958                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
1959             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1960                                                        stylesheet_name);
1961 #endif
1962             m_stylesheet_offset = 0;
1963             xfree(stylesheet_name);
1964
1965             m_parent->low_socket_open();
1966         }
1967         if (err == -1)
1968         {
1969             pr->recordComposition = 0;
1970             if (m_backend_type)
1971             {
1972
1973                 pr->preferredRecordSyntax =
1974                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
1975                                         m_backend_type);
1976             }
1977             else
1978                 pr->preferredRecordSyntax =
1979                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
1980                                            VAL_USMARC);
1981             m_marcxml_flag = 1;
1982         }
1983         else if (err)
1984         {
1985             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1986
1987             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1988             new_apdu->u.presentResponse->records =
1989                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1990             *new_apdu->u.presentResponse->presentStatus =
1991                 Z_PresentStatus_failure;
1992
1993             send_to_client(new_apdu);
1994
1995             return 0;
1996         }
1997         else if (m_backend_type)
1998         {
1999             pr->preferredRecordSyntax =
2000                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2001         }
2002     }
2003     return apdu;
2004 }
2005
2006 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2007 {
2008     if (!schema)
2009         return 0;
2010     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2011         odr_malloc(o, sizeof(Z_ElementSetNames));
2012     esn->which = Z_ElementSetNames_generic;
2013     esn->u.generic = odr_strdup(o, schema);
2014     return esn;
2015 }
2016
2017 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2018 {
2019     const char *t = 0;
2020     Yaz_ProxyConfig *cfg = check_reconfigure();
2021     if (cfg)
2022         t = cfg->get_explain_name(db, backend_db);
2023
2024     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2025     {
2026         releaseClient();
2027     }
2028
2029     if (t)
2030     {
2031         xfree(m_default_target);
2032         m_default_target = xstrdup(t);
2033     }
2034 }
2035
2036 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2037 {
2038     struct stat sbuf;
2039     yaz_log(YLOG_LOG, "file_access");
2040     if (strcmp(hreq->method, "GET"))
2041         return 0;
2042     if (hreq->path[0] != '/')
2043     {
2044         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2045         return 0;
2046     }
2047     const char *cp = hreq->path;
2048     while (*cp)
2049     {
2050         if (*cp == '/' && strchr("/.", cp[1]))
2051         {
2052             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2053             return 0;
2054         }
2055         cp++;
2056     }
2057     const char *fname = hreq->path+1;
2058     if (stat(fname, &sbuf))
2059     {
2060         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2061         return 0;
2062     }
2063     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2064     {
2065         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2066         return 0;
2067     }
2068     if (sbuf.st_size > (off_t) 1000000)
2069     {
2070         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2071         return 0;
2072     }
2073
2074     ODR o = odr_encode();
2075     Yaz_ProxyConfig *cfg = check_reconfigure();
2076     const char *ctype = cfg->check_mime_type(fname);
2077     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2078     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2079     if (m_http_version)
2080         hres->version = odr_strdup(o, m_http_version);
2081     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2082     if (m_http_keepalive)
2083         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2084     else
2085         timeout(0);
2086
2087     hres->content_len = sbuf.st_size;
2088     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2089     FILE *f = fopen(fname, "rb");
2090     if (f)
2091     {
2092         fread(hres->content_buf, 1, hres->content_len, f);
2093         fclose(f);
2094     }
2095     else
2096     {
2097         return 0;
2098     }
2099     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2100     {
2101         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2102                  fname);
2103     }
2104     int len;
2105     send_GDU(gdu, &len);
2106     return 1;
2107 }
2108
2109 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2110 {
2111     if (m_s2z_odr_init)
2112     {
2113         odr_destroy(m_s2z_odr_init);
2114         m_s2z_odr_init = 0;
2115     }
2116     if (m_s2z_odr_search)
2117     {
2118         odr_destroy(m_s2z_odr_search);
2119         m_s2z_odr_search = 0;
2120     }
2121
2122     m_http_keepalive = 0;
2123     m_http_version = 0;
2124     if (!strcmp(hreq->version, "1.0"))
2125     {
2126         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2127         if (v && !strcmp(v, "Keep-Alive"))
2128             m_http_keepalive = 1;
2129         else
2130             m_http_keepalive = 0;
2131         m_http_version = "1.0";
2132     }
2133     else
2134     {
2135         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2136         if (v && !strcmp(v, "close"))
2137             m_http_keepalive = 0;
2138         else
2139             m_http_keepalive = 1;
2140         m_http_version = "1.1";
2141     }
2142
2143     Z_SRW_PDU *srw_pdu = 0;
2144     Z_SOAP *soap_package = 0;
2145     char *charset = 0;
2146     Z_SRW_diagnostic *diagnostic = 0;
2147     int num_diagnostic = 0;
2148
2149     if (file_access(hreq))
2150     {
2151         return;
2152     }
2153     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2154                             &charset) == 0
2155              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2156                                &charset, &diagnostic, &num_diagnostic) == 0)
2157     {
2158         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2159         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2160         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2161         m_s2z_init_apdu = 0;
2162         m_s2z_search_apdu = 0;
2163         m_s2z_present_apdu = 0;
2164
2165         m_s2z_stylesheet = 0;
2166
2167         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2168         {
2169             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2170
2171             const char *backend_db = srw_req->database;
2172             srw_get_client(srw_req->database, &backend_db);
2173
2174             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2175             // recordXPath unsupported.
2176             if (srw_req->recordXPath)
2177             {
2178                 yaz_add_srw_diagnostic(odr_decode(),
2179                                        &diagnostic, &num_diagnostic,
2180                                        72, 0);
2181             }
2182             // sort unsupported
2183             if (srw_req->sort_type != Z_SRW_sort_type_none)
2184             {
2185                 yaz_add_srw_diagnostic(odr_decode(),
2186                                        &diagnostic, &num_diagnostic,
2187                                        80, 0);
2188             }
2189             // save stylesheet
2190             if (srw_req->stylesheet)
2191                 m_s2z_stylesheet =
2192                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2193
2194             // set packing for response records ..
2195             if (srw_req->recordPacking &&
2196                 !strcmp(srw_req->recordPacking, "xml"))
2197                 m_s2z_packing = Z_SRW_recordPacking_XML;
2198             else
2199                 m_s2z_packing = Z_SRW_recordPacking_string;
2200
2201             if (num_diagnostic)
2202             {
2203                 Z_SRW_PDU *srw_pdu =
2204                     yaz_srw_get(odr_encode(),
2205                                 Z_SRW_searchRetrieve_response);
2206                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2207
2208                 srw_res->diagnostics = diagnostic;
2209                 srw_res->num_diagnostics = num_diagnostic;
2210                 send_srw_response(srw_pdu);
2211                 return;
2212             }
2213
2214             // prepare search PDU
2215             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2216                                           Z_APDU_searchRequest);
2217             Z_SearchRequest *z_searchRequest =
2218                 m_s2z_search_apdu->u.searchRequest;
2219
2220             z_searchRequest->num_databaseNames = 1;
2221             z_searchRequest->databaseNames = (char**)
2222                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2223             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2224                                                            backend_db);
2225
2226             // query transformation
2227             Z_Query *query = (Z_Query *)
2228                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2229             z_searchRequest->query = query;
2230
2231             if (srw_req->query_type == Z_SRW_query_type_cql)
2232             {
2233                 Z_External *ext = (Z_External *)
2234                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2235                 ext->direct_reference =
2236                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2237                 ext->indirect_reference = 0;
2238                 ext->descriptor = 0;
2239                 ext->which = Z_External_CQL;
2240                 ext->u.cql = srw_req->query.cql;
2241
2242                 query->which = Z_Query_type_104;
2243                 query->u.type_104 =  ext;
2244             }
2245             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2246             {
2247                 Z_RPNQuery *RPNquery;
2248                 YAZ_PQF_Parser pqf_parser;
2249
2250                 pqf_parser = yaz_pqf_create ();
2251
2252                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2253                                           srw_req->query.pqf);
2254                 if (!RPNquery)
2255                 {
2256                     const char *pqf_msg;
2257                     size_t off;
2258                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2259                     yaz_log(YLOG_LOG, "%*s^\n", off+4, "");
2260                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2261
2262                     send_to_srw_client_error(10, 0);
2263                     return;
2264                 }
2265                 query->which = Z_Query_type_1;
2266                 query->u.type_1 =  RPNquery;
2267
2268                 yaz_pqf_destroy (pqf_parser);
2269             }
2270             else
2271             {
2272                 send_to_srw_client_error(7, "query");
2273                 return;
2274             }
2275
2276             // present
2277             m_s2z_present_apdu = 0;
2278             int max = 0;
2279             if (srw_req->maximumRecords)
2280                 max = *srw_req->maximumRecords;
2281             int start = 1;
2282             if (srw_req->startRecord)
2283                 start = *srw_req->startRecord;
2284             if (max > 0)
2285             {
2286                 // Some backend, such as Voyager doesn't honor piggyback
2287                 // So we use present always (0 &&).
2288                 if (0 && start <= 1)  // Z39.50 piggyback
2289                 {
2290                     *z_searchRequest->smallSetUpperBound = max;
2291                     *z_searchRequest->mediumSetPresentNumber = max;
2292                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2293
2294                     z_searchRequest->preferredRecordSyntax =
2295                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2296                                                VAL_TEXT_XML);
2297                     if (srw_req->recordSchema)
2298                     {
2299                         z_searchRequest->smallSetElementSetNames =
2300                             z_searchRequest->mediumSetElementSetNames =
2301                             mk_esn_from_schema(m_s2z_odr_search,
2302                                                srw_req->recordSchema);
2303                     }
2304                 }
2305                 else   // Z39.50 present
2306                 {
2307                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2308                                                    Z_APDU_presentRequest);
2309                     Z_PresentRequest *z_presentRequest =
2310                         m_s2z_present_apdu->u.presentRequest;
2311                     *z_presentRequest->resultSetStartPoint = start;
2312                     *z_presentRequest->numberOfRecordsRequested = max;
2313                     z_presentRequest->preferredRecordSyntax =
2314                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2315                                                VAL_TEXT_XML);
2316                     if (srw_req->recordSchema)
2317                     {
2318                         z_presentRequest->recordComposition =
2319                             (Z_RecordComposition *)
2320                             odr_malloc(m_s2z_odr_search,
2321                                        sizeof(Z_RecordComposition));
2322                         z_presentRequest->recordComposition->which =
2323                             Z_RecordComp_simple;
2324                         z_presentRequest->recordComposition->u.simple =
2325                             mk_esn_from_schema(m_s2z_odr_search,
2326                                                srw_req->recordSchema);
2327                     }
2328                 }
2329             }
2330             if (!m_client)
2331             {
2332                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2333                                             Z_APDU_initRequest);
2334
2335                 // prevent m_initRequest_apdu memory from being grabbed
2336                 // in Yaz_Proxy::handle_incoming_Z_PDU
2337                 m_initRequest_apdu = m_s2z_init_apdu;
2338                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2339                 return;
2340             }
2341             else
2342             {
2343                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2344                 return;
2345             }
2346         }
2347         else if (srw_pdu->which == Z_SRW_explain_request)
2348         {
2349             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2350
2351             const char *backend_db = srw_req->database;
2352             srw_get_client(srw_req->database, &backend_db);
2353
2354             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2355
2356             // save stylesheet
2357             if (srw_req->stylesheet)
2358                 m_s2z_stylesheet =
2359                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2360
2361             if (srw_req->recordPacking &&
2362                 !strcmp(srw_req->recordPacking, "xml"))
2363                 m_s2z_packing = Z_SRW_recordPacking_XML;
2364             else
2365                 m_s2z_packing = Z_SRW_recordPacking_string;
2366
2367             if (num_diagnostic)
2368             {
2369                 send_srw_explain_response(diagnostic, num_diagnostic);
2370                 return;
2371             }
2372
2373             if (!m_client)
2374             {
2375                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2376                                             Z_APDU_initRequest);
2377
2378                 // prevent m_initRequest_apdu memory from being grabbed
2379                 // in Yaz_Proxy::handle_incoming_Z_PDU
2380                 m_initRequest_apdu = m_s2z_init_apdu;
2381                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2382             }
2383             else
2384                 send_srw_explain_response(0, 0);
2385             return;
2386         }
2387         else if (srw_pdu->which == Z_SRW_scan_request)
2388         {
2389             m_s2z_database = odr_strdup(m_s2z_odr_init,
2390                                         srw_pdu->u.scan_request->database);
2391
2392             yaz_add_srw_diagnostic(odr_decode(),
2393                                    &diagnostic, &num_diagnostic,
2394                                    4, "scan");
2395             Z_SRW_PDU *srw_pdu =
2396                 yaz_srw_get(odr_encode(),
2397                             Z_SRW_scan_response);
2398             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2399
2400             srw_res->diagnostics = diagnostic;
2401             srw_res->num_diagnostics = num_diagnostic;
2402             send_srw_response(srw_pdu);
2403             return;
2404         }
2405         else
2406         {
2407             m_s2z_database = 0;
2408
2409             send_to_srw_client_error(4, 0);
2410         }
2411     }
2412     send_http_response(400);
2413 }
2414
2415 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2416 {
2417     Z_ReferenceId **refid = get_referenceIdP(apdu);
2418     nmem_reset(m_referenceId_mem);
2419     if (refid && *refid)
2420     {
2421         m_referenceId = (Z_ReferenceId *)
2422             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2423         m_referenceId->len = m_referenceId->size = (*refid)->len;
2424         m_referenceId->buf = (unsigned char *)
2425             nmem_malloc(m_referenceId_mem, (*refid)->len);
2426         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2427     }
2428     else
2429         m_referenceId = 0;
2430
2431     if (!m_client && m_invalid_session)
2432     {
2433         m_apdu_invalid_session = apdu;
2434         m_mem_invalid_session = odr_extract_mem(odr_decode());
2435         apdu = m_initRequest_apdu;
2436     }
2437
2438     // Determine our client.
2439     Z_OtherInformation **oi;
2440     get_otherInfoAPDU(apdu, &oi);
2441     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2442     if (!m_client)
2443     {
2444         if (m_http_version)
2445         {
2446             send_http_response(404);
2447             return;
2448         }
2449         else
2450         {
2451             delete this;
2452             return;
2453         }
2454     }
2455
2456     m_client->m_server = this;
2457
2458     if (apdu->which == Z_APDU_initRequest)
2459     {
2460         if (apdu->u.initRequest->implementationId)
2461             yaz_log(YLOG_LOG, "%simplementationId: %s",
2462                     m_session_str, apdu->u.initRequest->implementationId);
2463         if (apdu->u.initRequest->implementationName)
2464             yaz_log(YLOG_LOG, "%simplementationName: %s",
2465                     m_session_str, apdu->u.initRequest->implementationName);
2466         if (apdu->u.initRequest->implementationVersion)
2467             yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2468                     m_session_str, apdu->u.initRequest->implementationVersion);
2469         if (m_initRequest_apdu == 0)
2470         {
2471             if (m_initRequest_mem)
2472                 nmem_destroy(m_initRequest_mem);
2473
2474             m_initRequest_apdu = apdu;
2475             m_initRequest_mem = odr_extract_mem(odr_decode());
2476
2477             m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2478                 preferredMessageSize;
2479             *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2480             m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2481                 maximumRecordSize;
2482             *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2483
2484             // Save proposal charsets and langs.
2485             if (ODR_MASK_GET(apdu->u.initRequest->options,
2486                 Z_Options_negotiationModel))
2487             {
2488                 Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2489                     yaz_get_charneg_record(*oi);
2490
2491                 yaz_get_proposal_charneg(m_referenceId_mem,
2492                     charSetandLangRecord,
2493                     &m_initRequest_oi_negotiation_charsets,
2494                     &m_initRequest_oi_negotiation_num_charsets,
2495                     &m_initRequest_oi_negotiation_langs,
2496                     &m_initRequest_oi_negotiation_num_langs,
2497                     &m_initRequest_oi_negotiation_selected);
2498
2499                 for (int i=0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2500                 {
2501                     yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2502                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2503                         m_initRequest_oi_negotiation_charsets[i]:"none");
2504                 }
2505                 for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2506                 {
2507                     yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2508                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2509                         m_initRequest_oi_negotiation_langs[i]:"none");
2510                 }
2511                 yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2512                     m_session_str, m_initRequest_oi_negotiation_selected);
2513             }
2514             // save init options for the response..
2515             m_initRequest_options = apdu->u.initRequest->options;
2516
2517             apdu->u.initRequest->options =
2518                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2519                                            sizeof(Odr_bitmask));
2520             ODR_MASK_ZERO(apdu->u.initRequest->options);
2521             int i;
2522             for (i = 0; i<= 24; i++)
2523                 ODR_MASK_SET(apdu->u.initRequest->options, i);
2524             // check negotiation option
2525             if (!ODR_MASK_GET(m_initRequest_options,
2526                 Z_Options_negotiationModel))
2527             {
2528                 ODR_MASK_CLEAR(apdu->u.initRequest->options,
2529                     Z_Options_negotiationModel);
2530             }
2531             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2532                 Z_Options_concurrentOperations);
2533             // make new version
2534             m_initRequest_version = apdu->u.initRequest->protocolVersion;
2535             apdu->u.initRequest->protocolVersion =
2536                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2537                                            sizeof(Odr_bitmask));
2538             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2539
2540             for (i = 0; i<= 8; i++)
2541                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2542         }
2543         if (m_client->m_init_flag)
2544         {
2545             if (handle_init_response_for_invalid_session(apdu))
2546                 return;
2547             if (m_client->m_initResponse)
2548             {
2549                 Z_APDU *apdu2 = m_client->m_initResponse;
2550                 apdu2->u.initResponse->otherInfo = 0;
2551                 if (m_client->m_cookie && *m_client->m_cookie)
2552                     set_otherInformationString(apdu2, VAL_COOKIE, 1,
2553                                                m_client->m_cookie);
2554                 apdu2->u.initResponse->referenceId =
2555                     apdu->u.initRequest->referenceId;
2556                 apdu2->u.initResponse->options = m_client->m_initResponse_options;
2557                 apdu2->u.initResponse->protocolVersion =
2558                     m_client->m_initResponse_version;
2559
2560                 send_to_client(apdu2);
2561                 return;
2562             }
2563         }
2564         m_client->m_init_flag = 1;
2565     }
2566     handle_max_record_retrieve(apdu);
2567
2568     if (apdu)
2569         apdu = handle_syntax_validation(apdu);
2570
2571     if (apdu)
2572         apdu = handle_query_transformation(apdu);
2573
2574     if (apdu)
2575         apdu = handle_query_validation(apdu);
2576
2577     if (apdu)
2578         apdu = result_set_optimize(apdu);
2579     if (!apdu)
2580     {
2581         m_client->timeout(m_target_idletime);  // mark it active even
2582         // though we didn't use it
2583         return;
2584     }
2585     // Add otherInformation entry in APDU if
2586     // negotiatoin in use.
2587     if (apdu)
2588         handle_charset_lang_negotiation(apdu);
2589
2590     // delete other info construct completely if 0 elements
2591     get_otherInfoAPDU(apdu, &oi);
2592     if (oi && *oi && (*oi)->num_elements == 0)
2593         *oi = 0;
2594
2595     if (apdu->which == Z_APDU_presentRequest &&
2596         m_client->m_resultSetStartPoint == 0)
2597     {
2598         Z_PresentRequest *pr = apdu->u.presentRequest;
2599         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2600         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2601     } else {
2602         m_client->m_resultSetStartPoint = 0;
2603     }
2604     if (m_client->send_to_target(apdu) < 0)
2605     {
2606         delete m_client;
2607         m_client = 0;
2608         delete this;
2609     }
2610     else
2611         m_client->m_waiting = 1;
2612 }
2613
2614 void Yaz_Proxy::connectNotify()
2615 {
2616 }
2617
2618 void Yaz_Proxy::releaseClient()
2619 {
2620     xfree(m_proxyTarget);
2621     m_proxyTarget = 0;
2622     m_invalid_session = 0;
2623     // only keep if keep_alive flag is set...
2624     if (m_client &&
2625         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2626         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2627         m_client->m_waiting == 0)
2628     {
2629         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2630                  m_session_str,
2631                  m_client->get_hostname());
2632         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2633                 m_session_str, m_client->m_pdu_recv,
2634                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2635                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2636         assert (m_client->m_waiting != 2);
2637         // Tell client (if any) that no server connection is there..
2638         m_client->m_server = 0;
2639         m_client = 0;
2640     }
2641     else if (m_client)
2642     {
2643         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
2644                  m_session_str,
2645                  m_client->get_hostname());
2646         assert (m_client->m_waiting != 2);
2647         delete m_client;
2648         m_client = 0;
2649     }
2650     else if (!m_parent)
2651     {
2652         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
2653                  m_session_str);
2654         assert (m_parent);
2655     }
2656     else
2657     {
2658         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
2659                  m_session_str);
2660     }
2661     if (m_parent)
2662         m_parent->pre_init();
2663 }
2664
2665 void Yaz_Proxy::shutdown()
2666 {
2667     releaseClient();
2668     delete this;
2669 }
2670
2671 const char *Yaz_ProxyClient::get_session_str()
2672 {
2673     if (!m_server)
2674         return "0 ";
2675     return m_server->get_session_str();
2676 }
2677
2678 void Yaz_ProxyClient::shutdown()
2679 {
2680     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2681              get_hostname());
2682     delete m_server;
2683     delete this;
2684 }
2685
2686 void Yaz_Proxy::failNotify()
2687 {
2688     inc_request_no();
2689     yaz_log (YLOG_LOG, "%sConnection closed by client",
2690              get_session_str());
2691     shutdown();
2692 }
2693
2694 void Yaz_ProxyClient::failNotify()
2695 {
2696     if (m_server)
2697         m_server->inc_request_no();
2698     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
2699              get_session_str(), get_hostname());
2700     shutdown();
2701 }
2702
2703 void Yaz_ProxyClient::connectNotify()
2704 {
2705     const char *s = get_session_str();
2706     const char *h = get_hostname();
2707     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2708              m_target_idletime);
2709     timeout(m_target_idletime);
2710     if (!m_server)
2711         pre_init_client();
2712 }
2713
2714 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2715                                                   *the_PDU_Observable, int fd)
2716 {
2717     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2718 }
2719
2720 Yaz_ProxyClient::~Yaz_ProxyClient()
2721 {
2722     if (m_prev)
2723         *m_prev = m_next;
2724     if (m_next)
2725         m_next->m_prev = m_prev;
2726     m_waiting = 2;     // for debugging purposes only.
2727     odr_destroy(m_init_odr);
2728     delete m_last_query;
2729     xfree (m_last_resultSetId);
2730     xfree (m_cookie);
2731 }
2732
2733 void Yaz_ProxyClient::pre_init_client()
2734 {
2735     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2736     Z_InitRequest *req = apdu->u.initRequest;
2737
2738     int i;
2739     for (i = 0; i<= 24; i++)
2740         ODR_MASK_SET(req->options, i);
2741     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2742                    Z_Options_negotiationModel);
2743     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2744                    Z_Options_concurrentOperations);
2745     for (i = 0; i<= 10; i++)
2746         ODR_MASK_SET(req->protocolVersion, i);
2747
2748     if (send_to_target(apdu) < 0)
2749     {
2750         delete this;
2751     }
2752     else
2753     {
2754         m_waiting = 1;
2755         m_init_flag = 1;
2756     }
2757 }
2758
2759 void Yaz_Proxy::pre_init()
2760 {
2761     int i;
2762     const char *name = 0;
2763     const char *zurl_in_use[MAX_ZURL_PLEX];
2764     int limit_bw, limit_pdu, limit_req;
2765     int target_idletime, client_idletime;
2766     int max_clients;
2767     int keepalive_limit_bw, keepalive_limit_pdu;
2768     int pre_init;
2769     const char *cql2rpn = 0;
2770     const char *authentication = 0;
2771     const char *negotiation_charset = 0;
2772     const char *negotiation_lang = 0;
2773
2774     Yaz_ProxyConfig *cfg = check_reconfigure();
2775
2776     zurl_in_use[0] = 0;
2777
2778     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2779         set_APDU_yazlog(1);
2780     else
2781         set_APDU_yazlog(0);
2782
2783     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2784                                           &limit_bw, &limit_pdu, &limit_req,
2785                                           &target_idletime, &client_idletime,
2786                                           &max_clients,
2787                                           &keepalive_limit_bw,
2788                                           &keepalive_limit_pdu,
2789                                           &pre_init,
2790                                           &cql2rpn,
2791                                           &authentication,
2792                                           &negotiation_charset,
2793                                           &negotiation_lang) ; i++)
2794     {
2795         if (pre_init)
2796         {
2797             int j;
2798             for (j = 0; zurl_in_use[j]; j++)
2799             {
2800                 Yaz_ProxyClient *c;
2801                 int spare = 0;
2802                 int spare_waiting = 0;
2803                 int in_use = 0;
2804                 int other = 0;
2805                 for (c = m_clientPool; c; c = c->m_next)
2806                 {
2807                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
2808                     {
2809                         if (c->m_cookie == 0)
2810                         {
2811                             if (c->m_server == 0)
2812                                 if (c->m_waiting)
2813                                     spare_waiting++;
2814                                 else
2815                                     spare++;
2816                             else
2817                                 in_use++;
2818                         }
2819                         else
2820                             other++;
2821                     }
2822                 }
2823                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2824                         "sparew=%d preinit=%d",m_session_str,
2825                         name, zurl_in_use[j], in_use, other,
2826                         spare, spare_waiting, pre_init);
2827                 if (spare + spare_waiting < pre_init)
2828                 {
2829                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2830                     c->m_next = m_clientPool;
2831                     if (c->m_next)
2832                         c->m_next->m_prev = &c->m_next;
2833                     m_clientPool = c;
2834                     c->m_prev = &m_clientPool;
2835
2836                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
2837                         c->set_APDU_yazlog(1);
2838                     else
2839                         c->set_APDU_yazlog(0);
2840
2841                     if (c->client(zurl_in_use[j]))
2842                     {
2843                         timeout(60);
2844                         delete c;
2845                         return;
2846                     }
2847                     c->timeout(30);
2848                     c->m_waiting = 1;
2849                     c->m_target_idletime = target_idletime;
2850                     c->m_seqno = m_seqno++;
2851                 }
2852             }
2853         }
2854     }
2855 }
2856
2857 void Yaz_Proxy::timeoutNotify()
2858 {
2859     if (m_parent)
2860     {
2861         if (m_bw_hold_PDU)
2862         {
2863             timeout(m_client_idletime);
2864             Z_GDU *apdu = m_bw_hold_PDU;
2865             m_bw_hold_PDU = 0;
2866
2867             if (apdu->which == Z_GDU_Z3950)
2868                 handle_incoming_Z_PDU(apdu->u.z3950);
2869             else if (apdu->which == Z_GDU_HTTP_Request)
2870                 handle_incoming_HTTP(apdu->u.HTTP_Request);
2871         }
2872         else if (m_stylesheet_nprl)
2873             convert_xsl_delay();
2874         else
2875         {
2876             inc_request_no();
2877
2878             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2879             shutdown();
2880         }
2881     }
2882     else
2883     {
2884         timeout(600);
2885         pre_init();
2886     }
2887 }
2888
2889 void Yaz_Proxy::markInvalid()
2890 {
2891     m_client = 0;
2892     m_invalid_session = 1;
2893 }
2894
2895 void Yaz_ProxyClient::timeoutNotify()
2896 {
2897     if (m_server)
2898         m_server->inc_request_no();
2899
2900     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2901              get_hostname());
2902     m_waiting = 1;
2903     m_root->pre_init();
2904     if (m_server && m_init_flag)
2905     {
2906         // target timed out in a session that was properly initialized
2907         // server object stay alive but we mark it as invalid so it
2908         // gets initialized again
2909         m_server->markInvalid();
2910         m_server = 0;
2911     }
2912     shutdown();
2913 }
2914
2915 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2916                                  Yaz_Proxy *parent) :
2917     Yaz_Z_Assoc (the_PDU_Observable)
2918 {
2919     m_cookie = 0;
2920     m_next = 0;
2921     m_prev = 0;
2922     m_init_flag = 0;
2923     m_last_query = 0;
2924     m_last_resultSetId = 0;
2925     m_last_resultCount = 0;
2926     m_last_ok = 0;
2927     m_sr_transform = 0;
2928     m_waiting = 0;
2929     m_init_odr = odr_createmem (ODR_DECODE);
2930     m_initResponse = 0;
2931     m_initResponse_options = 0;
2932     m_initResponse_version = 0;
2933     m_initResponse_preferredMessageSize = 0;
2934     m_initResponse_maximumRecordSize = 0;
2935     m_resultSetStartPoint = 0;
2936     m_bytes_sent = m_bytes_recv = 0;
2937     m_pdu_recv = 0;
2938     m_server = 0;
2939     m_seqno = 0;
2940     m_target_idletime = 600;
2941     m_root = parent;
2942 }
2943
2944 const char *Yaz_Proxy::option(const char *name, const char *value)
2945 {
2946     if (!strcmp (name, "optimize")) {
2947         if (value) {
2948             xfree (m_optimize);
2949             m_optimize = xstrdup (value);
2950         }
2951         return m_optimize;
2952     }
2953     return 0;
2954 }
2955
2956 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2957 {
2958
2959 }
2960
2961 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2962 {
2963     if (apdu->which == Z_GDU_Z3950)
2964         recv_Z_PDU(apdu->u.z3950, len);
2965     else if (apdu->which == Z_GDU_HTTP_Response)
2966         recv_HTTP_response(apdu->u.HTTP_Response, len);
2967     else
2968         shutdown();
2969 }
2970
2971 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2972 {
2973     if (!m_invalid_session)
2974         return 0;
2975     m_invalid_session = 0;
2976     handle_incoming_Z_PDU(m_apdu_invalid_session);
2977     assert (m_mem_invalid_session);
2978     nmem_destroy(m_mem_invalid_session);
2979     m_mem_invalid_session = 0;
2980     return 1;
2981 }
2982
2983 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2984 {
2985     m_bytes_recv += len;
2986
2987     m_pdu_recv++;
2988     m_waiting = 0;
2989     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2990         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2991                  apdu_name(apdu), get_hostname(), len);
2992     if (apdu->which == Z_APDU_initResponse)
2993     {
2994         if (!m_server)  // if this is a pre init session , check for more
2995             m_root->pre_init();
2996         NMEM nmem = odr_extract_mem (odr_decode());
2997         odr_reset (m_init_odr);
2998         nmem_transfer (m_init_odr->mem, nmem);
2999         m_initResponse = apdu;
3000         m_initResponse_options = apdu->u.initResponse->options;
3001         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3002         m_initResponse_preferredMessageSize =
3003             *apdu->u.initResponse->preferredMessageSize;
3004         m_initResponse_maximumRecordSize =
3005             *apdu->u.initResponse->maximumRecordSize;
3006
3007         Z_InitResponse *ir = apdu->u.initResponse;
3008         char *im0 = ir->implementationName;
3009
3010         char *im1 = (char*)
3011             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3012         *im1 = '\0';
3013         if (im0)
3014         {
3015             strcat(im1, im0);
3016             strcat(im1, " ");
3017         }
3018         strcat(im1, "(YAZ Proxy)");
3019         ir->implementationName = im1;
3020
3021         nmem_destroy (nmem);
3022
3023         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3024             return;
3025     }
3026     if (apdu->which == Z_APDU_searchResponse)
3027     {
3028         Z_SearchResponse *sr = apdu->u.searchResponse;
3029         m_last_resultCount = *sr->resultCount;
3030         int status = *sr->searchStatus;
3031         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3032         {
3033             m_last_ok = 1;
3034
3035             if (sr->records && sr->records->which == Z_Records_DBOSD)
3036             {
3037                 m_cache.add(odr_decode(),
3038                             sr->records->u.databaseOrSurDiagnostics, 1,
3039                             *sr->resultCount);
3040             }
3041         }
3042     }
3043     if (apdu->which == Z_APDU_presentResponse)
3044     {
3045         Z_PresentResponse *pr = apdu->u.presentResponse;
3046         if (m_sr_transform)
3047         {
3048             m_sr_transform = 0;
3049             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3050             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3051             sr->referenceId = pr->referenceId;
3052             *sr->resultCount = m_last_resultCount;
3053             sr->records = pr->records;
3054             sr->nextResultSetPosition = pr->nextResultSetPosition;
3055             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3056             apdu = new_apdu;
3057         }
3058         if (pr->records &&
3059             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3060         {
3061             m_cache.add(odr_decode(),
3062                         pr->records->u.databaseOrSurDiagnostics,
3063                         m_resultSetStartPoint, -1);
3064             m_resultSetStartPoint = 0;
3065         }
3066     }
3067     if (m_cookie)
3068         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3069     if (m_server)
3070     {
3071         m_server->send_to_client(apdu);
3072     }
3073     if (apdu->which == Z_APDU_close)
3074     {
3075         shutdown();
3076     }
3077 }
3078
3079 void Yaz_Proxy::low_socket_close()
3080 {
3081 #if WIN32
3082 #else
3083     int i;
3084     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3085         if  (m_lo_fd[i] >= 0)
3086             ::close(m_lo_fd[i]);
3087 #endif
3088 }
3089
3090 void Yaz_Proxy::low_socket_open()
3091 {
3092 #if WIN32
3093 #else
3094     int i;
3095     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3096         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3097 #endif
3098 }
3099
3100 int Yaz_Proxy::server(const char *addr)
3101 {
3102     int r = Yaz_Z_Assoc::server(addr);
3103     if (!r)
3104     {
3105         yaz_log(YLOG_LOG, "%sStarted proxy "
3106 #ifdef VERSION
3107             VERSION
3108 #endif
3109             " on %s", m_session_str, addr);
3110         timeout(1);
3111     }
3112     return r;
3113 }
3114