src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.19 2005-02-07 13:29:38 adam Exp $
   2    Copyright (c) 1998-2005, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <time.h>
  42 #include <fcntl.h>
  43
  44 #include <yaz/srw.h>
  45 #include <yaz/marcdisp.h>
  46 #include <yaz/yaz-iconv.h>
  47 #include <yaz/log.h>
  48 #include <yaz/diagbib1.h>
  49 #include "proxyp.h"
  50 #include <yaz/pquery.h>
  51 #include <yaz/otherinfo.h>
  52
  53 static const char *apdu_name(Z_APDU *apdu)
  54 {
  55     switch (apdu->which)
  56     {
  57     case Z_APDU_initRequest:
  58         return "initRequest";
  59     case Z_APDU_initResponse:
  60         return "initResponse";
  61     case Z_APDU_searchRequest:
  62         return "searchRequest";
  63     case Z_APDU_searchResponse:
  64         return "searchResponse";
  65     case Z_APDU_presentRequest:
  66         return "presentRequest";
  67     case Z_APDU_presentResponse:
  68         return "presentResponse";
  69     case Z_APDU_deleteResultSetRequest:
  70         return "deleteResultSetRequest";
  71     case Z_APDU_deleteResultSetResponse:
  72         return "deleteResultSetResponse";
  73     case Z_APDU_scanRequest:
  74         return "scanRequest";
  75     case Z_APDU_scanResponse:
  76         return "scanResponse";
  77     case Z_APDU_sortRequest:
  78         return "sortRequest";
  79     case Z_APDU_sortResponse:
  80         return "sortResponse";
  81     case Z_APDU_extendedServicesRequest:
  82         return "extendedServicesRequest";
  83     case Z_APDU_extendedServicesResponse:
  84         return "extendedServicesResponse";
  85     case Z_APDU_close:
  86         return "close";
  87     }
  88     return "other";
  89 }
  90
  91 static const char *gdu_name(Z_GDU *gdu)
  92 {
  93     switch(gdu->which)
  94     {
  95     case Z_GDU_Z3950:
  96         return apdu_name(gdu->u.z3950);
  97     case Z_GDU_HTTP_Request:
  98         return "HTTP Request";
  99     case Z_GDU_HTTP_Response:
 100         return "HTTP Response";
 101     }
 102     return "Unknown request/response";
 103 }
 104
 105 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
 106                      Yaz_Proxy *parent) :
 107     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 108 {
 109     m_PDU_Observable = the_PDU_Observable;
 110     m_client = 0;
 111     m_parent = parent;
 112     m_clientPool = 0;
 113     m_seqno = 1;
 114     m_keepalive_limit_bw = 500000;
 115     m_keepalive_limit_pdu = 1000;
 116     m_proxyTarget = 0;
 117     m_default_target = 0;
 118     m_proxy_authentication = 0;
 119     m_max_clients = 150;
 120     m_log_mask = 0;
 121     m_seed = time(0);
 122     m_client_idletime = 600;
 123     m_target_idletime = 600;
 124     m_optimize = xstrdup ("1");
 125     strcpy(m_session_str, "0 ");
 126     m_session_no=0;
 127     m_bytes_sent = 0;
 128     m_bytes_recv = 0;
 129     m_bw_hold_PDU = 0;
 130     m_bw_max = 0;
 131     m_pdu_max = 0;
 132     m_max_record_retrieve = 0;
 133     m_reconfig_flag = 0;
 134     m_config_fname = 0;
 135     m_request_no = 0;
 136     m_invalid_session = 0;
 137     m_referenceId = 0;
 138     m_referenceId_mem = nmem_create();
 139     m_config = 0;
 140     m_marcxml_flag = 0;
 141     m_stylesheet_xsp = 0;
 142     m_stylesheet_nprl = 0;
 143     m_s2z_stylesheet = 0;
 144     m_s2z_database = 0;
 145     m_schema = 0;
 146     m_backend_type = 0;
 147     m_backend_charset = 0;
 148     m_frontend_type = 0;
 149     m_initRequest_apdu = 0;
 150     m_initRequest_mem = 0;
 151     m_initRequest_preferredMessageSize = 0;
 152     m_initRequest_maximumRecordSize = 0;
 153     m_initRequest_options = 0;
 154     m_initRequest_version = 0;
 155     m_apdu_invalid_session = 0;
 156     m_mem_invalid_session = 0;
 157     m_s2z_odr_init = 0;
 158     m_s2z_odr_search = 0;
 159     m_s2z_init_apdu = 0;
 160     m_s2z_search_apdu = 0;
 161     m_s2z_present_apdu = 0;
 162     m_http_keepalive = 0;
 163     m_http_version = 0;
 164     m_soap_ns = 0;
 165     m_s2z_packing = Z_SRW_recordPacking_string;
 166 #if HAVE_GETTIMEOFDAY
 167     m_time_tv = xmalloc(sizeof(struct timeval));
 168     struct timeval *tv = (struct timeval *) m_time_tv;
 169     tv->tv_sec = 0;
 170     tv->tv_usec = 0;
 171 #else
 172     m_time_tv = 0;
 173 #endif
 174     m_usemarcon_ini_stage1 = 0;
 175     m_usemarcon_ini_stage2 = 0;
 176     m_usemarcon = new Yaz_usemarcon();
 177     if (!m_parent)
 178         low_socket_open();
 179 }
 180
 181 Yaz_Proxy::~Yaz_Proxy()
 182 {
 183     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 184             m_bytes_sent, m_bytes_recv);
 185     nmem_destroy(m_initRequest_mem);
 186     nmem_destroy(m_mem_invalid_session);
 187     nmem_destroy(m_referenceId_mem);
 188
 189     xfree(m_proxyTarget);
 190     xfree(m_default_target);
 191     xfree(m_proxy_authentication);
 192     xfree(m_optimize);
 193
 194 #if HAVE_XSLT
 195     if (m_stylesheet_xsp)
 196         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 197 #endif
 198     xfree (m_time_tv);
 199
 200     xfree (m_schema);
 201     xfree (m_backend_type);
 202     xfree (m_backend_charset);
 203     xfree (m_usemarcon_ini_stage1);
 204     xfree (m_usemarcon_ini_stage2);
 205     delete m_usemarcon;
 206     if (m_s2z_odr_init)
 207         odr_destroy(m_s2z_odr_init);
 208     if (m_s2z_odr_search)
 209         odr_destroy(m_s2z_odr_search);
 210     if (!m_parent)
 211         low_socket_close();
 212     delete m_config;
 213 }
 214
 215 int Yaz_Proxy::set_config(const char *config)
 216 {
 217     delete m_config;
 218     m_config = new Yaz_ProxyConfig();
 219     xfree(m_config_fname);
 220     m_config_fname = xstrdup(config);
 221     int r = m_config->read_xml(config);
 222     if (!r)
 223         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 224     return r;
 225 }
 226
 227 void Yaz_Proxy::set_default_target(const char *target)
 228 {
 229     xfree (m_default_target);
 230     m_default_target = 0;
 231     if (target)
 232         m_default_target = (char *) xstrdup (target);
 233 }
 234
 235 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 236 {
 237     xfree (m_proxy_authentication);
 238     m_proxy_authentication = 0;
 239     if (auth)
 240         m_proxy_authentication = (char *) xstrdup (auth);
 241 }
 242
 243 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 244 {
 245     if (m_parent)
 246         return m_parent->check_reconfigure();
 247
 248     Yaz_ProxyConfig *cfg = m_config;
 249     if (m_reconfig_flag)
 250     {
 251         yaz_log(YLOG_LOG, "reconfigure");
 252         yaz_log_reopen();
 253         if (m_config_fname && cfg)
 254         {
 255             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 256             int r = cfg->read_xml(m_config_fname);
 257             if (r)
 258                 yaz_log(YLOG_WARN, "reconfigure failed");
 259             else
 260             {
 261                 m_log_mask = 0;
 262                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 263             }
 264         }
 265         else
 266             yaz_log(YLOG_LOG, "reconfigure");
 267         m_reconfig_flag = 0;
 268     }
 269     return cfg;
 270 }
 271
 272 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 273                                             *the_PDU_Observable, int fd)
 274 {
 275     check_reconfigure();
 276     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 277     new_proxy->m_config = 0;
 278     new_proxy->m_config_fname = 0;
 279     new_proxy->timeout(m_client_idletime);
 280     new_proxy->m_target_idletime = m_target_idletime;
 281     new_proxy->set_default_target(m_default_target);
 282     new_proxy->m_max_clients = m_max_clients;
 283     new_proxy->m_log_mask = m_log_mask;
 284     new_proxy->set_APDU_log(get_APDU_log());
 285     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 286         new_proxy->set_APDU_yazlog(1);
 287     else
 288         new_proxy->set_APDU_yazlog(0);
 289     new_proxy->set_proxy_authentication(m_proxy_authentication);
 290     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 291     m_session_no++;
 292     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 293              the_PDU_Observable->getpeername());
 294     return new_proxy;
 295 }
 296
 297 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 298 {
 299     int oid[OID_SIZE];
 300     Z_OtherInformationUnit *oi;
 301     struct oident ent;
 302     ent.proto = PROTO_Z3950;
 303     ent.oclass = CLASS_USERINFO;
 304     ent.value = (oid_value) VAL_COOKIE;
 305     assert (oid_ent_to_oid (&ent, oid));
 306
 307     if (oid_ent_to_oid (&ent, oid) &&
 308         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 309         oi->which == Z_OtherInfo_characterInfo)
 310         return oi->information.characterInfo;
 311     return 0;
 312 }
 313
 314 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 315 {
 316     int oid[OID_SIZE];
 317     Z_OtherInformationUnit *oi;
 318     struct oident ent;
 319     ent.proto = PROTO_Z3950;
 320     ent.oclass = CLASS_USERINFO;
 321     ent.value = (oid_value) VAL_PROXY;
 322     if (oid_ent_to_oid (&ent, oid) &&
 323         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 324         oi->which == Z_OtherInfo_characterInfo)
 325         return oi->information.characterInfo;
 326     return 0;
 327 }
 328
 329 const char *Yaz_Proxy::load_balance(const char **url)
 330 {
 331     int zurl_in_use[MAX_ZURL_PLEX];
 332     int zurl_in_spare[MAX_ZURL_PLEX];
 333     Yaz_ProxyClient *c;
 334     int i;
 335
 336     for (i = 0; i<MAX_ZURL_PLEX; i++)
 337     {
 338         zurl_in_use[i] = 0;
 339         zurl_in_spare[i] = 0;
 340     }
 341     for (c = m_parent->m_clientPool; c; c = c->m_next)
 342     {
 343         for (i = 0; url[i]; i++)
 344             if (!strcmp(url[i], c->get_hostname()))
 345             {
 346                 zurl_in_use[i]++;
 347                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 348                     zurl_in_spare[i]++;
 349             }
 350     }
 351     int min_use = 100000;
 352     int spare_for_min = 0;
 353     int max_spare = 0;
 354     const char *ret_min = 0;
 355     const char *ret_spare = 0;
 356     for (i = 0; url[i]; i++)
 357     {
 358         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 359                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 360         if (min_use > zurl_in_use[i])
 361         {
 362             ret_min = url[i];
 363             min_use = zurl_in_use[i];
 364             spare_for_min = zurl_in_spare[i];
 365         }
 366         if (max_spare < zurl_in_spare[i])
 367         {
 368             ret_spare = url[i];
 369             max_spare = zurl_in_spare[i];
 370         }
 371     }
 372     // use the one with minimum connections if spare is > 3
 373     if (spare_for_min > 3)
 374         return ret_min;
 375     // use one with most spares (if any)
 376     if (max_spare > 0)
 377         return ret_spare;
 378     return ret_min;
 379 }
 380
 381 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 382                                        const char *proxy_host)
 383 {
 384     assert (m_parent);
 385     Yaz_Proxy *parent = m_parent;
 386     Yaz_ProxyClient *c = m_client;
 387
 388     if (!m_proxyTarget)
 389     {
 390         const char *url[MAX_ZURL_PLEX];
 391         Yaz_ProxyConfig *cfg = check_reconfigure();
 392         if (proxy_host)
 393         {
 394 #if 0
 395 /* only to be enabled for debugging... */
 396             if (!strcmp(proxy_host, "stop"))
 397                 exit(0);
 398 #endif
 399             xfree(m_default_target);
 400             m_default_target = xstrdup(proxy_host);
 401         }
 402         proxy_host = m_default_target;
 403         int client_idletime = -1;
 404         const char *cql2rpn_fname = 0;
 405         const char *authentication = 0;
 406         url[0] = m_default_target;
 407         url[1] = 0;
 408         if (cfg)
 409         {
 410             int pre_init = 0;
 411             cfg->get_target_info(proxy_host, url, &m_bw_max,
 412                                  &m_pdu_max, &m_max_record_retrieve,
 413                                  &m_target_idletime, &client_idletime,
 414                                  &parent->m_max_clients,
 415                                  &m_keepalive_limit_bw,
 416                                  &m_keepalive_limit_pdu,
 417                                  &pre_init,
 418                                  &cql2rpn_fname,
 419                                  &authentication);
 420         }
 421         if (client_idletime != -1)
 422         {
 423             m_client_idletime = client_idletime;
 424             timeout(m_client_idletime);
 425         }
 426         if (cql2rpn_fname)
 427             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 428         if (authentication)
 429             set_proxy_authentication(authentication);
 430         if (!url[0])
 431         {
 432             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 433             return 0;
 434         }
 435         // we don't handle multiplexing for cookie session, so we just
 436         // pick the first one in this case (anonymous users will be able
 437         // to use any backend)
 438         if (cookie && *cookie)
 439             m_proxyTarget = (char*) xstrdup(url[0]);
 440         else
 441             m_proxyTarget = (char*) xstrdup(load_balance(url));
 442     }
 443     if (cookie && *cookie)
 444     {   // search in sessions with a cookie
 445         for (c = parent->m_clientPool; c; c = c->m_next)
 446         {
 447             assert (c->m_prev);
 448             assert (*c->m_prev == c);
 449             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 450                 !strcmp(m_proxyTarget, c->get_hostname()))
 451             {
 452                 // Found it in cache
 453                 // The following handles "cancel"
 454                 // If connection is busy (waiting for PDU) and
 455                 // we have an initRequest we can safely do re-open
 456                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 457                 {
 458                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 459                              c->get_hostname());
 460                     c->close();
 461                     c->m_init_flag = 0;
 462
 463                     c->m_last_ok = 0;
 464                     c->m_cache.clear();
 465                     c->m_last_resultCount = 0;
 466                     c->m_sr_transform = 0;
 467                     c->m_waiting = 0;
 468                     c->m_resultSetStartPoint = 0;
 469                     c->m_target_idletime = m_target_idletime;
 470                     if (c->client(m_proxyTarget))
 471                     {
 472                         delete c;
 473                         return 0;
 474                     }
 475                     c->timeout(30);
 476                 }
 477                 c->m_seqno = parent->m_seqno;
 478                 if (c->m_server && c->m_server != this)
 479                     c->m_server->m_client = 0;
 480                 c->m_server = this;
 481                 (parent->m_seqno)++;
 482                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 483                 return c;
 484             }
 485         }
 486     }
 487     else if (!c && apdu->which == Z_APDU_initRequest
 488              && apdu->u.initRequest->idAuthentication == 0)
 489     {
 490         // anonymous sessions without cookie.
 491         // if authentication is set it is NOT anonymous se we can't share them.
 492         for (c = parent->m_clientPool; c; c = c->m_next)
 493         {
 494             assert(c->m_prev);
 495             assert(*c->m_prev == c);
 496             if (c->m_server == 0 && c->m_cookie == 0 &&
 497                 c->m_waiting == 0 &&
 498                 !strcmp(m_proxyTarget, c->get_hostname()))
 499             {
 500                 // found it in cache
 501                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 502                          m_session_str, parent->m_seqno, c->get_hostname());
 503
 504                 c->m_seqno = parent->m_seqno;
 505                 assert(c->m_server == 0);
 506                 c->m_server = this;
 507
 508                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 509                     c->set_APDU_yazlog(1);
 510                 else
 511                     c->set_APDU_yazlog(0);
 512
 513                 (parent->m_seqno)++;
 514
 515                 parent->pre_init();
 516
 517                 return c;
 518             }
 519         }
 520     }
 521     if (!m_client)
 522     {
 523         if (apdu->which != Z_APDU_initRequest)
 524         {
 525             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 526             return 0;
 527         }
 528         Z_InitRequest *initRequest = apdu->u.initRequest;
 529
 530         if (!initRequest->idAuthentication)
 531         {
 532             if (m_proxy_authentication)
 533             {
 534                 initRequest->idAuthentication =
 535                     (Z_IdAuthentication *)
 536                     odr_malloc (odr_encode(),
 537                                 sizeof(*initRequest->idAuthentication));
 538                 initRequest->idAuthentication->which =
 539                     Z_IdAuthentication_open;
 540                 initRequest->idAuthentication->u.open =
 541                     odr_strdup (odr_encode(), m_proxy_authentication);
 542             }
 543         }
 544         else
 545         {
 546             // the client use authentication. We set the keepalive PDU
 547             // to 0 so we don't cache it in releaseClient
 548             m_keepalive_limit_pdu = 0;
 549         }
 550         // go through list of clients - and find the lowest/oldest one.
 551         Yaz_ProxyClient *c_min = 0;
 552         int min_seq = -1;
 553         int no_of_clients = 0;
 554         if (parent->m_clientPool)
 555             yaz_log (YLOG_DEBUG, "Existing sessions");
 556         for (c = parent->m_clientPool; c; c = c->m_next)
 557         {
 558             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 559                                c->m_waiting, c->get_hostname(),
 560                                c->m_cookie ? c->m_cookie : "");
 561             no_of_clients++;
 562             if (min_seq < 0 || c->m_seqno < min_seq)
 563             {
 564                 min_seq = c->m_seqno;
 565                 c_min = c;
 566             }
 567         }
 568         if (no_of_clients >= parent->m_max_clients)
 569         {
 570             c = c_min;
 571             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 572             {
 573                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 574                          m_session_str, parent->m_max_clients, c->m_seqno);
 575                 if (c->m_server && c->m_server != this)
 576                     delete c->m_server;
 577                 c->m_server = 0;
 578             }
 579             else
 580             {
 581                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 582                          m_session_str, parent->m_max_clients,
 583                          c->m_seqno, parent->m_seqno, c->get_hostname());
 584                 xfree (c->m_cookie);
 585                 c->m_cookie = 0;
 586                 if (cookie)
 587                     c->m_cookie = xstrdup(cookie);
 588                 c->m_seqno = parent->m_seqno;
 589                 if (c->m_server && c->m_server != this)
 590                 {
 591                     c->m_server->m_client = 0;
 592                     delete c->m_server;
 593                 }
 594                 (parent->m_seqno)++;
 595                 c->m_target_idletime = m_target_idletime;
 596                 c->timeout(m_target_idletime);
 597
 598                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 599                     c->set_APDU_yazlog(1);
 600                 else
 601                     c->set_APDU_yazlog(0);
 602
 603                 return c;
 604             }
 605         }
 606         else
 607         {
 608             yaz_log (YLOG_LOG, "%sNEW %d %s",
 609                      m_session_str, parent->m_seqno, m_proxyTarget);
 610             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 611             c->m_next = parent->m_clientPool;
 612             if (c->m_next)
 613                 c->m_next->m_prev = &c->m_next;
 614             parent->m_clientPool = c;
 615             c->m_prev = &parent->m_clientPool;
 616         }
 617
 618         xfree (c->m_cookie);
 619         c->m_cookie = 0;
 620         if (cookie)
 621             c->m_cookie = xstrdup(cookie);
 622
 623         c->m_seqno = parent->m_seqno;
 624         c->m_init_flag = 0;
 625         c->m_last_resultCount = 0;
 626         c->m_last_ok = 0;
 627         c->m_cache.clear();
 628         c->m_sr_transform = 0;
 629         c->m_waiting = 0;
 630         c->m_resultSetStartPoint = 0;
 631         (parent->m_seqno)++;
 632         if (c->client(m_proxyTarget))
 633         {
 634             delete c;
 635             return 0;
 636         }
 637         c->m_target_idletime = m_target_idletime;
 638         c->timeout(30);
 639
 640         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 641             c->set_APDU_yazlog(1);
 642         else
 643             c->set_APDU_yazlog(0);
 644     }
 645     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 646     return c;
 647 }
 648
 649 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 650 {
 651     int i;
 652     for (i = 0; i<num; i++)
 653     {
 654         oident *ent;
 655         Z_DefaultDiagFormat *r;
 656         Z_DiagRec *p = pp[i];
 657         if (p->which != Z_DiagRec_defaultFormat)
 658         {
 659             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 660             return;
 661         }
 662         else
 663             r = p->u.defaultFormat;
 664         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 665             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 666             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 667         switch (r->which)
 668         {
 669         case Z_DefaultDiagFormat_v2Addinfo:
 670             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 671                     m_session_str,
 672                     *r->condition, diagbib1_str(*r->condition),
 673                     r->u.v2Addinfo);
 674             break;
 675         case Z_DefaultDiagFormat_v3Addinfo:
 676             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 677                     m_session_str,
 678                     *r->condition, diagbib1_str(*r->condition),
 679                     r->u.v3Addinfo);
 680             break;
 681         }
 682     }
 683 }
 684
 685 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 686 {
 687     if (!m_stylesheet_xsp || p->num_records <= 0)
 688         return 0;  /* no XSLT to be done ... */
 689
 690     m_stylesheet_offset = 0;
 691     m_stylesheet_nprl = p;
 692     m_stylesheet_apdu = apdu;
 693     timeout(0);
 694     return 1;
 695 }
 696
 697 void Yaz_Proxy::convert_xsl_delay()
 698 {
 699 #if HAVE_XSLT
 700     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 701     if (npr->which == Z_NamePlusRecord_databaseRecord)
 702     {
 703         Z_External *r = npr->u.databaseRecord;
 704         if (r->which == Z_External_octet)
 705         {
 706 #if 0
 707             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 708 #endif
 709             xmlDocPtr res, doc = xmlParseMemory(
 710                 (char*) r->u.octet_aligned->buf,
 711                 r->u.octet_aligned->len);
 712
 713
 714             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 715                     m_session_str, m_stylesheet_offset);
 716             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 717                                       doc, 0);
 718
 719             if (res)
 720             {
 721                 xmlChar *out_buf;
 722                 int out_len;
 723                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 724
 725                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 726                     u.databaseRecord =
 727                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 728                                  (char*) out_buf, out_len);
 729                 xmlFree(out_buf);
 730                 xmlFreeDoc(res);
 731             }
 732
 733             xmlFreeDoc(doc);
 734         }
 735     }
 736 #endif
 737     m_stylesheet_offset++;
 738     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 739     {
 740         m_stylesheet_nprl = 0;
 741 #if HAVE_XSLT
 742         if (m_stylesheet_xsp)
 743             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 744 #endif
 745         m_stylesheet_xsp = 0;
 746         timeout(m_client_idletime);
 747         send_PDU_convert(m_stylesheet_apdu);
 748     }
 749     else
 750         timeout(0);
 751 }
 752
 753 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 754 {
 755     if (m_frontend_type != VAL_NONE)
 756     {
 757         int i;
 758         for (i = 0; i < p->num_records; i++)
 759         {
 760             Z_NamePlusRecord *npr = p->records[i];
 761             if (npr->which == Z_NamePlusRecord_databaseRecord)
 762             {
 763                 Z_External *r = npr->u.databaseRecord;
 764                 if (r->which == Z_External_octet)
 765                 {
 766 #if HAVE_USEMARCON
 767                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 768                     {
 769                         if (!m_usemarcon->m_stage1)
 770                         {
 771                             m_usemarcon->m_stage1 = new CDetails();
 772                         }
 773                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 774                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 775                         int res = m_usemarcon->m_stage1->Start();
 776                         if (res == 0)
 777                         {
 778                             char *converted;
 779                             int convlen;
 780                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 781                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 782                             {
 783                                 if (!m_usemarcon->m_stage2)
 784                                 {
 785                                     m_usemarcon->m_stage2 = new CDetails();
 786                                 }
 787                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 788                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 789                                 res = m_usemarcon->m_stage2->Start();
 790                                 if (res == 0)
 791                                 {
 792                                     free(converted);
 793                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 794                                 }
 795                                 else
 796                                 {
 797                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 798                                 }
 799                             }
 800                             npr->u.databaseRecord =
 801                                 z_ext_record(odr_encode(),
 802                                              m_frontend_type,
 803                                              converted,
 804                                              strlen(converted));
 805                             free(converted);
 806                         }
 807                         else
 808                         {
 809                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 810                         }
 811                         continue;
 812                     }
 813 #endif
 814                     npr->u.databaseRecord =
 815                         z_ext_record(odr_encode(),
 816                                      m_frontend_type,
 817                                      (char*) r->u.octet_aligned->buf,
 818                                      r->u.octet_aligned->len);
 819                 }
 820             }
 821         }
 822     }
 823 }
 824
 825 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
 826                                    const char *backend_charset)
 827 {
 828     int i;
 829
 830     if (!backend_charset)
 831         backend_charset = "MARC-8";
 832     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
 833     yaz_marc_t mt = yaz_marc_create();
 834     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 835     yaz_marc_iconv(mt, cd);
 836     for (i = 0; i < p->num_records; i++)
 837     {
 838         Z_NamePlusRecord *npr = p->records[i];
 839         if (npr->which == Z_NamePlusRecord_databaseRecord)
 840         {
 841             Z_External *r = npr->u.databaseRecord;
 842             if (r->which == Z_External_octet)
 843             {
 844                 int rlen;
 845                 char *result;
 846                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 847                                         r->u.octet_aligned->len,
 848                                         &result, &rlen))
 849                 {
 850                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 851                                                          VAL_TEXT_XML,
 852                                                          result, rlen);
 853                 }
 854             }
 855         }
 856     }
 857     if (cd)
 858         yaz_iconv_close(cd);
 859     yaz_marc_destroy(mt);
 860 }
 861
 862 void Yaz_Proxy::logtime()
 863 {
 864 #if HAVE_GETTIMEOFDAY
 865     struct timeval *tv = (struct timeval*) m_time_tv;
 866     if (tv->tv_sec)
 867     {
 868         struct timeval tv1;
 869         gettimeofday(&tv1, 0);
 870         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
 871             (tv1.tv_usec - tv->tv_usec);
 872         if (diff >= 0)
 873             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 874                     diff/1000000, (diff/1000)%1000);
 875     }
 876     tv->tv_sec = 0;
 877     tv->tv_usec = 0;
 878 #endif
 879 }
 880
 881 int Yaz_Proxy::send_http_response(int code)
 882 {
 883     ODR o = odr_encode();
 884     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 885     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 886     if (m_http_version)
 887         hres->version = odr_strdup(o, m_http_version);
 888     if (m_http_keepalive)
 889         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 890     else
 891         timeout(0);
 892
 893     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 894     {
 895         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
 896                  gdu_name(gdu));
 897     }
 898     int len;
 899     int r = send_GDU(gdu, &len);
 900     m_bytes_sent += len;
 901     m_bw_stat.add_bytes(len);
 902     logtime();
 903     return r;
 904 }
 905
 906 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 907 {
 908     ODR o = odr_encode();
 909     const char *ctype = "text/xml";
 910     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 911     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 912     if (m_http_version)
 913         hres->version = odr_strdup(o, m_http_version);
 914     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 915     if (m_http_keepalive)
 916         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 917     else
 918         timeout(0);
 919
 920     static Z_SOAP_Handler soap_handlers[2] = {
 921 #if HAVE_XSLT
 922         {"http://www.loc.gov/zing/srw/", 0,
 923          (Z_SOAP_fun) yaz_srw_codec},
 924 #endif
 925         {0, 0, 0}
 926     };
 927
 928     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 929     soap_package->which = Z_SOAP_generic;
 930     soap_package->u.generic =
 931         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 932     soap_package->u.generic->no = 0;
 933     soap_package->u.generic->ns = soap_handlers[0].ns;
 934     soap_package->u.generic->p = (void *) srw_pdu;
 935     soap_package->ns = m_soap_ns;
 936     z_soap_codec_enc_xsl(o, &soap_package,
 937                          &hres->content_buf, &hres->content_len,
 938                          soap_handlers, 0, m_s2z_stylesheet);
 939     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 940     {
 941         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
 942                  gdu_name(gdu));
 943     }
 944     int len;
 945     int r = send_GDU(gdu, &len);
 946     m_bytes_sent += len;
 947     m_bw_stat.add_bytes(len);
 948     logtime();
 949     return r;
 950 }
 951
 952 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
 953 {
 954     ODR o = odr_encode();
 955     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 956     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 957
 958     srw_res->num_diagnostics = 1;
 959     srw_res->diagnostics = (Z_SRW_diagnostic *)
 960         odr_malloc(o, sizeof(*srw_res->diagnostics));
 961     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
 962     return send_srw_response(srw_pdu);
 963 }
 964
 965 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 966                              Z_DefaultDiagFormat *ddf)
 967 {
 968     int bib1_code = *ddf->condition;
 969     if (bib1_code == 109)
 970         return 404;
 971     srw_res->num_diagnostics = 1;
 972     srw_res->diagnostics = (Z_SRW_diagnostic *)
 973         odr_malloc(o, sizeof(*srw_res->diagnostics));
 974     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
 975                           yaz_diag_bib1_to_srw(*ddf->condition),
 976                           ddf->u.v2Addinfo);
 977     return 0;
 978 }
 979
 980 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
 981 {
 982     ODR o = odr_encode();
 983     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 984     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 985
 986     srw_res->numberOfRecords = odr_intdup (o, hits);
 987     if (records && records->which == Z_Records_DBOSD)
 988     {
 989         srw_res->num_records =
 990             records->u.databaseOrSurDiagnostics->num_records;
 991         int i;
 992         srw_res->records = (Z_SRW_record *)
 993             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
 994         for (i = 0; i < srw_res->num_records; i++)
 995         {
 996             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
 997             if (npr->which != Z_NamePlusRecord_databaseRecord)
 998             {
 999                 srw_res->records[i].recordSchema = "diagnostic";
1000                 srw_res->records[i].recordPacking = m_s2z_packing;
1001                 srw_res->records[i].recordData_buf = "67";
1002                 srw_res->records[i].recordData_len = 2;
1003                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1004                 continue;
1005             }
1006             Z_External *r = npr->u.databaseRecord;
1007             oident *ent = oid_getentbyoid(r->direct_reference);
1008             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1009             {
1010                 srw_res->records[i].recordSchema = m_schema;
1011                 srw_res->records[i].recordPacking = m_s2z_packing;
1012                 srw_res->records[i].recordData_buf = (char*)
1013                     r->u.octet_aligned->buf;
1014                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1015                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1016             }
1017             else
1018             {
1019                 srw_res->records[i].recordSchema = "diagnostic";
1020                 srw_res->records[i].recordPacking = m_s2z_packing;
1021                 srw_res->records[i].recordData_buf = "67";
1022                 srw_res->records[i].recordData_len = 2;
1023                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1024             }
1025         }
1026     }
1027     if (records && records->which == Z_Records_NSD)
1028     {
1029         int http_code;
1030         http_code = z_to_srw_diag(odr_encode(), srw_res,
1031                                    records->u.nonSurrogateDiagnostic);
1032         if (http_code)
1033             return send_http_response(http_code);
1034     }
1035     return send_srw_response(srw_pdu);
1036
1037 }
1038
1039 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1040                                         int num_diagnostics)
1041 {
1042     Yaz_ProxyConfig *cfg = check_reconfigure();
1043     if (cfg)
1044     {
1045         int len;
1046         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1047                                        m_s2z_database, &len);
1048         if (b)
1049         {
1050             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1051             Z_SRW_explainResponse *er = res->u.explain_response;
1052
1053             er->record.recordData_buf = b;
1054             er->record.recordData_len = len;
1055             er->record.recordPacking = m_s2z_packing;
1056             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1057
1058             er->diagnostics = diagnostics;
1059             er->num_diagnostics = num_diagnostics;
1060             return send_srw_response(res);
1061         }
1062     }
1063     return send_http_response(404);
1064 }
1065
1066 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1067 {
1068     if (m_http_version)
1069     {
1070         if (apdu->which == Z_APDU_initResponse)
1071         {
1072             Z_InitResponse *res = apdu->u.initResponse;
1073             if (*res->result == 0)
1074             {
1075                 send_to_srw_client_error(3, 0);
1076             }
1077             else if (!m_s2z_search_apdu)
1078             {
1079                 send_srw_explain_response(0, 0);
1080             }
1081             else
1082             {
1083                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1084             }
1085         }
1086         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1087         {
1088             m_s2z_search_apdu = 0;
1089             Z_SearchResponse *res = apdu->u.searchResponse;
1090             m_s2z_hit_count = *res->resultCount;
1091             if (res->records && res->records->which == Z_Records_NSD)
1092             {
1093                 send_to_srw_client_ok(0, res->records, 1);
1094             }
1095             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1096             {
1097                 // adjust
1098                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1099
1100                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1101                 {
1102                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1103                         > m_s2z_hit_count)
1104                         *pr->numberOfRecordsRequested =
1105                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1106                 }
1107                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1108             }
1109             else
1110             {
1111                 m_s2z_present_apdu = 0;
1112                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1113             }
1114         }
1115         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1116         {
1117             int start =
1118                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1119
1120             m_s2z_present_apdu = 0;
1121             Z_PresentResponse *res = apdu->u.presentResponse;
1122             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1123         }
1124     }
1125     else
1126     {
1127         int len = 0;
1128         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1129             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1130                      apdu_name(apdu));
1131         int r = send_Z_PDU(apdu, &len);
1132         m_bytes_sent += len;
1133         m_bw_stat.add_bytes(len);
1134         logtime();
1135         return r;
1136     }
1137     return 0;
1138 }
1139
1140 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1141 {
1142     int kill_session = 0;
1143     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1144
1145     if (new_id)
1146         *new_id = m_referenceId;
1147
1148     if (apdu->which == Z_APDU_searchResponse)
1149     {
1150         Z_SearchResponse *sr = apdu->u.searchResponse;
1151         Z_Records *p = sr->records;
1152         if (p && p->which == Z_Records_NSD)
1153         {
1154             Z_DiagRec dr, *dr_p = &dr;
1155             dr.which = Z_DiagRec_defaultFormat;
1156             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1157
1158             *sr->searchStatus = 0;
1159             display_diagrecs(&dr_p, 1);
1160         }
1161         else
1162         {
1163             if (p && p->which == Z_Records_DBOSD)
1164             {
1165                 if (m_backend_type
1166 #if HAVE_USEMARCON
1167                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1168 #endif
1169                     )
1170                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1171                 if (m_marcxml_flag)
1172                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1173                                        m_backend_charset);
1174                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1175                     return 0;
1176
1177             }
1178             if (sr->resultCount)
1179             {
1180                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1181                         *sr->resultCount);
1182                 if (*sr->resultCount < 0)
1183                 {
1184                     m_invalid_session = 1;
1185                     kill_session = 1;
1186
1187                     *sr->searchStatus = 0;
1188                     sr->records =
1189                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1190                     *sr->resultCount = 0;
1191                 }
1192             }
1193         }
1194     }
1195     else if (apdu->which == Z_APDU_presentResponse)
1196     {
1197         Z_PresentResponse *sr = apdu->u.presentResponse;
1198         Z_Records *p = sr->records;
1199         if (p && p->which == Z_Records_NSD)
1200         {
1201             Z_DiagRec dr, *dr_p = &dr;
1202             dr.which = Z_DiagRec_defaultFormat;
1203             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1204             if (*sr->presentStatus == Z_PresentStatus_success)
1205                 *sr->presentStatus = Z_PresentStatus_failure;
1206             display_diagrecs(&dr_p, 1);
1207         }
1208         if (p && p->which == Z_Records_DBOSD)
1209         {
1210             if (m_backend_type
1211 #if HAVE_USEMARCON
1212                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1213 #endif
1214                 )
1215                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1216             if (m_marcxml_flag)
1217                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1218                                    m_backend_charset);
1219             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1220                 return 0;
1221         }
1222     }
1223     else if (apdu->which == Z_APDU_initResponse)
1224     {
1225         if (m_initRequest_options)
1226         {
1227             Z_Options *nopt =
1228                 (Odr_bitmask *)odr_malloc(odr_encode(),
1229                                           sizeof(Odr_bitmask));
1230             ODR_MASK_ZERO(nopt);
1231
1232             int i;
1233             for (i = 0; i<24; i++)
1234                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1235                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1236                     ODR_MASK_SET(nopt, i);
1237             apdu->u.initResponse->options = nopt;
1238         }
1239         if (m_initRequest_version)
1240         {
1241             Z_ProtocolVersion *nopt =
1242                 (Odr_bitmask *)odr_malloc(odr_encode(),
1243                                           sizeof(Odr_bitmask));
1244             ODR_MASK_ZERO(nopt);
1245
1246             int i;
1247             for (i = 0; i<8; i++)
1248                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1249                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1250                     ODR_MASK_SET(nopt, i);
1251             apdu->u.initResponse->protocolVersion = nopt;
1252         }
1253         apdu->u.initResponse->preferredMessageSize =
1254             odr_intdup(odr_encode(),
1255                        m_client->m_initResponse_preferredMessageSize >
1256                        m_initRequest_preferredMessageSize ?
1257                        m_initRequest_preferredMessageSize :
1258                        m_client->m_initResponse_preferredMessageSize);
1259         apdu->u.initResponse->maximumRecordSize =
1260             odr_intdup(odr_encode(),
1261                        m_client->m_initResponse_maximumRecordSize >
1262                        m_initRequest_maximumRecordSize ?
1263                        m_initRequest_maximumRecordSize :
1264                        m_client->m_initResponse_maximumRecordSize);
1265     }
1266     int r = send_PDU_convert(apdu);
1267     if (r)
1268         return r;
1269     if (kill_session)
1270     {
1271         delete m_client;
1272         m_client = 0;
1273         m_parent->pre_init();
1274     }
1275     return r;
1276 }
1277
1278 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1279 {
1280     int len = 0;
1281     const char *apdu_name_tmp = apdu_name(apdu);
1282     int r = send_Z_PDU(apdu, &len);
1283     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1284         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1285                  get_session_str(),
1286                  apdu_name_tmp, get_hostname(), len);
1287     m_bytes_sent += len;
1288     return r;
1289 }
1290
1291 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1292 {
1293     if (apdu->which == Z_APDU_presentRequest)
1294     {
1295         Z_PresentRequest *pr = apdu->u.presentRequest;
1296         int toget = *pr->numberOfRecordsRequested;
1297         int start = *pr->resultSetStartPoint;
1298
1299         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1300                 pr->resultSetId, start, toget);
1301
1302         if (*m_parent->m_optimize == '0')
1303             return apdu;
1304
1305         if (!m_client->m_last_resultSetId)
1306         {
1307             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1308             new_apdu->u.presentResponse->records =
1309                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1310                                                pr->resultSetId);
1311             send_to_client(new_apdu);
1312             return 0;
1313         }
1314         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1315         {
1316             if (start+toget-1 > m_client->m_last_resultCount)
1317             {
1318                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1319                 new_apdu->u.presentResponse->records =
1320                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1321                 send_to_client(new_apdu);
1322                 return 0;
1323             }
1324             Z_NamePlusRecordList *npr;
1325 #if 0
1326             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1327                     m_session_str, start, toget, yaz_z3950oid_to_str(
1328                         pr->preferredRecordSyntax, &oclass));
1329 #endif
1330             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1331                                           pr->preferredRecordSyntax,
1332                                           pr->recordComposition))
1333             {
1334                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1335                          m_session_str);
1336                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1337                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1338
1339                 new_apdu->u.presentResponse->numberOfRecordsReturned
1340                     = odr_intdup(odr_encode(), toget);
1341
1342                 new_apdu->u.presentResponse->records = (Z_Records*)
1343                     odr_malloc(odr_encode(), sizeof(Z_Records));
1344                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1345                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1346                 new_apdu->u.presentResponse->nextResultSetPosition =
1347                     odr_intdup(odr_encode(), start+toget);
1348
1349                 send_to_client(new_apdu);
1350                 return 0;
1351             }
1352         }
1353     }
1354
1355     if (apdu->which != Z_APDU_searchRequest)
1356         return apdu;
1357     Z_SearchRequest *sr = apdu->u.searchRequest;
1358     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1359     Yaz_Z_Databases this_databases;
1360
1361     this_databases.set(sr->num_databaseNames, (const char **)
1362                        sr->databaseNames);
1363
1364     this_query->set_Z_Query(sr->query);
1365
1366     char query_str[120];
1367     this_query->print(query_str, sizeof(query_str)-1);
1368     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1369
1370     if (*m_parent->m_optimize != '0' &&
1371         m_client->m_last_ok && m_client->m_last_query &&
1372         m_client->m_last_query->match(this_query) &&
1373         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1374         m_client->m_last_databases.match(this_databases))
1375     {
1376         delete this_query;
1377         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1378             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1379         {
1380             Z_NamePlusRecordList *npr;
1381             int toget = *sr->mediumSetPresentNumber;
1382             Z_RecordComposition *comp = 0;
1383
1384             if (toget > m_client->m_last_resultCount)
1385                 toget = m_client->m_last_resultCount;
1386
1387             if (sr->mediumSetElementSetNames)
1388             {
1389                 comp = (Z_RecordComposition *)
1390                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1391                 comp->which = Z_RecordComp_simple;
1392                 comp->u.simple = sr->mediumSetElementSetNames;
1393             }
1394
1395             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1396                                           sr->preferredRecordSyntax, comp))
1397             {
1398                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1399                          m_session_str);
1400                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1401                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1402                 new_apdu->u.searchResponse->resultCount =
1403                     &m_client->m_last_resultCount;
1404
1405                 new_apdu->u.searchResponse->numberOfRecordsReturned
1406                     = odr_intdup(odr_encode(), toget);
1407
1408                 new_apdu->u.searchResponse->presentStatus =
1409                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1410                 new_apdu->u.searchResponse->records = (Z_Records*)
1411                     odr_malloc(odr_encode(), sizeof(Z_Records));
1412                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1413                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1414                 new_apdu->u.searchResponse->nextResultSetPosition =
1415                     odr_intdup(odr_encode(), toget+1);
1416                 send_to_client(new_apdu);
1417                 return 0;
1418             }
1419             else
1420             {
1421                 // medium Set
1422                 // send present request (medium size)
1423                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1424                          m_session_str);
1425
1426                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1427                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1428                 pr->referenceId = sr->referenceId;
1429                 pr->resultSetId = sr->resultSetName;
1430                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1431                 *pr->numberOfRecordsRequested = toget;
1432                 pr->recordComposition = comp;
1433                 m_client->m_sr_transform = 1;
1434                 return new_apdu;
1435             }
1436         }
1437         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1438             m_client->m_last_resultCount <= 0)
1439         {
1440             // large set. Return pseudo-search response immediately
1441             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1442                      m_session_str);
1443             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1444             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1445             new_apdu->u.searchResponse->resultCount =
1446                 &m_client->m_last_resultCount;
1447             send_to_client(new_apdu);
1448             return 0;
1449         }
1450         else
1451         {
1452             Z_NamePlusRecordList *npr;
1453             int toget = m_client->m_last_resultCount;
1454             Z_RecordComposition *comp = 0;
1455             // small set
1456             // send a present request (small set)
1457
1458             if (sr->smallSetElementSetNames)
1459             {
1460                 comp = (Z_RecordComposition *)
1461                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1462                 comp->which = Z_RecordComp_simple;
1463                 comp->u.simple = sr->smallSetElementSetNames;
1464             }
1465
1466             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1467                                           sr->preferredRecordSyntax, comp))
1468             {
1469                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1470                          m_session_str);
1471                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1472                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1473                 new_apdu->u.searchResponse->resultCount =
1474                     &m_client->m_last_resultCount;
1475
1476                 new_apdu->u.searchResponse->numberOfRecordsReturned
1477                     = odr_intdup(odr_encode(), toget);
1478
1479                 new_apdu->u.searchResponse->presentStatus =
1480                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1481                 new_apdu->u.searchResponse->records = (Z_Records*)
1482                     odr_malloc(odr_encode(), sizeof(Z_Records));
1483                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1484                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1485                 new_apdu->u.searchResponse->nextResultSetPosition =
1486                     odr_intdup(odr_encode(), toget+1);
1487                 send_to_client(new_apdu);
1488                 return 0;
1489             }
1490             else
1491             {
1492                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1493                          m_session_str);
1494                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1495                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1496                 pr->referenceId = sr->referenceId;
1497                 pr->resultSetId = sr->resultSetName;
1498                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1499                 *pr->numberOfRecordsRequested = toget;
1500                 pr->recordComposition = comp;
1501                 m_client->m_sr_transform = 1;
1502                 return new_apdu;
1503             }
1504         }
1505     }
1506     else  // query doesn't match
1507     {
1508         delete m_client->m_last_query;
1509         m_client->m_last_query = this_query;
1510         m_client->m_last_ok = 0;
1511         m_client->m_cache.clear();
1512         m_client->m_resultSetStartPoint = 0;
1513
1514         xfree (m_client->m_last_resultSetId);
1515         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1516
1517         m_client->m_last_databases.set(sr->num_databaseNames,
1518                                        (const char **) sr->databaseNames);
1519     }
1520     return apdu;
1521 }
1522
1523
1524 void Yaz_Proxy::inc_request_no()
1525 {
1526     char *cp = strchr(m_session_str, ' ');
1527     m_request_no++;
1528     if (cp)
1529         sprintf(cp+1, "%d ", m_request_no);
1530 }
1531
1532 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1533 {
1534     inc_request_no();
1535
1536     m_bytes_recv += len;
1537
1538     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1539         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1540                  m_session_str, gdu_name(apdu), len);
1541
1542     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1543         shutdown();
1544
1545     m_bw_stat.add_bytes(len);
1546     m_pdu_stat.add_bytes(1);
1547
1548 #if HAVE_GETTIMEOFDAY
1549     gettimeofday((struct timeval *) m_time_tv, 0);
1550 #endif
1551
1552     int bw_total = m_bw_stat.get_total();
1553     int pdu_total = m_pdu_stat.get_total();
1554
1555     int reduce = 0;
1556     if (m_bw_max)
1557     {
1558         if (bw_total > m_bw_max)
1559         {
1560             reduce = (bw_total/m_bw_max);
1561         }
1562     }
1563     if (m_pdu_max)
1564     {
1565         if (pdu_total > m_pdu_max)
1566         {
1567             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1568             reduce = (reduce > nreduce) ? reduce : nreduce;
1569         }
1570     }
1571     m_http_version = 0;
1572     if (reduce)
1573     {
1574         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1575                 m_session_str, reduce, bw_total, pdu_total,
1576                 m_bw_max, m_pdu_max);
1577
1578         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1579         timeout(reduce);       // call us reduce seconds later
1580     }
1581     else if (apdu->which == Z_GDU_Z3950)
1582         handle_incoming_Z_PDU(apdu->u.z3950);
1583     else if (apdu->which == Z_GDU_HTTP_Request)
1584         handle_incoming_HTTP(apdu->u.HTTP_Request);
1585 }
1586
1587 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1588 {
1589     if (m_max_record_retrieve)
1590     {
1591         if (apdu->which == Z_APDU_presentRequest)
1592         {
1593             Z_PresentRequest *pr = apdu->u.presentRequest;
1594             if (pr->numberOfRecordsRequested &&
1595                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1596                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1597         }
1598     }
1599 }
1600
1601 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1602                                                      int error,
1603                                                      const char *addinfo)
1604 {
1605     Z_Records *rec = (Z_Records *)
1606         odr_malloc (odr, sizeof(*rec));
1607     int *err = (int *)
1608         odr_malloc (odr, sizeof(*err));
1609     Z_DiagRec *drec = (Z_DiagRec *)
1610         odr_malloc (odr, sizeof(*drec));
1611     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1612         odr_malloc (odr, sizeof(*dr));
1613     *err = error;
1614     rec->which = Z_Records_NSD;
1615     rec->u.nonSurrogateDiagnostic = dr;
1616     dr->diagnosticSetId =
1617         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1618     dr->condition = err;
1619     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1620     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1621     return rec;
1622 }
1623
1624 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1625 {
1626     if (apdu->which == Z_APDU_searchRequest &&
1627         apdu->u.searchRequest->query &&
1628         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1629         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1630     {
1631         Z_RPNQuery *rpnquery = 0;
1632         Z_SearchRequest *sr = apdu->u.searchRequest;
1633         char *addinfo = 0;
1634
1635         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
1636                 sr->query->u.type_104->u.cql);
1637
1638         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1639                                           &rpnquery, odr_encode(),
1640                                           &addinfo);
1641         if (r == -3)
1642             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
1643         else if (r)
1644         {
1645             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1646             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1647
1648             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1649             new_apdu->u.searchResponse->records =
1650                 create_nonSurrogateDiagnostics(odr_encode(),
1651                                                yaz_diag_srw_to_bib1(r),
1652                                                addinfo);
1653             *new_apdu->u.searchResponse->searchStatus = 0;
1654
1655             send_to_client(new_apdu);
1656
1657             return 0;
1658         }
1659         else
1660         {
1661             sr->query->which = Z_Query_type_1;
1662             sr->query->u.type_1 = rpnquery;
1663         }
1664         return apdu;
1665     }
1666     return apdu;
1667 }
1668
1669 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1670 {
1671     if (apdu->which == Z_APDU_searchRequest)
1672     {
1673         Z_SearchRequest *sr = apdu->u.searchRequest;
1674         int err = 0;
1675         char *addinfo = 0;
1676
1677         Yaz_ProxyConfig *cfg = check_reconfigure();
1678         if (cfg)
1679             err = cfg->check_query(odr_encode(), m_default_target,
1680                                    sr->query, &addinfo);
1681         if (err)
1682         {
1683             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1684
1685             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1686             new_apdu->u.searchResponse->records =
1687                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1688             *new_apdu->u.searchResponse->searchStatus = 0;
1689
1690             send_to_client(new_apdu);
1691
1692             return 0;
1693         }
1694     }
1695     return apdu;
1696 }
1697
1698 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1699 {
1700     m_marcxml_flag = 0;
1701     if (apdu->which == Z_APDU_searchRequest)
1702     {
1703         Z_SearchRequest *sr = apdu->u.searchRequest;
1704         int err = 0;
1705         char *addinfo = 0;
1706         Yaz_ProxyConfig *cfg = check_reconfigure();
1707
1708         Z_RecordComposition rc_temp, *rc = 0;
1709         if (sr->smallSetElementSetNames)
1710         {
1711             rc_temp.which = Z_RecordComp_simple;
1712             rc_temp.u.simple = sr->smallSetElementSetNames;
1713             rc = &rc_temp;
1714         }
1715
1716         if (sr->preferredRecordSyntax)
1717         {
1718             struct oident *ent;
1719             ent = oid_getentbyoid(sr->preferredRecordSyntax);
1720             m_frontend_type = ent->value;
1721         }
1722         else
1723             m_frontend_type = VAL_NONE;
1724
1725         char *stylesheet_name = 0;
1726         if (cfg)
1727             err = cfg->check_syntax(odr_encode(),
1728                                     m_default_target,
1729                                     sr->preferredRecordSyntax, rc,
1730                                     &addinfo, &stylesheet_name, &m_schema,
1731                                     &m_backend_type, &m_backend_charset,
1732                                     &m_usemarcon_ini_stage1,
1733                                     &m_usemarcon_ini_stage2);
1734         if (stylesheet_name)
1735         {
1736             m_parent->low_socket_close();
1737
1738 #if HAVE_XSLT
1739             if (m_stylesheet_xsp)
1740                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
1741             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1742                                                        stylesheet_name);
1743 #endif
1744             m_stylesheet_offset = 0;
1745             xfree(stylesheet_name);
1746
1747             m_parent->low_socket_open();
1748         }
1749         if (err == -1)
1750         {
1751             sr->smallSetElementSetNames = 0;
1752             sr->mediumSetElementSetNames = 0;
1753             if (m_backend_type)
1754             {
1755
1756                 sr->preferredRecordSyntax =
1757                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
1758                                         m_backend_type);
1759             }
1760             else
1761                 sr->preferredRecordSyntax =
1762                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
1763                                            VAL_USMARC);
1764             m_marcxml_flag = 1;
1765         }
1766         else if (err)
1767         {
1768             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1769
1770             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1771             new_apdu->u.searchResponse->records =
1772                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1773             *new_apdu->u.searchResponse->searchStatus = 0;
1774
1775             send_to_client(new_apdu);
1776
1777             return 0;
1778         }
1779         else if (m_backend_type)
1780         {
1781             sr->preferredRecordSyntax =
1782                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
1783         }
1784     }
1785     else if (apdu->which == Z_APDU_presentRequest)
1786     {
1787         Z_PresentRequest *pr = apdu->u.presentRequest;
1788         int err = 0;
1789         char *addinfo = 0;
1790         Yaz_ProxyConfig *cfg = check_reconfigure();
1791
1792         if (pr->preferredRecordSyntax)
1793         {
1794             struct oident *ent;
1795             ent = oid_getentbyoid(pr->preferredRecordSyntax);
1796             m_frontend_type = ent->value;
1797         }
1798         else
1799             m_frontend_type = VAL_NONE;
1800
1801         char *stylesheet_name = 0;
1802         if (cfg)
1803             err = cfg->check_syntax(odr_encode(), m_default_target,
1804                                     pr->preferredRecordSyntax,
1805                                     pr->recordComposition,
1806                                     &addinfo, &stylesheet_name, &m_schema,
1807                                     &m_backend_type, &m_backend_charset,
1808                                     &m_usemarcon_ini_stage1,
1809                                     &m_usemarcon_ini_stage2
1810                                     );
1811         if (stylesheet_name)
1812         {
1813             m_parent->low_socket_close();
1814
1815 #if HAVE_XSLT
1816             if (m_stylesheet_xsp)
1817                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
1818             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1819                                                        stylesheet_name);
1820 #endif
1821             m_stylesheet_offset = 0;
1822             xfree(stylesheet_name);
1823
1824             m_parent->low_socket_open();
1825         }
1826         if (err == -1)
1827         {
1828             pr->recordComposition = 0;
1829             if (m_backend_type)
1830             {
1831
1832                 pr->preferredRecordSyntax =
1833                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
1834                                         m_backend_type);
1835             }
1836             else
1837                 pr->preferredRecordSyntax =
1838                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
1839                                            VAL_USMARC);
1840             m_marcxml_flag = 1;
1841         }
1842         else if (err)
1843         {
1844             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1845
1846             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1847             new_apdu->u.presentResponse->records =
1848                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1849             *new_apdu->u.presentResponse->presentStatus =
1850                 Z_PresentStatus_failure;
1851
1852             send_to_client(new_apdu);
1853
1854             return 0;
1855         }
1856         else if (m_backend_type)
1857         {
1858             pr->preferredRecordSyntax =
1859                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
1860         }
1861     }
1862     return apdu;
1863 }
1864
1865 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
1866 {
1867     if (!schema)
1868         return 0;
1869     Z_ElementSetNames *esn = (Z_ElementSetNames *)
1870         odr_malloc(o, sizeof(Z_ElementSetNames));
1871     esn->which = Z_ElementSetNames_generic;
1872     esn->u.generic = odr_strdup(o, schema);
1873     return esn;
1874 }
1875
1876 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
1877 {
1878     const char *t = 0;
1879     Yaz_ProxyConfig *cfg = check_reconfigure();
1880     if (cfg)
1881         t = cfg->get_explain_name(db, backend_db);
1882
1883     if (m_client && m_default_target && t && strcmp(m_default_target, t))
1884     {
1885         releaseClient();
1886     }
1887
1888     if (t)
1889     {
1890         xfree(m_default_target);
1891         m_default_target = xstrdup(t);
1892     }
1893 }
1894
1895 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
1896 {
1897     struct stat sbuf;
1898     yaz_log(YLOG_LOG, "file_access");
1899     if (strcmp(hreq->method, "GET"))
1900         return 0;
1901     if (hreq->path[0] != '/')
1902     {
1903         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
1904         return 0;
1905     }
1906     const char *cp = hreq->path;
1907     while (*cp)
1908     {
1909         if (*cp == '/' && strchr("/.", cp[1]))
1910         {
1911             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
1912             return 0;
1913         }
1914         cp++;
1915     }
1916     const char *fname = hreq->path+1;
1917     if (stat(fname, &sbuf))
1918     {
1919         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
1920         return 0;
1921     }
1922     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
1923     {
1924         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
1925         return 0;
1926     }
1927     if (sbuf.st_size > (off_t) 1000000)
1928     {
1929         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
1930         return 0;
1931     }
1932
1933     ODR o = odr_encode();
1934     Yaz_ProxyConfig *cfg = check_reconfigure();
1935     const char *ctype = cfg->check_mime_type(fname);
1936     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
1937     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1938     if (m_http_version)
1939         hres->version = odr_strdup(o, m_http_version);
1940     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1941     if (m_http_keepalive)
1942         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1943     else
1944         timeout(0);
1945
1946     hres->content_len = sbuf.st_size;
1947     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
1948     FILE *f = fopen(fname, "rb");
1949     if (f)
1950     {
1951         fread(hres->content_buf, 1, hres->content_len, f);
1952         fclose(f);
1953     }
1954     else
1955     {
1956         return 0;
1957     }
1958     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1959     {
1960         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
1961                  fname);
1962     }
1963     int len;
1964     send_GDU(gdu, &len);
1965     return 1;
1966 }
1967
1968 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1969 {
1970     if (m_s2z_odr_init)
1971     {
1972         odr_destroy(m_s2z_odr_init);
1973         m_s2z_odr_init = 0;
1974     }
1975     if (m_s2z_odr_search)
1976     {
1977         odr_destroy(m_s2z_odr_search);
1978         m_s2z_odr_search = 0;
1979     }
1980
1981     m_http_keepalive = 0;
1982     m_http_version = 0;
1983     if (!strcmp(hreq->version, "1.0"))
1984     {
1985         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1986         if (v && !strcmp(v, "Keep-Alive"))
1987             m_http_keepalive = 1;
1988         else
1989             m_http_keepalive = 0;
1990         m_http_version = "1.0";
1991     }
1992     else
1993     {
1994         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1995         if (v && !strcmp(v, "close"))
1996             m_http_keepalive = 0;
1997         else
1998             m_http_keepalive = 1;
1999         m_http_version = "1.1";
2000     }
2001
2002     Z_SRW_PDU *srw_pdu = 0;
2003     Z_SOAP *soap_package = 0;
2004     char *charset = 0;
2005     Z_SRW_diagnostic *diagnostic = 0;
2006     int num_diagnostic = 0;
2007
2008     if (file_access(hreq))
2009     {
2010         return;
2011     }
2012     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2013                             &charset) == 0
2014              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2015                                &charset, &diagnostic, &num_diagnostic) == 0)
2016     {
2017         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2018         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2019         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2020         m_s2z_init_apdu = 0;
2021         m_s2z_search_apdu = 0;
2022         m_s2z_present_apdu = 0;
2023
2024         m_s2z_stylesheet = 0;
2025
2026         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2027         {
2028             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2029
2030             const char *backend_db = srw_req->database;
2031             srw_get_client(srw_req->database, &backend_db);
2032
2033             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2034             // recordXPath unsupported.
2035             if (srw_req->recordXPath)
2036             {
2037                 yaz_add_srw_diagnostic(odr_decode(),
2038                                        &diagnostic, &num_diagnostic,
2039                                        72, 0);
2040             }
2041             // sort unsupported
2042             if (srw_req->sort_type != Z_SRW_sort_type_none)
2043             {
2044                 yaz_add_srw_diagnostic(odr_decode(),
2045                                        &diagnostic, &num_diagnostic,
2046                                        80, 0);
2047             }
2048             // save stylesheet
2049             if (srw_req->stylesheet)
2050                 m_s2z_stylesheet =
2051                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2052
2053             // set packing for response records ..
2054             if (srw_req->recordPacking &&
2055                 !strcmp(srw_req->recordPacking, "xml"))
2056                 m_s2z_packing = Z_SRW_recordPacking_XML;
2057             else
2058                 m_s2z_packing = Z_SRW_recordPacking_string;
2059
2060             if (num_diagnostic)
2061             {
2062                 Z_SRW_PDU *srw_pdu =
2063                     yaz_srw_get(odr_encode(),
2064                                 Z_SRW_searchRetrieve_response);
2065                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2066
2067                 srw_res->diagnostics = diagnostic;
2068                 srw_res->num_diagnostics = num_diagnostic;
2069                 send_srw_response(srw_pdu);
2070                 return;
2071             }
2072
2073             // prepare search PDU
2074             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2075                                           Z_APDU_searchRequest);
2076             Z_SearchRequest *z_searchRequest =
2077                 m_s2z_search_apdu->u.searchRequest;
2078
2079             z_searchRequest->num_databaseNames = 1;
2080             z_searchRequest->databaseNames = (char**)
2081                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2082             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2083                                                            backend_db);
2084
2085             // query transformation
2086             Z_Query *query = (Z_Query *)
2087                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2088             z_searchRequest->query = query;
2089
2090             if (srw_req->query_type == Z_SRW_query_type_cql)
2091             {
2092                 Z_External *ext = (Z_External *)
2093                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2094                 ext->direct_reference =
2095                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2096                 ext->indirect_reference = 0;
2097                 ext->descriptor = 0;
2098                 ext->which = Z_External_CQL;
2099                 ext->u.cql = srw_req->query.cql;
2100
2101                 query->which = Z_Query_type_104;
2102                 query->u.type_104 =  ext;
2103             }
2104             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2105             {
2106                 Z_RPNQuery *RPNquery;
2107                 YAZ_PQF_Parser pqf_parser;
2108
2109                 pqf_parser = yaz_pqf_create ();
2110
2111                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2112                                           srw_req->query.pqf);
2113                 if (!RPNquery)
2114                 {
2115                     const char *pqf_msg;
2116                     size_t off;
2117                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2118                     yaz_log(YLOG_LOG, "%*s^\n", off+4, "");
2119                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2120
2121                     send_to_srw_client_error(10, 0);
2122                     return;
2123                 }
2124                 query->which = Z_Query_type_1;
2125                 query->u.type_1 =  RPNquery;
2126
2127                 yaz_pqf_destroy (pqf_parser);
2128             }
2129             else
2130             {
2131                 send_to_srw_client_error(7, "query");
2132                 return;
2133             }
2134
2135             // present
2136             m_s2z_present_apdu = 0;
2137             int max = 0;
2138             if (srw_req->maximumRecords)
2139                 max = *srw_req->maximumRecords;
2140             int start = 1;
2141             if (srw_req->startRecord)
2142                 start = *srw_req->startRecord;
2143             if (max > 0)
2144             {
2145                 // Some backend, such as Voyager doesn't honor piggyback
2146                 // So we use present always (0 &&).
2147                 if (0 && start <= 1)  // Z39.50 piggyback
2148                 {
2149                     *z_searchRequest->smallSetUpperBound = max;
2150                     *z_searchRequest->mediumSetPresentNumber = max;
2151                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2152
2153                     z_searchRequest->preferredRecordSyntax =
2154                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2155                                                VAL_TEXT_XML);
2156                     if (srw_req->recordSchema)
2157                     {
2158                         z_searchRequest->smallSetElementSetNames =
2159                             z_searchRequest->mediumSetElementSetNames =
2160                             mk_esn_from_schema(m_s2z_odr_search,
2161                                                srw_req->recordSchema);
2162                     }
2163                 }
2164                 else   // Z39.50 present
2165                 {
2166                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2167                                                    Z_APDU_presentRequest);
2168                     Z_PresentRequest *z_presentRequest =
2169                         m_s2z_present_apdu->u.presentRequest;
2170                     *z_presentRequest->resultSetStartPoint = start;
2171                     *z_presentRequest->numberOfRecordsRequested = max;
2172                     z_presentRequest->preferredRecordSyntax =
2173                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2174                                                VAL_TEXT_XML);
2175                     if (srw_req->recordSchema)
2176                     {
2177                         z_presentRequest->recordComposition =
2178                             (Z_RecordComposition *)
2179                             odr_malloc(m_s2z_odr_search,
2180                                        sizeof(Z_RecordComposition));
2181                         z_presentRequest->recordComposition->which =
2182                             Z_RecordComp_simple;
2183                         z_presentRequest->recordComposition->u.simple =
2184                             mk_esn_from_schema(m_s2z_odr_search,
2185                                                srw_req->recordSchema);
2186                     }
2187                 }
2188             }
2189             if (!m_client)
2190             {
2191                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2192                                             Z_APDU_initRequest);
2193
2194                 // prevent m_initRequest_apdu memory from being grabbed
2195                 // in Yaz_Proxy::handle_incoming_Z_PDU
2196                 m_initRequest_apdu = m_s2z_init_apdu;
2197                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2198                 return;
2199             }
2200             else
2201             {
2202                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2203                 return;
2204             }
2205         }
2206         else if (srw_pdu->which == Z_SRW_explain_request)
2207         {
2208             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2209
2210             const char *backend_db = srw_req->database;
2211             srw_get_client(srw_req->database, &backend_db);
2212
2213             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2214
2215             // save stylesheet
2216             if (srw_req->stylesheet)
2217                 m_s2z_stylesheet =
2218                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2219
2220             if (srw_req->recordPacking &&
2221                 !strcmp(srw_req->recordPacking, "xml"))
2222                 m_s2z_packing = Z_SRW_recordPacking_XML;
2223             else
2224                 m_s2z_packing = Z_SRW_recordPacking_string;
2225
2226             if (num_diagnostic)
2227             {
2228                 send_srw_explain_response(diagnostic, num_diagnostic);
2229                 return;
2230             }
2231
2232             if (!m_client)
2233             {
2234                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2235                                             Z_APDU_initRequest);
2236
2237                 // prevent m_initRequest_apdu memory from being grabbed
2238                 // in Yaz_Proxy::handle_incoming_Z_PDU
2239                 m_initRequest_apdu = m_s2z_init_apdu;
2240                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2241             }
2242             else
2243                 send_srw_explain_response(0, 0);
2244             return;
2245         }
2246         else if (srw_pdu->which == Z_SRW_scan_request)
2247         {
2248             m_s2z_database = odr_strdup(m_s2z_odr_init,
2249                                         srw_pdu->u.scan_request->database);
2250
2251             yaz_add_srw_diagnostic(odr_decode(),
2252                                    &diagnostic, &num_diagnostic,
2253                                    4, "scan");
2254             Z_SRW_PDU *srw_pdu =
2255                 yaz_srw_get(odr_encode(),
2256                             Z_SRW_scan_response);
2257             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2258
2259             srw_res->diagnostics = diagnostic;
2260             srw_res->num_diagnostics = num_diagnostic;
2261             send_srw_response(srw_pdu);
2262             return;
2263         }
2264         else
2265         {
2266             m_s2z_database = 0;
2267
2268             send_to_srw_client_error(4, 0);
2269         }
2270     }
2271     send_http_response(400);
2272 }
2273
2274 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2275 {
2276     Z_ReferenceId **refid = get_referenceIdP(apdu);
2277     nmem_reset(m_referenceId_mem);
2278     if (refid && *refid)
2279     {
2280         m_referenceId = (Z_ReferenceId *)
2281             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2282         m_referenceId->len = m_referenceId->size = (*refid)->len;
2283         m_referenceId->buf = (unsigned char *)
2284             nmem_malloc(m_referenceId_mem, (*refid)->len);
2285         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2286     }
2287     else
2288         m_referenceId = 0;
2289
2290     if (!m_client && m_invalid_session)
2291     {
2292         m_apdu_invalid_session = apdu;
2293         m_mem_invalid_session = odr_extract_mem(odr_decode());
2294         apdu = m_initRequest_apdu;
2295     }
2296
2297     // Determine our client.
2298     Z_OtherInformation **oi;
2299     get_otherInfoAPDU(apdu, &oi);
2300     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2301     if (!m_client)
2302     {
2303         if (m_http_version)
2304         {
2305             send_http_response(404);
2306             return;
2307         }
2308         else
2309         {
2310             delete this;
2311             return;
2312         }
2313     }
2314     m_client->m_server = this;
2315
2316     if (apdu->which == Z_APDU_initRequest)
2317     {
2318         if (apdu->u.initRequest->implementationId)
2319             yaz_log(YLOG_LOG, "%simplementationId: %s",
2320                     m_session_str, apdu->u.initRequest->implementationId);
2321         if (apdu->u.initRequest->implementationName)
2322             yaz_log(YLOG_LOG, "%simplementationName: %s",
2323                     m_session_str, apdu->u.initRequest->implementationName);
2324         if (apdu->u.initRequest->implementationVersion)
2325             yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2326                     m_session_str, apdu->u.initRequest->implementationVersion);
2327         if (m_initRequest_apdu == 0)
2328         {
2329             if (m_initRequest_mem)
2330                 nmem_destroy(m_initRequest_mem);
2331             m_initRequest_apdu = apdu;
2332             m_initRequest_mem = odr_extract_mem(odr_decode());
2333
2334             m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2335                 preferredMessageSize;
2336             *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2337             m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2338                 maximumRecordSize;
2339             *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2340
2341             // save init options for the response..
2342             m_initRequest_options = apdu->u.initRequest->options;
2343
2344             apdu->u.initRequest->options =
2345                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2346                                            sizeof(Odr_bitmask));
2347             ODR_MASK_ZERO(apdu->u.initRequest->options);
2348             int i;
2349             for (i = 0; i<= 24; i++)
2350                 ODR_MASK_SET(apdu->u.initRequest->options, i);
2351             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2352                            Z_Options_negotiationModel);
2353             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2354                            Z_Options_concurrentOperations);
2355
2356             // make new version
2357             m_initRequest_version = apdu->u.initRequest->protocolVersion;
2358             apdu->u.initRequest->protocolVersion =
2359                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2360                                            sizeof(Odr_bitmask));
2361             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2362
2363             for (i = 0; i<= 8; i++)
2364                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2365         }
2366         if (m_client->m_init_flag)
2367         {
2368             if (handle_init_response_for_invalid_session(apdu))
2369                 return;
2370             if (m_client->m_initResponse)
2371             {
2372                 Z_APDU *apdu2 = m_client->m_initResponse;
2373                 apdu2->u.initResponse->otherInfo = 0;
2374                 if (m_client->m_cookie && *m_client->m_cookie)
2375                     set_otherInformationString(apdu2, VAL_COOKIE, 1,
2376                                                m_client->m_cookie);
2377                 apdu2->u.initResponse->referenceId =
2378                     apdu->u.initRequest->referenceId;
2379                 apdu2->u.initResponse->options = m_client->m_initResponse_options;
2380                 apdu2->u.initResponse->protocolVersion =
2381                     m_client->m_initResponse_version;
2382
2383                 send_to_client(apdu2);
2384                 return;
2385             }
2386         }
2387         m_client->m_init_flag = 1;
2388     }
2389     handle_max_record_retrieve(apdu);
2390
2391     if (apdu)
2392         apdu = handle_syntax_validation(apdu);
2393
2394     if (apdu)
2395         apdu = handle_query_transformation(apdu);
2396
2397     if (apdu)
2398         apdu = handle_query_validation(apdu);
2399
2400     if (apdu)
2401         apdu = result_set_optimize(apdu);
2402     if (!apdu)
2403     {
2404         m_client->timeout(m_target_idletime);  // mark it active even
2405         // though we didn't use it
2406         return;
2407     }
2408
2409     // delete other info construct completely if 0 elements
2410     get_otherInfoAPDU(apdu, &oi);
2411     if (oi && *oi && (*oi)->num_elements == 0)
2412         *oi = 0;
2413
2414     if (apdu->which == Z_APDU_presentRequest &&
2415         m_client->m_resultSetStartPoint == 0)
2416     {
2417         Z_PresentRequest *pr = apdu->u.presentRequest;
2418         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2419         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2420     } else {
2421         m_client->m_resultSetStartPoint = 0;
2422     }
2423     if (m_client->send_to_target(apdu) < 0)
2424     {
2425         delete m_client;
2426         m_client = 0;
2427         delete this;
2428     }
2429     else
2430         m_client->m_waiting = 1;
2431 }
2432
2433 void Yaz_Proxy::connectNotify()
2434 {
2435 }
2436
2437 void Yaz_Proxy::releaseClient()
2438 {
2439     xfree(m_proxyTarget);
2440     m_proxyTarget = 0;
2441     m_invalid_session = 0;
2442     // only keep if keep_alive flag is set...
2443     if (m_client &&
2444         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2445         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2446         m_client->m_waiting == 0)
2447     {
2448         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2449                  m_session_str,
2450                  m_client->get_hostname());
2451         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2452                 m_session_str, m_client->m_pdu_recv,
2453                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2454                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2455         assert (m_client->m_waiting != 2);
2456         // Tell client (if any) that no server connection is there..
2457         m_client->m_server = 0;
2458         m_client = 0;
2459     }
2460     else if (m_client)
2461     {
2462         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
2463                  m_session_str,
2464                  m_client->get_hostname());
2465         assert (m_client->m_waiting != 2);
2466         delete m_client;
2467         m_client = 0;
2468     }
2469     else if (!m_parent)
2470     {
2471         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
2472                  m_session_str);
2473         assert (m_parent);
2474     }
2475     else
2476     {
2477         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
2478                  m_session_str);
2479     }
2480     if (m_parent)
2481         m_parent->pre_init();
2482 }
2483
2484 void Yaz_Proxy::shutdown()
2485 {
2486     releaseClient();
2487     delete this;
2488 }
2489
2490 const char *Yaz_ProxyClient::get_session_str()
2491 {
2492     if (!m_server)
2493         return "0 ";
2494     return m_server->get_session_str();
2495 }
2496
2497 void Yaz_ProxyClient::shutdown()
2498 {
2499     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2500              get_hostname());
2501     delete m_server;
2502     delete this;
2503 }
2504
2505 void Yaz_Proxy::failNotify()
2506 {
2507     inc_request_no();
2508     yaz_log (YLOG_LOG, "%sConnection closed by client",
2509              get_session_str());
2510     shutdown();
2511 }
2512
2513 void Yaz_ProxyClient::failNotify()
2514 {
2515     if (m_server)
2516         m_server->inc_request_no();
2517     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
2518              get_session_str(), get_hostname());
2519     shutdown();
2520 }
2521
2522 void Yaz_ProxyClient::connectNotify()
2523 {
2524     const char *s = get_session_str();
2525     const char *h = get_hostname();
2526     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2527              m_target_idletime);
2528     timeout(m_target_idletime);
2529     if (!m_server)
2530         pre_init_client();
2531 }
2532
2533 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2534                                                   *the_PDU_Observable, int fd)
2535 {
2536     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2537 }
2538
2539 Yaz_ProxyClient::~Yaz_ProxyClient()
2540 {
2541     if (m_prev)
2542         *m_prev = m_next;
2543     if (m_next)
2544         m_next->m_prev = m_prev;
2545     m_waiting = 2;     // for debugging purposes only.
2546     odr_destroy(m_init_odr);
2547     delete m_last_query;
2548     xfree (m_last_resultSetId);
2549     xfree (m_cookie);
2550 }
2551
2552 void Yaz_ProxyClient::pre_init_client()
2553 {
2554     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2555     Z_InitRequest *req = apdu->u.initRequest;
2556
2557     int i;
2558     for (i = 0; i<= 24; i++)
2559         ODR_MASK_SET(req->options, i);
2560     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2561                    Z_Options_negotiationModel);
2562     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2563                    Z_Options_concurrentOperations);
2564     for (i = 0; i<= 10; i++)
2565         ODR_MASK_SET(req->protocolVersion, i);
2566
2567     if (send_to_target(apdu) < 0)
2568     {
2569         delete this;
2570     }
2571     else
2572     {
2573         m_waiting = 1;
2574         m_init_flag = 1;
2575     }
2576 }
2577
2578 void Yaz_Proxy::pre_init()
2579 {
2580     int i;
2581     const char *name = 0;
2582     const char *zurl_in_use[MAX_ZURL_PLEX];
2583     int limit_bw, limit_pdu, limit_req;
2584     int target_idletime, client_idletime;
2585     int max_clients;
2586     int keepalive_limit_bw, keepalive_limit_pdu;
2587     int pre_init;
2588     const char *cql2rpn = 0;
2589     const char *authentication = 0;
2590
2591     Yaz_ProxyConfig *cfg = check_reconfigure();
2592
2593     zurl_in_use[0] = 0;
2594
2595     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2596         set_APDU_yazlog(1);
2597     else
2598         set_APDU_yazlog(0);
2599
2600     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2601                                           &limit_bw, &limit_pdu, &limit_req,
2602                                           &target_idletime, &client_idletime,
2603                                           &max_clients,
2604                                           &keepalive_limit_bw,
2605                                           &keepalive_limit_pdu,
2606                                           &pre_init,
2607                                           &cql2rpn,
2608                                           &authentication) ; i++)
2609     {
2610         if (pre_init)
2611         {
2612             int j;
2613             for (j = 0; zurl_in_use[j]; j++)
2614             {
2615                 Yaz_ProxyClient *c;
2616                 int spare = 0;
2617                 int spare_waiting = 0;
2618                 int in_use = 0;
2619                 int other = 0;
2620                 for (c = m_clientPool; c; c = c->m_next)
2621                 {
2622                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
2623                     {
2624                         if (c->m_cookie == 0)
2625                         {
2626                             if (c->m_server == 0)
2627                                 if (c->m_waiting)
2628                                     spare_waiting++;
2629                                 else
2630                                     spare++;
2631                             else
2632                                 in_use++;
2633                         }
2634                         else
2635                             other++;
2636                     }
2637                 }
2638                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2639                         "sparew=%d preinit=%d",m_session_str,
2640                         name, zurl_in_use[j], in_use, other,
2641                         spare, spare_waiting, pre_init);
2642                 if (spare + spare_waiting < pre_init)
2643                 {
2644                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2645                     c->m_next = m_clientPool;
2646                     if (c->m_next)
2647                         c->m_next->m_prev = &c->m_next;
2648                     m_clientPool = c;
2649                     c->m_prev = &m_clientPool;
2650
2651                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
2652                         c->set_APDU_yazlog(1);
2653                     else
2654                         c->set_APDU_yazlog(0);
2655
2656                     if (c->client(zurl_in_use[j]))
2657                     {
2658                         timeout(60);
2659                         delete c;
2660                         return;
2661                     }
2662                     c->timeout(30);
2663                     c->m_waiting = 1;
2664                     c->m_target_idletime = target_idletime;
2665                     c->m_seqno = m_seqno++;
2666                 }
2667             }
2668         }
2669     }
2670 }
2671
2672 void Yaz_Proxy::timeoutNotify()
2673 {
2674     if (m_parent)
2675     {
2676         if (m_bw_hold_PDU)
2677         {
2678             timeout(m_client_idletime);
2679             Z_GDU *apdu = m_bw_hold_PDU;
2680             m_bw_hold_PDU = 0;
2681
2682             if (apdu->which == Z_GDU_Z3950)
2683                 handle_incoming_Z_PDU(apdu->u.z3950);
2684             else if (apdu->which == Z_GDU_HTTP_Request)
2685                 handle_incoming_HTTP(apdu->u.HTTP_Request);
2686         }
2687         else if (m_stylesheet_nprl)
2688             convert_xsl_delay();
2689         else
2690         {
2691             inc_request_no();
2692
2693             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2694             shutdown();
2695         }
2696     }
2697     else
2698     {
2699         timeout(600);
2700         pre_init();
2701     }
2702 }
2703
2704 void Yaz_Proxy::markInvalid()
2705 {
2706     m_client = 0;
2707     m_invalid_session = 1;
2708 }
2709
2710 void Yaz_ProxyClient::timeoutNotify()
2711 {
2712     if (m_server)
2713         m_server->inc_request_no();
2714
2715     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2716              get_hostname());
2717     m_waiting = 1;
2718     m_root->pre_init();
2719     if (m_server && m_init_flag)
2720     {
2721         // target timed out in a session that was properly initialized
2722         // server object stay alive but we mark it as invalid so it
2723         // gets initialized again
2724         m_server->markInvalid();
2725         m_server = 0;
2726     }
2727     shutdown();
2728 }
2729
2730 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2731                                  Yaz_Proxy *parent) :
2732     Yaz_Z_Assoc (the_PDU_Observable)
2733 {
2734     m_cookie = 0;
2735     m_next = 0;
2736     m_prev = 0;
2737     m_init_flag = 0;
2738     m_last_query = 0;
2739     m_last_resultSetId = 0;
2740     m_last_resultCount = 0;
2741     m_last_ok = 0;
2742     m_sr_transform = 0;
2743     m_waiting = 0;
2744     m_init_odr = odr_createmem (ODR_DECODE);
2745     m_initResponse = 0;
2746     m_initResponse_options = 0;
2747     m_initResponse_version = 0;
2748     m_initResponse_preferredMessageSize = 0;
2749     m_initResponse_maximumRecordSize = 0;
2750     m_resultSetStartPoint = 0;
2751     m_bytes_sent = m_bytes_recv = 0;
2752     m_pdu_recv = 0;
2753     m_server = 0;
2754     m_seqno = 0;
2755     m_target_idletime = 600;
2756     m_root = parent;
2757 }
2758
2759 const char *Yaz_Proxy::option(const char *name, const char *value)
2760 {
2761     if (!strcmp (name, "optimize")) {
2762         if (value) {
2763             xfree (m_optimize);
2764             m_optimize = xstrdup (value);
2765         }
2766         return m_optimize;
2767     }
2768     return 0;
2769 }
2770
2771 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2772 {
2773
2774 }
2775
2776 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2777 {
2778     if (apdu->which == Z_GDU_Z3950)
2779         recv_Z_PDU(apdu->u.z3950, len);
2780     else if (apdu->which == Z_GDU_HTTP_Response)
2781         recv_HTTP_response(apdu->u.HTTP_Response, len);
2782     else
2783         shutdown();
2784 }
2785
2786 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2787 {
2788     if (!m_invalid_session)
2789         return 0;
2790     m_invalid_session = 0;
2791     handle_incoming_Z_PDU(m_apdu_invalid_session);
2792     assert (m_mem_invalid_session);
2793     nmem_destroy(m_mem_invalid_session);
2794     m_mem_invalid_session = 0;
2795     return 1;
2796 }
2797
2798 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2799 {
2800     m_bytes_recv += len;
2801
2802     m_pdu_recv++;
2803     m_waiting = 0;
2804     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2805         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2806                  apdu_name(apdu), get_hostname(), len);
2807     if (apdu->which == Z_APDU_initResponse)
2808     {
2809         if (!m_server)  // if this is a pre init session , check for more
2810             m_root->pre_init();
2811         NMEM nmem = odr_extract_mem (odr_decode());
2812         odr_reset (m_init_odr);
2813         nmem_transfer (m_init_odr->mem, nmem);
2814         m_initResponse = apdu;
2815         m_initResponse_options = apdu->u.initResponse->options;
2816         m_initResponse_version = apdu->u.initResponse->protocolVersion;
2817         m_initResponse_preferredMessageSize =
2818             *apdu->u.initResponse->preferredMessageSize;
2819         m_initResponse_maximumRecordSize =
2820             *apdu->u.initResponse->maximumRecordSize;
2821
2822         Z_InitResponse *ir = apdu->u.initResponse;
2823         char *im0 = ir->implementationName;
2824
2825         char *im1 = (char*)
2826             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2827         *im1 = '\0';
2828         if (im0)
2829         {
2830             strcat(im1, im0);
2831             strcat(im1, " ");
2832         }
2833         strcat(im1, "(YAZ Proxy)");
2834         ir->implementationName = im1;
2835
2836         nmem_destroy (nmem);
2837
2838         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2839             return;
2840     }
2841     if (apdu->which == Z_APDU_searchResponse)
2842     {
2843         Z_SearchResponse *sr = apdu->u.searchResponse;
2844         m_last_resultCount = *sr->resultCount;
2845         int status = *sr->searchStatus;
2846         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2847         {
2848             m_last_ok = 1;
2849
2850             if (sr->records && sr->records->which == Z_Records_DBOSD)
2851             {
2852                 m_cache.add(odr_decode(),
2853                             sr->records->u.databaseOrSurDiagnostics, 1,
2854                             *sr->resultCount);
2855             }
2856         }
2857     }
2858     if (apdu->which == Z_APDU_presentResponse)
2859     {
2860         Z_PresentResponse *pr = apdu->u.presentResponse;
2861         if (m_sr_transform)
2862         {
2863             m_sr_transform = 0;
2864             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2865             Z_SearchResponse *sr = new_apdu->u.searchResponse;
2866             sr->referenceId = pr->referenceId;
2867             *sr->resultCount = m_last_resultCount;
2868             sr->records = pr->records;
2869             sr->nextResultSetPosition = pr->nextResultSetPosition;
2870             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2871             apdu = new_apdu;
2872         }
2873         if (pr->records &&
2874             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2875         {
2876             m_cache.add(odr_decode(),
2877                         pr->records->u.databaseOrSurDiagnostics,
2878                         m_resultSetStartPoint, -1);
2879             m_resultSetStartPoint = 0;
2880         }
2881     }
2882     if (m_cookie)
2883         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2884     if (m_server)
2885     {
2886         m_server->send_to_client(apdu);
2887     }
2888     if (apdu->which == Z_APDU_close)
2889     {
2890         shutdown();
2891     }
2892 }
2893
2894 void Yaz_Proxy::low_socket_close()
2895 {
2896 #if WIN32
2897 #else
2898     int i;
2899     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
2900         if  (m_lo_fd[i] >= 0)
2901             ::close(m_lo_fd[i]);
2902 #endif
2903 }
2904
2905 void Yaz_Proxy::low_socket_open()
2906 {
2907 #if WIN32
2908 #else
2909     int i;
2910     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
2911         m_lo_fd[i] = open("/dev/null", O_RDONLY);
2912 #endif
2913 }
2914
2915 int Yaz_Proxy::server(const char *addr)
2916 {
2917     int r = Yaz_Z_Assoc::server(addr);
2918     if (!r)
2919     {
2920         yaz_log(YLOG_LOG, "%sStarted proxy "
2921 #ifdef VERSION
2922             VERSION
2923 #endif
2924             " on %s", m_session_str, addr);
2925         timeout(1);
2926     }
2927     return r;
2928 }
2929