src/yaz-proxy.cpp

   1 /* $Id: yaz-proxy.cpp,v 1.26 2005-05-18 20:15:23 adam Exp $
   2    Copyright (c) 1998-2005, Index Data.
   3
   4 This file is part of the yaz-proxy.
   5
   6 YAZ proxy is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with YAZ proxy; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 #ifdef WIN32
  23 #define HAVE_SYS_STAT_H 1
  24 #define HAVE_SYS_TYPES_H 1
  25 #endif
  26
  27 #if HAVE_UNISTD_H
  28 #include <unistd.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_TYPES_H
  34 #include <sys/types.h>
  35 #endif
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 #include <assert.h>
  41 #include <stdlib.h>
  42 #include <time.h>
  43 #include <fcntl.h>
  44
  45 #include <yaz/srw.h>
  46 #include <yaz/marcdisp.h>
  47 #include <yaz/yaz-iconv.h>
  48 #include <yaz/log.h>
  49 #include <yaz/diagbib1.h>
  50 #include "proxyp.h"
  51 #include <yaz/pquery.h>
  52 #include <yaz/otherinfo.h>
  53 #include <yaz/charneg.h>
  54
  55
  56 static const char *apdu_name(Z_APDU *apdu)
  57 {
  58     switch (apdu->which)
  59     {
  60     case Z_APDU_initRequest:
  61         return "initRequest";
  62     case Z_APDU_initResponse:
  63         return "initResponse";
  64     case Z_APDU_searchRequest:
  65         return "searchRequest";
  66     case Z_APDU_searchResponse:
  67         return "searchResponse";
  68     case Z_APDU_presentRequest:
  69         return "presentRequest";
  70     case Z_APDU_presentResponse:
  71         return "presentResponse";
  72     case Z_APDU_deleteResultSetRequest:
  73         return "deleteResultSetRequest";
  74     case Z_APDU_deleteResultSetResponse:
  75         return "deleteResultSetResponse";
  76     case Z_APDU_scanRequest:
  77         return "scanRequest";
  78     case Z_APDU_scanResponse:
  79         return "scanResponse";
  80     case Z_APDU_sortRequest:
  81         return "sortRequest";
  82     case Z_APDU_sortResponse:
  83         return "sortResponse";
  84     case Z_APDU_extendedServicesRequest:
  85         return "extendedServicesRequest";
  86     case Z_APDU_extendedServicesResponse:
  87         return "extendedServicesResponse";
  88     case Z_APDU_close:
  89         return "close";
  90     }
  91     return "other";
  92 }
  93
  94 static const char *gdu_name(Z_GDU *gdu)
  95 {
  96     switch(gdu->which)
  97     {
  98     case Z_GDU_Z3950:
  99         return apdu_name(gdu->u.z3950);
 100     case Z_GDU_HTTP_Request:
 101         return "HTTP Request";
 102     case Z_GDU_HTTP_Response:
 103         return "HTTP Response";
 104     }
 105     return "Unknown request/response";
 106 }
 107 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
 108                      Yaz_Proxy *parent) :
 109     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 110 {
 111     m_PDU_Observable = the_PDU_Observable;
 112     m_client = 0;
 113     m_parent = parent;
 114     m_clientPool = 0;
 115     m_seqno = 1;
 116     m_keepalive_limit_bw = 500000;
 117     m_keepalive_limit_pdu = 1000;
 118     m_proxyTarget = 0;
 119     m_default_target = 0;
 120     m_proxy_negotiation_charset = 0;
 121     m_proxy_negotiation_lang = 0;
 122     m_charset_converter = new Yaz_CharsetConverter;
 123     m_max_clients = 150;
 124     m_log_mask = 0;
 125     m_seed = time(0);
 126     m_client_idletime = 600;
 127     m_target_idletime = 600;
 128     m_optimize = xstrdup ("1");
 129     strcpy(m_session_str, "0 ");
 130     m_session_no=0;
 131     m_bytes_sent = 0;
 132     m_bytes_recv = 0;
 133     m_bw_hold_PDU = 0;
 134     m_bw_max = 0;
 135     m_pdu_max = 0;
 136     m_max_record_retrieve = 0;
 137     m_reconfig_flag = 0;
 138     m_config_fname = 0;
 139     m_request_no = 0;
 140     m_invalid_session = 0;
 141     m_referenceId = 0;
 142     m_referenceId_mem = nmem_create();
 143     m_config = 0;
 144     m_marcxml_mode = none;
 145     m_stylesheet_xsp = 0;
 146     m_stylesheet_nprl = 0;
 147     m_s2z_stylesheet = 0;
 148     m_s2z_database = 0;
 149     m_schema = 0;
 150     m_backend_type = 0;
 151     m_backend_charset = 0;
 152     m_frontend_type = 0;
 153     m_initRequest_apdu = 0;
 154     m_initRequest_mem = 0;
 155     m_initRequest_preferredMessageSize = 0;
 156     m_initRequest_maximumRecordSize = 0;
 157     m_initRequest_options = 0;
 158     m_initRequest_version = 0;
 159     m_initRequest_oi_negotiation_charsets = 0;
 160     m_initRequest_oi_negotiation_num_charsets = 0;
 161     m_initRequest_oi_negotiation_langs = 0;
 162     m_initRequest_oi_negotiation_num_langs = 0;
 163     m_initRequest_oi_negotiation_selected = 0;
 164     m_apdu_invalid_session = 0;
 165     m_mem_invalid_session = 0;
 166     m_s2z_odr_init = 0;
 167     m_s2z_odr_search = 0;
 168     m_s2z_init_apdu = 0;
 169     m_s2z_search_apdu = 0;
 170     m_s2z_present_apdu = 0;
 171     m_http_keepalive = 0;
 172     m_http_version = 0;
 173     m_soap_ns = 0;
 174     m_s2z_packing = Z_SRW_recordPacking_string;
 175 #if HAVE_GETTIMEOFDAY
 176     m_time_tv = xmalloc(sizeof(struct timeval));
 177     struct timeval *tv = (struct timeval *) m_time_tv;
 178     tv->tv_sec = 0;
 179     tv->tv_usec = 0;
 180 #else
 181     m_time_tv = 0;
 182 #endif
 183     m_usemarcon_ini_stage1 = 0;
 184     m_usemarcon_ini_stage2 = 0;
 185     m_usemarcon = new Yaz_usemarcon();
 186     if (!m_parent)
 187         low_socket_open();
 188 }
 189
 190 Yaz_Proxy::~Yaz_Proxy()
 191 {
 192     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 193             m_bytes_sent, m_bytes_recv);
 194     nmem_destroy(m_initRequest_mem);
 195     nmem_destroy(m_mem_invalid_session);
 196     nmem_destroy(m_referenceId_mem);
 197
 198     xfree(m_proxyTarget);
 199     xfree(m_default_target);
 200     xfree(m_proxy_negotiation_charset);
 201     xfree(m_proxy_negotiation_lang);
 202     delete m_charset_converter;
 203     xfree(m_optimize);
 204
 205 #if HAVE_XSLT
 206     if (m_stylesheet_xsp)
 207         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 208 #endif
 209     xfree (m_time_tv);
 210
 211     xfree (m_schema);
 212     xfree (m_backend_type);
 213     xfree (m_backend_charset);
 214     xfree (m_usemarcon_ini_stage1);
 215     xfree (m_usemarcon_ini_stage2);
 216     delete m_usemarcon;
 217     if (m_s2z_odr_init)
 218         odr_destroy(m_s2z_odr_init);
 219     if (m_s2z_odr_search)
 220         odr_destroy(m_s2z_odr_search);
 221     if (!m_parent)
 222         low_socket_close();
 223     delete m_config;
 224 }
 225
 226 void Yaz_Proxy::set_debug_mode(int mode)
 227 {
 228     m_debug_mode = mode;
 229 }
 230
 231 int Yaz_Proxy::set_config(const char *config)
 232 {
 233     delete m_config;
 234     m_config = new Yaz_ProxyConfig();
 235     xfree(m_config_fname);
 236     m_config_fname = xstrdup(config);
 237     int r = m_config->read_xml(config);
 238     if (!r)
 239         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 240     return r;
 241 }
 242
 243 void Yaz_Proxy::set_default_target(const char *target)
 244 {
 245     xfree (m_default_target);
 246     m_default_target = 0;
 247     if (target)
 248         m_default_target = (char *) xstrdup (target);
 249 }
 250
 251 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang)
 252 {
 253     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
 254         "language to '%s'", m_session_str, charset?charset:"none",
 255         lang?lang:"none");
 256     xfree (m_proxy_negotiation_charset);
 257     xfree (m_proxy_negotiation_lang);
 258     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
 259     if (charset)
 260         m_proxy_negotiation_charset = (char *) xstrdup (charset);
 261     if (lang)
 262         m_proxy_negotiation_lang = (char *) xstrdup (lang);
 263 }
 264
 265 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 266 {
 267     if (m_parent)
 268         return m_parent->check_reconfigure();
 269
 270     Yaz_ProxyConfig *cfg = m_config;
 271     if (m_reconfig_flag)
 272     {
 273         yaz_log(YLOG_LOG, "reconfigure");
 274         yaz_log_reopen();
 275         if (m_config_fname && cfg)
 276         {
 277             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
 278             int r = cfg->read_xml(m_config_fname);
 279             if (r)
 280                 yaz_log(YLOG_WARN, "reconfigure failed");
 281             else
 282             {
 283                 m_log_mask = 0;
 284                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 285             }
 286         }
 287         else
 288             yaz_log(YLOG_LOG, "reconfigure");
 289         m_reconfig_flag = 0;
 290     }
 291     return cfg;
 292 }
 293
 294 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 295                                             *the_PDU_Observable, int fd)
 296 {
 297     check_reconfigure();
 298     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 299     new_proxy->m_config = 0;
 300     new_proxy->m_config_fname = 0;
 301     new_proxy->timeout(m_client_idletime);
 302     new_proxy->m_target_idletime = m_target_idletime;
 303     new_proxy->set_default_target(m_default_target);
 304     new_proxy->m_max_clients = m_max_clients;
 305     new_proxy->m_log_mask = m_log_mask;
 306     new_proxy->set_APDU_log(get_APDU_log());
 307     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 308         new_proxy->set_APDU_yazlog(1);
 309     else
 310         new_proxy->set_APDU_yazlog(0);
 311     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 312     m_session_no++;
 313     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 314              the_PDU_Observable->getpeername());
 315     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
 316         m_proxy_negotiation_lang);
 317     return new_proxy;
 318 }
 319
 320 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 321 {
 322     int oid[OID_SIZE];
 323     Z_OtherInformationUnit *oi;
 324     struct oident ent;
 325     ent.proto = PROTO_Z3950;
 326     ent.oclass = CLASS_USERINFO;
 327     ent.value = (oid_value) VAL_COOKIE;
 328     assert (oid_ent_to_oid (&ent, oid));
 329
 330     if (oid_ent_to_oid (&ent, oid) &&
 331         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 332         oi->which == Z_OtherInfo_characterInfo)
 333         return oi->information.characterInfo;
 334     return 0;
 335 }
 336 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 337 {
 338     int oid[OID_SIZE];
 339     Z_OtherInformationUnit *oi;
 340     struct oident ent;
 341     ent.proto = PROTO_Z3950;
 342     ent.oclass = CLASS_USERINFO;
 343     ent.value = (oid_value) VAL_PROXY;
 344     if (oid_ent_to_oid (&ent, oid) &&
 345         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 346         oi->which == Z_OtherInfo_characterInfo)
 347         return oi->information.characterInfo;
 348     return 0;
 349 }
 350 const char *Yaz_Proxy::load_balance(const char **url)
 351 {
 352     int zurl_in_use[MAX_ZURL_PLEX];
 353     int zurl_in_spare[MAX_ZURL_PLEX];
 354     Yaz_ProxyClient *c;
 355     int i;
 356
 357     for (i = 0; i<MAX_ZURL_PLEX; i++)
 358     {
 359         zurl_in_use[i] = 0;
 360         zurl_in_spare[i] = 0;
 361     }
 362     for (c = m_parent->m_clientPool; c; c = c->m_next)
 363     {
 364         for (i = 0; url[i]; i++)
 365             if (!strcmp(url[i], c->get_hostname()))
 366             {
 367                 zurl_in_use[i]++;
 368                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
 369                     zurl_in_spare[i]++;
 370             }
 371     }
 372     int min_use = 100000;
 373     int spare_for_min = 0;
 374     int max_spare = 0;
 375     const char *ret_min = 0;
 376     const char *ret_spare = 0;
 377     for (i = 0; url[i]; i++)
 378     {
 379         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
 380                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
 381         if (min_use > zurl_in_use[i])
 382         {
 383             ret_min = url[i];
 384             min_use = zurl_in_use[i];
 385             spare_for_min = zurl_in_spare[i];
 386         }
 387         if (max_spare < zurl_in_spare[i])
 388         {
 389             ret_spare = url[i];
 390             max_spare = zurl_in_spare[i];
 391         }
 392     }
 393     // use the one with minimum connections if spare is > 3
 394     if (spare_for_min > 3)
 395         return ret_min;
 396     // use one with most spares (if any)
 397     if (max_spare > 0)
 398         return ret_spare;
 399     return ret_min;
 400 }
 401
 402 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 403                                        const char *proxy_host)
 404 {
 405     assert (m_parent);
 406     Yaz_Proxy *parent = m_parent;
 407     Yaz_ProxyClient *c = m_client;
 408
 409     if (!m_proxyTarget)
 410     {
 411         const char *url[MAX_ZURL_PLEX];
 412         Yaz_ProxyConfig *cfg = check_reconfigure();
 413         if (proxy_host)
 414         {
 415             if (parent && parent->m_debug_mode)
 416             {
 417                 // only to be enabled for debugging...
 418                 if (!strcmp(proxy_host, "stop"))
 419                     exit(0);
 420             }
 421             xfree(m_default_target);
 422             m_default_target = xstrdup(proxy_host);
 423         }
 424         proxy_host = m_default_target;
 425         int client_idletime = -1;
 426         const char *cql2rpn_fname = 0;
 427         const char *negotiation_charset = 0;
 428         const char *negotiation_lang = 0;
 429         const char *query_charset = 0;
 430         url[0] = m_default_target;
 431         url[1] = 0;
 432         if (cfg)
 433         {
 434             int pre_init = 0;
 435             cfg->get_target_info(proxy_host, url, &m_bw_max,
 436                                  &m_pdu_max, &m_max_record_retrieve,
 437                                  &m_target_idletime, &client_idletime,
 438                                  &parent->m_max_clients,
 439                                  &m_keepalive_limit_bw,
 440                                  &m_keepalive_limit_pdu,
 441                                  &pre_init,
 442                                  &cql2rpn_fname,
 443                                  &negotiation_charset,
 444                                  &negotiation_lang,
 445                                  &query_charset);
 446         }
 447         if (client_idletime != -1)
 448         {
 449             m_client_idletime = client_idletime;
 450             timeout(m_client_idletime);
 451         }
 452         if (cql2rpn_fname)
 453             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 454         if (negotiation_charset || negotiation_lang)
 455         {
 456             set_proxy_negotiation(negotiation_charset,
 457                 negotiation_lang);
 458         }
 459         m_charset_converter->set_target_query_charset(query_charset);
 460         if (!url[0])
 461         {
 462             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
 463             return 0;
 464         }
 465         // we don't handle multiplexing for cookie session, so we just
 466         // pick the first one in this case (anonymous users will be able
 467         // to use any backend)
 468         if (cookie && *cookie)
 469             m_proxyTarget = (char*) xstrdup(url[0]);
 470         else
 471             m_proxyTarget = (char*) xstrdup(load_balance(url));
 472     }
 473     if (cookie && *cookie)
 474     {   // search in sessions with a cookie
 475         for (c = parent->m_clientPool; c; c = c->m_next)
 476         {
 477             assert (c->m_prev);
 478             assert (*c->m_prev == c);
 479             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 480                 !strcmp(m_proxyTarget, c->get_hostname()))
 481             {
 482                 // Found it in cache
 483                 // The following handles "cancel"
 484                 // If connection is busy (waiting for PDU) and
 485                 // we have an initRequest we can safely do re-open
 486                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 487                 {
 488                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
 489                              c->get_hostname());
 490                     c->close();
 491                     c->m_init_flag = 0;
 492
 493                     c->m_last_ok = 0;
 494                     c->m_cache.clear();
 495                     c->m_last_resultCount = 0;
 496                     c->m_sr_transform = 0;
 497                     c->m_waiting = 0;
 498                     c->m_resultSetStartPoint = 0;
 499                     c->m_target_idletime = m_target_idletime;
 500                     if (c->client(m_proxyTarget))
 501                     {
 502                         delete c;
 503                         return 0;
 504                     }
 505                     c->timeout(30);
 506                 }
 507                 c->m_seqno = parent->m_seqno;
 508                 if (c->m_server && c->m_server != this)
 509                     c->m_server->m_client = 0;
 510                 c->m_server = this;
 511                 (parent->m_seqno)++;
 512                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
 513                 return c;
 514             }
 515         }
 516     }
 517     else if (!c &&
 518             apdu->which == Z_APDU_initRequest &&
 519             apdu->u.initRequest->idAuthentication == 0 &&
 520             !ODR_MASK_GET(apdu->u.initRequest->options, Z_Options_negotiationModel))
 521     {
 522         // anonymous sessions without cookie.
 523         // if authentication is set it is NOT anonymous se we can't share them.
 524         // If charset and lang negotiation is use it is NOT anonymous session too.
 525         for (c = parent->m_clientPool; c; c = c->m_next)
 526         {
 527             assert(c->m_prev);
 528             assert(*c->m_prev == c);
 529             if (c->m_server == 0 && c->m_cookie == 0 &&
 530                 c->m_waiting == 0 &&
 531                 !strcmp(m_proxyTarget, c->get_hostname()))
 532             {
 533                 // found it in cache
 534                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
 535                          m_session_str, parent->m_seqno, c->get_hostname());
 536
 537                 c->m_seqno = parent->m_seqno;
 538                 assert(c->m_server == 0);
 539                 c->m_server = this;
 540
 541                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 542                     c->set_APDU_yazlog(1);
 543                 else
 544                     c->set_APDU_yazlog(0);
 545
 546                 (parent->m_seqno)++;
 547
 548                 parent->pre_init();
 549
 550                 return c;
 551             }
 552         }
 553     }
 554     if (!m_client)
 555     {
 556         if (apdu->which != Z_APDU_initRequest)
 557         {
 558             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
 559             return 0;
 560         }
 561         Z_InitRequest *initRequest = apdu->u.initRequest;
 562
 563         if (initRequest->idAuthentication)
 564         {
 565             // the client uses authentication. We set the keepalive PDU
 566             // to 0 so we don't cache it in releaseClient
 567             m_keepalive_limit_pdu = 0;
 568         }
 569         // go through list of clients - and find the lowest/oldest one.
 570         Yaz_ProxyClient *c_min = 0;
 571         int min_seq = -1;
 572         int no_of_clients = 0;
 573         if (parent->m_clientPool)
 574             yaz_log (YLOG_DEBUG, "Existing sessions");
 575         for (c = parent->m_clientPool; c; c = c->m_next)
 576         {
 577             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 578                                c->m_waiting, c->get_hostname(),
 579                                c->m_cookie ? c->m_cookie : "");
 580             no_of_clients++;
 581             if (min_seq < 0 || c->m_seqno < min_seq)
 582             {
 583                 min_seq = c->m_seqno;
 584                 c_min = c;
 585             }
 586         }
 587         if (no_of_clients >= parent->m_max_clients)
 588         {
 589             c = c_min;
 590             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 591             {
 592                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
 593                          m_session_str, parent->m_max_clients, c->m_seqno);
 594                 if (c->m_server && c->m_server != this)
 595                     delete c->m_server;
 596                 c->m_server = 0;
 597             }
 598             else
 599             {
 600                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
 601                          m_session_str, parent->m_max_clients,
 602                          c->m_seqno, parent->m_seqno, c->get_hostname());
 603                 xfree (c->m_cookie);
 604                 c->m_cookie = 0;
 605                 if (cookie)
 606                     c->m_cookie = xstrdup(cookie);
 607                 c->m_seqno = parent->m_seqno;
 608                 if (c->m_server && c->m_server != this)
 609                 {
 610                     c->m_server->m_client = 0;
 611                     delete c->m_server;
 612                 }
 613                 (parent->m_seqno)++;
 614                 c->m_target_idletime = m_target_idletime;
 615                 c->timeout(m_target_idletime);
 616
 617                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 618                     c->set_APDU_yazlog(1);
 619                 else
 620                     c->set_APDU_yazlog(0);
 621
 622                 return c;
 623             }
 624         }
 625         else
 626         {
 627             yaz_log (YLOG_LOG, "%sNEW %d %s",
 628                      m_session_str, parent->m_seqno, m_proxyTarget);
 629             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 630             c->m_next = parent->m_clientPool;
 631             if (c->m_next)
 632                 c->m_next->m_prev = &c->m_next;
 633             parent->m_clientPool = c;
 634             c->m_prev = &parent->m_clientPool;
 635         }
 636
 637         xfree (c->m_cookie);
 638         c->m_cookie = 0;
 639         if (cookie)
 640             c->m_cookie = xstrdup(cookie);
 641
 642         c->m_seqno = parent->m_seqno;
 643         c->m_init_flag = 0;
 644         c->m_last_resultCount = 0;
 645         c->m_last_ok = 0;
 646         c->m_cache.clear();
 647         c->m_sr_transform = 0;
 648         c->m_waiting = 0;
 649         c->m_resultSetStartPoint = 0;
 650         (parent->m_seqno)++;
 651         if (c->client(m_proxyTarget))
 652         {
 653             delete c;
 654             return 0;
 655         }
 656         c->m_target_idletime = m_target_idletime;
 657         c->timeout(30);
 658
 659         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 660             c->set_APDU_yazlog(1);
 661         else
 662             c->set_APDU_yazlog(0);
 663     }
 664     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
 665     return c;
 666 }
 667 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 668 {
 669     int i;
 670     for (i = 0; i<num; i++)
 671     {
 672         oident *ent;
 673         Z_DefaultDiagFormat *r;
 674         Z_DiagRec *p = pp[i];
 675         if (p->which != Z_DiagRec_defaultFormat)
 676         {
 677             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
 678             return;
 679         }
 680         else
 681             r = p->u.defaultFormat;
 682         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 683             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 684             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
 685         switch (r->which)
 686         {
 687         case Z_DefaultDiagFormat_v2Addinfo:
 688             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 689                     m_session_str,
 690                     *r->condition, diagbib1_str(*r->condition),
 691                     r->u.v2Addinfo);
 692             break;
 693         case Z_DefaultDiagFormat_v3Addinfo:
 694             yaz_log(YLOG_LOG, "%sError %d %s:%s",
 695                     m_session_str,
 696                     *r->condition, diagbib1_str(*r->condition),
 697                     r->u.v3Addinfo);
 698             break;
 699         }
 700     }
 701 }
 702
 703 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 704 {
 705     if (!m_stylesheet_xsp || p->num_records <= 0)
 706         return 0;  /* no XSLT to be done ... */
 707
 708     m_stylesheet_offset = 0;
 709     m_stylesheet_nprl = p;
 710     m_stylesheet_apdu = apdu;
 711     timeout(0);
 712     return 1;
 713 }
 714
 715 void Yaz_Proxy::convert_xsl_delay()
 716 {
 717 #if HAVE_XSLT
 718     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 719     if (npr->which == Z_NamePlusRecord_databaseRecord)
 720     {
 721         Z_External *r = npr->u.databaseRecord;
 722         if (r->which == Z_External_octet)
 723         {
 724 #if 0
 725             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
 726 #endif
 727             xmlDocPtr res, doc = xmlParseMemory(
 728                 (char*) r->u.octet_aligned->buf,
 729                 r->u.octet_aligned->len);
 730
 731
 732             yaz_log(YLOG_LOG, "%sXSLT convert %d",
 733                     m_session_str, m_stylesheet_offset);
 734             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
 735                                       doc, 0);
 736
 737             if (res)
 738             {
 739                 xmlChar *out_buf;
 740                 int out_len;
 741                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 742
 743                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 744                     u.databaseRecord =
 745                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 746                                  (char*) out_buf, out_len);
 747                 xmlFree(out_buf);
 748                 xmlFreeDoc(res);
 749             }
 750
 751             xmlFreeDoc(doc);
 752         }
 753     }
 754 #endif
 755     m_stylesheet_offset++;
 756     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 757     {
 758         m_stylesheet_nprl = 0;
 759 #if HAVE_XSLT
 760         if (m_stylesheet_xsp)
 761             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 762 #endif
 763         m_stylesheet_xsp = 0;
 764         timeout(m_client_idletime);
 765         send_PDU_convert(m_stylesheet_apdu);
 766     }
 767     else
 768         timeout(0);
 769 }
 770
 771 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 772 {
 773     if (m_frontend_type != VAL_NONE)
 774     {
 775         int i;
 776         for (i = 0; i < p->num_records; i++)
 777         {
 778             Z_NamePlusRecord *npr = p->records[i];
 779             if (npr->which == Z_NamePlusRecord_databaseRecord)
 780             {
 781                 Z_External *r = npr->u.databaseRecord;
 782                 if (r->which == Z_External_octet)
 783                 {
 784 #if HAVE_USEMARCON
 785                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
 786                     {
 787                         if (!m_usemarcon->m_stage1)
 788                         {
 789                             m_usemarcon->m_stage1 = new CDetails();
 790                         }
 791                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
 792                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
 793                         int res = m_usemarcon->m_stage1->Start();
 794                         if (res == 0)
 795                         {
 796                             char *converted;
 797                             int convlen;
 798                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
 799                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
 800                             {
 801                                 if (!m_usemarcon->m_stage2)
 802                                 {
 803                                     m_usemarcon->m_stage2 = new CDetails();
 804                                 }
 805                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
 806                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
 807                                 res = m_usemarcon->m_stage2->Start();
 808                                 if (res == 0)
 809                                 {
 810                                     free(converted);
 811                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
 812                                 }
 813                                 else
 814                                 {
 815                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
 816                                 }
 817                             }
 818                             npr->u.databaseRecord =
 819                                 z_ext_record(odr_encode(),
 820                                              m_frontend_type,
 821                                              converted,
 822                                              strlen(converted));
 823                             free(converted);
 824                         }
 825                         else
 826                         {
 827                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
 828                         }
 829                         continue;
 830                     }
 831 #endif
 832 /* HAVE_USEMARCON */
 833                     npr->u.databaseRecord =
 834                         z_ext_record(odr_encode(),
 835                                      m_frontend_type,
 836                                      (char*) r->u.octet_aligned->buf,
 837                                      r->u.octet_aligned->len);
 838                 }
 839             }
 840         }
 841     }
 842 }
 843
 844 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
 845                                         const char *backend_charset)
 846 {
 847     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
 848     int sel =   m_charset_converter->get_client_charset_selected();
 849     const char *client_record_charset =
 850         m_charset_converter->get_client_query_charset();
 851     if (sel && backend_charset && client_record_charset &&
 852         strcmp(backend_charset, client_record_charset))
 853     {
 854         int i;
 855         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
 856                                         backend_charset);
 857         yaz_marc_t mt = yaz_marc_create();
 858         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
 859         yaz_marc_iconv(mt, cd);
 860         for (i = 0; i < p->num_records; i++)
 861         {
 862             Z_NamePlusRecord *npr = p->records[i];
 863             if (npr->which == Z_NamePlusRecord_databaseRecord)
 864             {
 865                 Z_External *r = npr->u.databaseRecord;
 866                 oident *ent = oid_getentbyoid(r->direct_reference);
 867                 if (!ent || ent->value == VAL_NONE)
 868                     continue;
 869
 870                 if (ent->value == VAL_SUTRS)
 871                 {
 872                     WRBUF w = wrbuf_alloc();
 873
 874                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
 875                                       r->u.octet_aligned->len);
 876                     npr->u.databaseRecord =
 877                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
 878                                      wrbuf_len(w));
 879                     wrbuf_free(w, 1);
 880                 }
 881                 else if (ent->value == VAL_TEXT_XML)
 882                 {
 883                     ;
 884                 }
 885                 else if (r->which == Z_External_octet)
 886                 {
 887                     int rlen;
 888                     char *result;
 889                     if (yaz_marc_decode_buf(mt,
 890                                             (char*) r->u.octet_aligned->buf,
 891                                             r->u.octet_aligned->len,
 892                                             &result, &rlen))
 893                     {
 894                         npr->u.databaseRecord =
 895                             z_ext_record(odr_encode(), ent->value, result, rlen);
 896                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
 897                                 m_session_str);
 898                     }
 899                 }
 900             }
 901         }
 902         if (cd)
 903             yaz_iconv_close(cd);
 904         yaz_marc_destroy(mt);
 905     }
 906     else
 907     {
 908         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
 909     }
 910 }
 911
 912 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
 913                                    const char *backend_charset)
 914 {
 915     int i;
 916     if (!backend_charset)
 917         backend_charset = "MARC-8";
 918     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
 919     yaz_marc_t mt = yaz_marc_create();
 920     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 921     yaz_marc_iconv(mt, cd);
 922     for (i = 0; i < p->num_records; i++)
 923     {
 924         Z_NamePlusRecord *npr = p->records[i];
 925         if (npr->which == Z_NamePlusRecord_databaseRecord)
 926         {
 927             Z_External *r = npr->u.databaseRecord;
 928             if (r->which == Z_External_OPAC)
 929             {
 930                 WRBUF w = wrbuf_alloc();
 931
 932                 yaz_display_OPAC(w, r->u.opac, 0);
 933                 npr->u.databaseRecord = z_ext_record(
 934                     odr_encode(), VAL_TEXT_XML,
 935                     wrbuf_buf(w), wrbuf_len(w)
 936                     );
 937                 wrbuf_free(w, 1);
 938             }
 939             else if (r->which == Z_External_octet)
 940             {
 941                 int rlen;
 942                 char *result;
 943                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 944                                         r->u.octet_aligned->len,
 945                                         &result, &rlen))
 946                 {
 947                     npr->u.databaseRecord =
 948                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
 949                 }
 950             }
 951         }
 952     }
 953     if (cd)
 954         yaz_iconv_close(cd);
 955     yaz_marc_destroy(mt);
 956 }
 957
 958 void Yaz_Proxy::logtime()
 959 {
 960 #if HAVE_GETTIMEOFDAY
 961     struct timeval *tv = (struct timeval*) m_time_tv;
 962     if (tv->tv_sec)
 963     {
 964         struct timeval tv1;
 965         gettimeofday(&tv1, 0);
 966         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
 967             (tv1.tv_usec - tv->tv_usec);
 968         if (diff >= 0)
 969             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 970                     diff/1000000, (diff/1000)%1000);
 971     }
 972     tv->tv_sec = 0;
 973     tv->tv_usec = 0;
 974 #endif
 975 }
 976
 977 int Yaz_Proxy::send_http_response(int code)
 978 {
 979     ODR o = odr_encode();
 980     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 981     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 982     if (m_http_version)
 983         hres->version = odr_strdup(o, m_http_version);
 984     if (m_http_keepalive)
 985         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 986     else
 987         timeout(0);
 988
 989     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 990     {
 991         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
 992                  gdu_name(gdu));
 993     }
 994     int len;
 995     int r = send_GDU(gdu, &len);
 996     m_bytes_sent += len;
 997     m_bw_stat.add_bytes(len);
 998     logtime();
 999     return r;
1000 }
1001
1002 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
1003 {
1004     ODR o = odr_encode();
1005     const char *ctype = "text/xml";
1006     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
1007     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1008     if (m_http_version)
1009         hres->version = odr_strdup(o, m_http_version);
1010     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1011     if (m_http_keepalive)
1012         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1013     else
1014         timeout(0);
1015
1016     static Z_SOAP_Handler soap_handlers[2] = {
1017 #if HAVE_XSLT
1018         {"http://www.loc.gov/zing/srw/", 0,
1019          (Z_SOAP_fun) yaz_srw_codec},
1020 #endif
1021         {0, 0, 0}
1022     };
1023
1024     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1025     soap_package->which = Z_SOAP_generic;
1026     soap_package->u.generic =
1027         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1028     soap_package->u.generic->no = 0;
1029     soap_package->u.generic->ns = soap_handlers[0].ns;
1030     soap_package->u.generic->p = (void *) srw_pdu;
1031     soap_package->ns = m_soap_ns;
1032     z_soap_codec_enc_xsl(o, &soap_package,
1033                          &hres->content_buf, &hres->content_len,
1034                          soap_handlers, 0, m_s2z_stylesheet);
1035     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1036     {
1037         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1038                  gdu_name(gdu));
1039     }
1040     int len;
1041     int r = send_GDU(gdu, &len);
1042     m_bytes_sent += len;
1043     m_bw_stat.add_bytes(len);
1044     logtime();
1045     return r;
1046 }
1047
1048 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1049 {
1050     ODR o = odr_encode();
1051     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1052     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1053
1054     srw_res->num_diagnostics = 1;
1055     srw_res->diagnostics = (Z_SRW_diagnostic *)
1056         odr_malloc(o, sizeof(*srw_res->diagnostics));
1057     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
1058     return send_srw_response(srw_pdu);
1059 }
1060
1061 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1062                              Z_DefaultDiagFormat *ddf)
1063 {
1064     int bib1_code = *ddf->condition;
1065     if (bib1_code == 109)
1066         return 404;
1067     srw_res->num_diagnostics = 1;
1068     srw_res->diagnostics = (Z_SRW_diagnostic *)
1069         odr_malloc(o, sizeof(*srw_res->diagnostics));
1070     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1071                           yaz_diag_bib1_to_srw(*ddf->condition),
1072                           ddf->u.v2Addinfo);
1073     return 0;
1074 }
1075
1076 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1077 {
1078     ODR o = odr_encode();
1079     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1080     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1081
1082     srw_res->numberOfRecords = odr_intdup (o, hits);
1083     if (records && records->which == Z_Records_DBOSD)
1084     {
1085         srw_res->num_records =
1086             records->u.databaseOrSurDiagnostics->num_records;
1087         int i;
1088         srw_res->records = (Z_SRW_record *)
1089             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1090         for (i = 0; i < srw_res->num_records; i++)
1091         {
1092             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1093             if (npr->which != Z_NamePlusRecord_databaseRecord)
1094             {
1095                 srw_res->records[i].recordSchema = "diagnostic";
1096                 srw_res->records[i].recordPacking = m_s2z_packing;
1097                 srw_res->records[i].recordData_buf = "67";
1098                 srw_res->records[i].recordData_len = 2;
1099                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1100                 continue;
1101             }
1102             Z_External *r = npr->u.databaseRecord;
1103             oident *ent = oid_getentbyoid(r->direct_reference);
1104             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1105             {
1106                 srw_res->records[i].recordSchema = m_schema;
1107                 srw_res->records[i].recordPacking = m_s2z_packing;
1108                 srw_res->records[i].recordData_buf = (char*)
1109                     r->u.octet_aligned->buf;
1110                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1111                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1112             }
1113             else
1114             {
1115                 srw_res->records[i].recordSchema = "diagnostic";
1116                 srw_res->records[i].recordPacking = m_s2z_packing;
1117                 srw_res->records[i].recordData_buf = "67";
1118                 srw_res->records[i].recordData_len = 2;
1119                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1120             }
1121         }
1122     }
1123     if (records && records->which == Z_Records_NSD)
1124     {
1125         int http_code;
1126         http_code = z_to_srw_diag(odr_encode(), srw_res,
1127                                    records->u.nonSurrogateDiagnostic);
1128         if (http_code)
1129             return send_http_response(http_code);
1130     }
1131     return send_srw_response(srw_pdu);
1132
1133 }
1134
1135 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1136                                         int num_diagnostics)
1137 {
1138     Yaz_ProxyConfig *cfg = check_reconfigure();
1139     if (cfg)
1140     {
1141         int len;
1142         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1143                                        m_s2z_database, &len);
1144         if (b)
1145         {
1146             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1147             Z_SRW_explainResponse *er = res->u.explain_response;
1148
1149             er->record.recordData_buf = b;
1150             er->record.recordData_len = len;
1151             er->record.recordPacking = m_s2z_packing;
1152             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1153
1154             er->diagnostics = diagnostics;
1155             er->num_diagnostics = num_diagnostics;
1156             return send_srw_response(res);
1157         }
1158     }
1159     return send_http_response(404);
1160 }
1161
1162 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1163 {
1164     if (m_http_version)
1165     {
1166         if (apdu->which == Z_APDU_initResponse)
1167         {
1168             Z_InitResponse *res = apdu->u.initResponse;
1169             if (*res->result == 0)
1170             {
1171                 send_to_srw_client_error(3, 0);
1172             }
1173             else if (!m_s2z_search_apdu)
1174             {
1175                 send_srw_explain_response(0, 0);
1176             }
1177             else
1178             {
1179                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1180             }
1181         }
1182         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1183         {
1184             m_s2z_search_apdu = 0;
1185             Z_SearchResponse *res = apdu->u.searchResponse;
1186             m_s2z_hit_count = *res->resultCount;
1187             if (res->records && res->records->which == Z_Records_NSD)
1188             {
1189                 send_to_srw_client_ok(0, res->records, 1);
1190             }
1191             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1192             {
1193                 // adjust
1194                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1195
1196                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1197                 {
1198                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1199                         > m_s2z_hit_count)
1200                         *pr->numberOfRecordsRequested =
1201                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1202                 }
1203                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1204             }
1205             else
1206             {
1207                 m_s2z_present_apdu = 0;
1208                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1209             }
1210         }
1211         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1212         {
1213             int start =
1214                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1215
1216             m_s2z_present_apdu = 0;
1217             Z_PresentResponse *res = apdu->u.presentResponse;
1218             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1219         }
1220     }
1221     else
1222     {
1223         int len = 0;
1224         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1225             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1226                      apdu_name(apdu));
1227         int r = send_Z_PDU(apdu, &len);
1228         m_bytes_sent += len;
1229         m_bw_stat.add_bytes(len);
1230         logtime();
1231         return r;
1232     }
1233     return 0;
1234 }
1235
1236 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1237 {
1238     int kill_session = 0;
1239     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1240
1241     if (new_id)
1242         *new_id = m_referenceId;
1243
1244     if (apdu->which == Z_APDU_searchResponse)
1245     {
1246         Z_SearchResponse *sr = apdu->u.searchResponse;
1247         Z_Records *p = sr->records;
1248         if (p && p->which == Z_Records_NSD)
1249         {
1250             Z_DiagRec dr, *dr_p = &dr;
1251             dr.which = Z_DiagRec_defaultFormat;
1252             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1253
1254             *sr->searchStatus = 0;
1255             display_diagrecs(&dr_p, 1);
1256         }
1257         else
1258         {
1259             if (p && p->which == Z_Records_DBOSD)
1260             {
1261                 if (m_backend_type
1262 #if HAVE_USEMARCON
1263                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1264 #endif
1265                     )
1266                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1267                 if (m_marcxml_mode == marcxml)
1268                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1269                                        m_backend_charset);
1270                 else
1271                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1272                                             m_backend_charset);
1273                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1274                     return 0;
1275
1276             }
1277             if (sr->resultCount)
1278             {
1279                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1280                         *sr->resultCount);
1281                 if (*sr->resultCount < 0)
1282                 {
1283                     m_invalid_session = 1;
1284                     kill_session = 1;
1285
1286                     *sr->searchStatus = 0;
1287                     sr->records =
1288                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1289                     *sr->resultCount = 0;
1290                 }
1291             }
1292         }
1293     }
1294     else if (apdu->which == Z_APDU_presentResponse)
1295     {
1296         Z_PresentResponse *sr = apdu->u.presentResponse;
1297         Z_Records *p = sr->records;
1298         if (p && p->which == Z_Records_NSD)
1299         {
1300             Z_DiagRec dr, *dr_p = &dr;
1301             dr.which = Z_DiagRec_defaultFormat;
1302             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1303             if (*sr->presentStatus == Z_PresentStatus_success)
1304                 *sr->presentStatus = Z_PresentStatus_failure;
1305             display_diagrecs(&dr_p, 1);
1306         }
1307         if (p && p->which == Z_Records_DBOSD)
1308         {
1309             if (m_backend_type
1310 #if HAVE_USEMARCON
1311                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1312 #endif
1313                 )
1314                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1315             if (m_marcxml_mode == marcxml)
1316                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1317                                    m_backend_charset);
1318             else
1319                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1320                                         m_backend_charset);
1321             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1322                 return 0;
1323         }
1324     }
1325     else if (apdu->which == Z_APDU_initResponse)
1326     {
1327         //Get and check negotiation record
1328         //from init response.
1329         handle_charset_lang_negotiation(apdu);
1330
1331         if (m_initRequest_options)
1332         {
1333             Z_Options *nopt =
1334                 (Odr_bitmask *)odr_malloc(odr_encode(),
1335                                           sizeof(Odr_bitmask));
1336             ODR_MASK_ZERO(nopt);
1337
1338             int i;
1339             for (i = 0; i<24; i++)
1340                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1341                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1342                     ODR_MASK_SET(nopt, i);
1343             apdu->u.initResponse->options = nopt;
1344         }
1345         if (m_initRequest_version)
1346         {
1347             Z_ProtocolVersion *nopt =
1348                 (Odr_bitmask *)odr_malloc(odr_encode(),
1349                                           sizeof(Odr_bitmask));
1350             ODR_MASK_ZERO(nopt);
1351
1352             int i;
1353             for (i = 0; i<8; i++)
1354                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1355                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1356                     ODR_MASK_SET(nopt, i);
1357             apdu->u.initResponse->protocolVersion = nopt;
1358         }
1359         apdu->u.initResponse->preferredMessageSize =
1360             odr_intdup(odr_encode(),
1361                        m_client->m_initResponse_preferredMessageSize >
1362                        m_initRequest_preferredMessageSize ?
1363                        m_initRequest_preferredMessageSize :
1364                        m_client->m_initResponse_preferredMessageSize);
1365         apdu->u.initResponse->maximumRecordSize =
1366             odr_intdup(odr_encode(),
1367                        m_client->m_initResponse_maximumRecordSize >
1368                        m_initRequest_maximumRecordSize ?
1369                        m_initRequest_maximumRecordSize :
1370                        m_client->m_initResponse_maximumRecordSize);
1371     }
1372
1373     int r = send_PDU_convert(apdu);
1374     if (r)
1375         return r;
1376     if (kill_session)
1377     {
1378         delete m_client;
1379         m_client = 0;
1380         m_parent->pre_init();
1381     }
1382     return r;
1383 }
1384
1385 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1386 {
1387     int len = 0;
1388     const char *apdu_name_tmp = apdu_name(apdu);
1389     int r = send_Z_PDU(apdu, &len);
1390     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1391         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1392                  get_session_str(),
1393                  apdu_name_tmp, get_hostname(), len);
1394     m_bytes_sent += len;
1395     return r;
1396 }
1397
1398 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1399 {
1400     if (apdu->which == Z_APDU_presentRequest)
1401     {
1402         Z_PresentRequest *pr = apdu->u.presentRequest;
1403         int toget = *pr->numberOfRecordsRequested;
1404         int start = *pr->resultSetStartPoint;
1405
1406         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1407                 pr->resultSetId, start, toget);
1408
1409         if (*m_parent->m_optimize == '0')
1410             return apdu;
1411
1412         if (!m_client->m_last_resultSetId)
1413         {
1414             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1415             new_apdu->u.presentResponse->records =
1416                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1417                                                pr->resultSetId);
1418             send_to_client(new_apdu);
1419             return 0;
1420         }
1421         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1422         {
1423             if (start+toget-1 > m_client->m_last_resultCount)
1424             {
1425                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1426                 new_apdu->u.presentResponse->records =
1427                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1428                 send_to_client(new_apdu);
1429                 return 0;
1430             }
1431             Z_NamePlusRecordList *npr;
1432 #if 0
1433             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1434                     m_session_str, start, toget, yaz_z3950oid_to_str(
1435                         pr->preferredRecordSyntax, &oclass));
1436 #endif
1437             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1438                                           pr->preferredRecordSyntax,
1439                                           pr->recordComposition))
1440             {
1441                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1442                          m_session_str);
1443                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1444                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1445
1446                 new_apdu->u.presentResponse->numberOfRecordsReturned
1447                     = odr_intdup(odr_encode(), toget);
1448
1449                 new_apdu->u.presentResponse->records = (Z_Records*)
1450                     odr_malloc(odr_encode(), sizeof(Z_Records));
1451                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1452                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1453                 new_apdu->u.presentResponse->nextResultSetPosition =
1454                     odr_intdup(odr_encode(), start+toget);
1455
1456                 send_to_client(new_apdu);
1457                 return 0;
1458             }
1459         }
1460     }
1461
1462     if (apdu->which != Z_APDU_searchRequest)
1463         return apdu;
1464     Z_SearchRequest *sr = apdu->u.searchRequest;
1465     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1466     Yaz_Z_Databases this_databases;
1467
1468     this_databases.set(sr->num_databaseNames, (const char **)
1469                        sr->databaseNames);
1470
1471     this_query->set_Z_Query(sr->query);
1472
1473     char query_str[120];
1474     this_query->print(query_str, sizeof(query_str)-1);
1475     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1476
1477     if (*m_parent->m_optimize != '0' &&
1478         m_client->m_last_ok && m_client->m_last_query &&
1479         m_client->m_last_query->match(this_query) &&
1480         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1481         m_client->m_last_databases.match(this_databases))
1482     {
1483         delete this_query;
1484         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1485             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1486         {
1487             Z_NamePlusRecordList *npr;
1488             int toget = *sr->mediumSetPresentNumber;
1489             Z_RecordComposition *comp = 0;
1490
1491             if (toget > m_client->m_last_resultCount)
1492                 toget = m_client->m_last_resultCount;
1493
1494             if (sr->mediumSetElementSetNames)
1495             {
1496                 comp = (Z_RecordComposition *)
1497                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1498                 comp->which = Z_RecordComp_simple;
1499                 comp->u.simple = sr->mediumSetElementSetNames;
1500             }
1501
1502             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1503                                           sr->preferredRecordSyntax, comp))
1504             {
1505                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1506                          m_session_str);
1507                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1508                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1509                 new_apdu->u.searchResponse->resultCount =
1510                     &m_client->m_last_resultCount;
1511
1512                 new_apdu->u.searchResponse->numberOfRecordsReturned
1513                     = odr_intdup(odr_encode(), toget);
1514
1515                 new_apdu->u.searchResponse->presentStatus =
1516                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1517                 new_apdu->u.searchResponse->records = (Z_Records*)
1518                     odr_malloc(odr_encode(), sizeof(Z_Records));
1519                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1520                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1521                 new_apdu->u.searchResponse->nextResultSetPosition =
1522                     odr_intdup(odr_encode(), toget+1);
1523                 send_to_client(new_apdu);
1524                 return 0;
1525             }
1526             else
1527             {
1528                 // medium Set
1529                 // send present request (medium size)
1530                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1531                          m_session_str);
1532
1533                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1534                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1535                 pr->referenceId = sr->referenceId;
1536                 pr->resultSetId = sr->resultSetName;
1537                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1538                 *pr->numberOfRecordsRequested = toget;
1539                 pr->recordComposition = comp;
1540                 m_client->m_sr_transform = 1;
1541                 return new_apdu;
1542             }
1543         }
1544         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1545             m_client->m_last_resultCount <= 0)
1546         {
1547             // large set. Return pseudo-search response immediately
1548             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1549                      m_session_str);
1550             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1551             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1552             new_apdu->u.searchResponse->resultCount =
1553                 &m_client->m_last_resultCount;
1554             send_to_client(new_apdu);
1555             return 0;
1556         }
1557         else
1558         {
1559             Z_NamePlusRecordList *npr;
1560             int toget = m_client->m_last_resultCount;
1561             Z_RecordComposition *comp = 0;
1562             // small set
1563             // send a present request (small set)
1564
1565             if (sr->smallSetElementSetNames)
1566             {
1567                 comp = (Z_RecordComposition *)
1568                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1569                 comp->which = Z_RecordComp_simple;
1570                 comp->u.simple = sr->smallSetElementSetNames;
1571             }
1572
1573             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1574                                           sr->preferredRecordSyntax, comp))
1575             {
1576                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1577                          m_session_str);
1578                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1579                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1580                 new_apdu->u.searchResponse->resultCount =
1581                     &m_client->m_last_resultCount;
1582
1583                 new_apdu->u.searchResponse->numberOfRecordsReturned
1584                     = odr_intdup(odr_encode(), toget);
1585
1586                 new_apdu->u.searchResponse->presentStatus =
1587                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1588                 new_apdu->u.searchResponse->records = (Z_Records*)
1589                     odr_malloc(odr_encode(), sizeof(Z_Records));
1590                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1591                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1592                 new_apdu->u.searchResponse->nextResultSetPosition =
1593                     odr_intdup(odr_encode(), toget+1);
1594                 send_to_client(new_apdu);
1595                 return 0;
1596             }
1597             else
1598             {
1599                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1600                          m_session_str);
1601                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1602                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1603                 pr->referenceId = sr->referenceId;
1604                 pr->resultSetId = sr->resultSetName;
1605                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1606                 *pr->numberOfRecordsRequested = toget;
1607                 pr->recordComposition = comp;
1608                 m_client->m_sr_transform = 1;
1609                 return new_apdu;
1610             }
1611         }
1612     }
1613     else  // query doesn't match
1614     {
1615         delete m_client->m_last_query;
1616         m_client->m_last_query = this_query;
1617         m_client->m_last_ok = 0;
1618         m_client->m_cache.clear();
1619         m_client->m_resultSetStartPoint = 0;
1620
1621         xfree (m_client->m_last_resultSetId);
1622         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1623
1624         m_client->m_last_databases.set(sr->num_databaseNames,
1625                                        (const char **) sr->databaseNames);
1626     }
1627     return apdu;
1628 }
1629
1630
1631 void Yaz_Proxy::inc_request_no()
1632 {
1633     char *cp = strchr(m_session_str, ' ');
1634     m_request_no++;
1635     if (cp)
1636         sprintf(cp+1, "%d ", m_request_no);
1637 }
1638
1639 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1640 {
1641     inc_request_no();
1642
1643     m_bytes_recv += len;
1644
1645     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1646         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1647                  m_session_str, gdu_name(apdu), len);
1648
1649     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1650         shutdown();
1651
1652     m_bw_stat.add_bytes(len);
1653     m_pdu_stat.add_bytes(1);
1654
1655 #if HAVE_GETTIMEOFDAY
1656     gettimeofday((struct timeval *) m_time_tv, 0);
1657 #endif
1658
1659     int bw_total = m_bw_stat.get_total();
1660     int pdu_total = m_pdu_stat.get_total();
1661
1662     int reduce = 0;
1663     if (m_bw_max)
1664     {
1665         if (bw_total > m_bw_max)
1666         {
1667             reduce = (bw_total/m_bw_max);
1668         }
1669     }
1670     if (m_pdu_max)
1671     {
1672         if (pdu_total > m_pdu_max)
1673         {
1674             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1675             reduce = (reduce > nreduce) ? reduce : nreduce;
1676         }
1677     }
1678     m_http_version = 0;
1679     if (reduce)
1680     {
1681         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1682                 m_session_str, reduce, bw_total, pdu_total,
1683                 m_bw_max, m_pdu_max);
1684
1685         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1686         timeout(reduce);       // call us reduce seconds later
1687     }
1688     else if (apdu->which == Z_GDU_Z3950)
1689         handle_incoming_Z_PDU(apdu->u.z3950);
1690     else if (apdu->which == Z_GDU_HTTP_Request)
1691         handle_incoming_HTTP(apdu->u.HTTP_Request);
1692 }
1693
1694 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1695 {
1696     if (m_max_record_retrieve)
1697     {
1698         if (apdu->which == Z_APDU_presentRequest)
1699         {
1700             Z_PresentRequest *pr = apdu->u.presentRequest;
1701             if (pr->numberOfRecordsRequested &&
1702                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1703                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1704         }
1705     }
1706 }
1707 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1708 {
1709     if (apdu->which == Z_APDU_initRequest)
1710     {
1711         yaz_log(YLOG_LOG, "%shandle_charset_lang_negotiation",
1712                 m_session_str);
1713         if (m_initRequest_options &&
1714             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1715             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1716         {
1717             // There is no negotiation proposal from
1718             // client's side. OK. The proxy negotiation
1719             // in use, only.
1720             Z_InitRequest *initRequest = apdu->u.initRequest;
1721             Z_OtherInformation **otherInfo;
1722             Z_OtherInformationUnit *oi;
1723             get_otherInfoAPDU(apdu, &otherInfo);
1724             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1725             if (oi)
1726             {
1727                 ODR_MASK_SET(initRequest->options,
1728                     Z_Options_negotiationModel);
1729                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1730                 oi->information.externallyDefinedInfo =
1731                 yaz_set_proposal_charneg(odr_encode(),
1732                     (const char**)&m_proxy_negotiation_charset,
1733                     m_proxy_negotiation_charset ? 1:0,
1734                     (const char**)&m_proxy_negotiation_lang,
1735                     m_proxy_negotiation_lang ? 1:0,
1736                     1);
1737             }
1738         }
1739         else if (m_initRequest_options &&
1740                  ODR_MASK_GET(m_initRequest_options,
1741                               Z_Options_negotiationModel) &&
1742                  m_charset_converter->get_target_query_charset())
1743         {
1744             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1745                     m_session_str,
1746                     m_charset_converter->get_target_query_charset());
1747             Z_InitRequest *initRequest = apdu->u.initRequest;
1748             Z_CharSetandLanguageNegotiation *negotiation =
1749                 yaz_get_charneg_record (initRequest->otherInfo);
1750             if (negotiation &&
1751                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1752             {
1753                 NMEM nmem = nmem_create();
1754                 char **charsets = 0;
1755                 int num_charsets = 0;
1756                 char **langs = 0;
1757                 int num_langs = 0;
1758                 int selected = 0;
1759                 yaz_get_proposal_charneg (nmem, negotiation,
1760                                           &charsets, &num_charsets,
1761                                           &langs, &num_langs, &selected);
1762                 int i;
1763                 for (i = 0; i<num_charsets; i++)
1764                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
1765                             charsets[i]);
1766                 for (i = 0; i<num_langs; i++)
1767                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
1768                             langs[i]);
1769
1770                 const char *t_charset =
1771                     m_charset_converter->get_target_query_charset();
1772                 // sweep through charsets and pick the first supported
1773                 // conversion
1774                 for (i = 0; i<num_charsets; i++)
1775                 {
1776                     const char *c_charset = charsets[i];
1777                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
1778                         break;
1779                 }
1780                 if (i != num_charsets)
1781                 {
1782                     // got one .. set up ODR for reverse direction
1783                     const char *c_charset = charsets[i];
1784                     odr_set_charset(odr_encode(), c_charset, t_charset);
1785                     m_charset_converter->set_client_query_charset(c_charset);
1786                     m_charset_converter->set_client_charset_selected(selected);
1787                 }
1788                 nmem_destroy(nmem);
1789                 ODR_MASK_CLEAR(m_initRequest_options,
1790                                Z_Options_negotiationModel);
1791                 yaz_del_charneg_record(&initRequest->otherInfo);
1792             }
1793             else
1794             {
1795                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
1796                         m_session_str);
1797             }
1798         }
1799     }
1800     else if (apdu->which == Z_APDU_initResponse)
1801     {
1802         Z_InitResponse *initResponse = apdu->u.initResponse;
1803         Z_OtherInformation **otherInfo;
1804
1805         if (ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel))
1806         {
1807             char *charset = 0;
1808             char *lang = 0;
1809             int selected = 0;
1810
1811             get_otherInfoAPDU(apdu, &otherInfo);
1812
1813             if (!otherInfo && !(*otherInfo))
1814                 return;
1815
1816             Z_CharSetandLanguageNegotiation *charneg =
1817                 yaz_get_charneg_record(*otherInfo);
1818
1819             if (!charneg)
1820                 return;
1821
1822             yaz_get_response_charneg(m_referenceId_mem, charneg,
1823                 &charset, &lang, &selected);
1824
1825             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
1826                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
1827
1828             if (m_initRequest_options &&
1829                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
1830             {
1831                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
1832                     m_session_str);
1833             }
1834             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1835             {
1836                 // negotiation-charset, negotiation-lang
1837                 // elements of config file in use.
1838
1839                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
1840                     m_session_str);
1841
1842                 // clear negotiation option.
1843                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
1844
1845                 // Delete negotiation (charneg-3) entry.
1846                 yaz_del_charneg_record(otherInfo);
1847             }
1848         }
1849         else
1850         {
1851             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1852             {
1853                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
1854                         m_session_str);
1855             }
1856             else if (m_charset_converter->get_client_query_charset())
1857             {
1858                 Z_OtherInformation **otherInfo;
1859                 Z_OtherInformationUnit *oi;
1860                 get_otherInfoAPDU(apdu, &otherInfo);
1861                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1862                 if (oi)
1863                 {
1864                     ODR_MASK_SET(initResponse->options,
1865                                  Z_Options_negotiationModel);
1866                     ODR_MASK_SET(m_initRequest_options,
1867                                  Z_Options_negotiationModel);
1868
1869                     oi->which = Z_OtherInfo_externallyDefinedInfo;
1870                     oi->information.externallyDefinedInfo =
1871                         yaz_set_response_charneg(
1872                             odr_encode(),
1873                             m_charset_converter->get_client_query_charset(),
1874                             0 /* no lang */,
1875                             m_charset_converter->get_client_charset_selected());
1876                 }
1877             }
1878         }
1879     }
1880 }
1881
1882 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1883                                                      int error,
1884                                                      const char *addinfo)
1885 {
1886     Z_Records *rec = (Z_Records *)
1887         odr_malloc (odr, sizeof(*rec));
1888     int *err = (int *)
1889         odr_malloc (odr, sizeof(*err));
1890     Z_DiagRec *drec = (Z_DiagRec *)
1891         odr_malloc (odr, sizeof(*drec));
1892     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1893         odr_malloc (odr, sizeof(*dr));
1894     *err = error;
1895     rec->which = Z_Records_NSD;
1896     rec->u.nonSurrogateDiagnostic = dr;
1897     dr->diagnosticSetId =
1898         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1899     dr->condition = err;
1900     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1901     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1902     return rec;
1903 }
1904
1905 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1906 {
1907     if (apdu->which == Z_APDU_searchRequest &&
1908         apdu->u.searchRequest->query &&
1909         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1910         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1911     {
1912         Z_RPNQuery *rpnquery = 0;
1913         Z_SearchRequest *sr = apdu->u.searchRequest;
1914         char *addinfo = 0;
1915
1916         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
1917                 sr->query->u.type_104->u.cql);
1918
1919         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1920                                           &rpnquery, odr_encode(),
1921                                           &addinfo);
1922         if (r == -3)
1923             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
1924         else if (r)
1925         {
1926             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1927             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1928
1929             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1930             new_apdu->u.searchResponse->records =
1931                 create_nonSurrogateDiagnostics(odr_encode(),
1932                                                yaz_diag_srw_to_bib1(r),
1933                                                addinfo);
1934             *new_apdu->u.searchResponse->searchStatus = 0;
1935
1936             send_to_client(new_apdu);
1937
1938             return 0;
1939         }
1940         else
1941         {
1942             sr->query->which = Z_Query_type_1;
1943             sr->query->u.type_1 = rpnquery;
1944         }
1945         return apdu;
1946     }
1947     return apdu;
1948 }
1949
1950 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
1951 {
1952     if (apdu->which == Z_APDU_searchRequest &&
1953         apdu->u.searchRequest->query)
1954     {
1955         if (apdu->u.searchRequest->query->which == Z_Query_type_1
1956             || apdu->u.searchRequest->query->which == Z_Query_type_101)
1957         {
1958             if (m_http_version)
1959                 m_charset_converter->set_client_query_charset("UTF-8");
1960             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
1961             m_charset_converter->convert_type_1(rpnquery, odr_encode());
1962         }
1963     }
1964     return apdu;
1965 }
1966
1967
1968 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1969 {
1970     if (apdu->which == Z_APDU_searchRequest)
1971     {
1972         Z_SearchRequest *sr = apdu->u.searchRequest;
1973         int err = 0;
1974         char *addinfo = 0;
1975
1976         Yaz_ProxyConfig *cfg = check_reconfigure();
1977         if (cfg)
1978             err = cfg->check_query(odr_encode(), m_default_target,
1979                                    sr->query, &addinfo);
1980         if (err)
1981         {
1982             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1983
1984             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1985             new_apdu->u.searchResponse->records =
1986                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1987             *new_apdu->u.searchResponse->searchStatus = 0;
1988
1989             send_to_client(new_apdu);
1990
1991             return 0;
1992         }
1993     }
1994     return apdu;
1995 }
1996
1997 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
1998 {
1999     if (apdu->which != Z_APDU_initRequest)
2000         return 1;  // pass if no init request
2001     Z_InitRequest *req = apdu->u.initRequest;
2002
2003     Yaz_ProxyConfig *cfg = check_reconfigure();
2004     if (!cfg)
2005         return 1;  // pass if no config
2006
2007     int ret;
2008     if (req->idAuthentication == 0)
2009     {
2010         ret = cfg->client_authentication(m_default_target, 0, 0, 0);
2011     }
2012     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2013     {
2014         ret = cfg->client_authentication(m_default_target,
2015             req->idAuthentication->u.idPass->userId,
2016             req->idAuthentication->u.idPass->groupId,
2017             req->idAuthentication->u.idPass->password);
2018     }
2019     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2020     {
2021         char user[64], pass[64];
2022         *user = '\0';
2023         *pass = '\0';
2024         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2025         ret = cfg->client_authentication(m_default_target, user, 0, pass);
2026     }
2027     else
2028         ret = cfg->client_authentication(m_default_target, 0, 0, 0);
2029
2030     cfg->target_authentication(m_default_target, odr_encode(), req);
2031
2032     return ret;
2033 }
2034
2035 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2036 {
2037     m_marcxml_mode = none;
2038     if (apdu->which == Z_APDU_searchRequest)
2039     {
2040         Z_SearchRequest *sr = apdu->u.searchRequest;
2041         int err = 0;
2042         char *addinfo = 0;
2043         Yaz_ProxyConfig *cfg = check_reconfigure();
2044
2045         Z_RecordComposition rc_temp, *rc = 0;
2046         if (sr->smallSetElementSetNames)
2047         {
2048             rc_temp.which = Z_RecordComp_simple;
2049             rc_temp.u.simple = sr->smallSetElementSetNames;
2050             rc = &rc_temp;
2051         }
2052
2053         if (sr->preferredRecordSyntax)
2054         {
2055             struct oident *ent;
2056             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2057             m_frontend_type = ent->value;
2058         }
2059         else
2060             m_frontend_type = VAL_NONE;
2061
2062         char *stylesheet_name = 0;
2063         if (cfg)
2064             err = cfg->check_syntax(odr_encode(),
2065                                     m_default_target,
2066                                     sr->preferredRecordSyntax, rc,
2067                                     &addinfo, &stylesheet_name, &m_schema,
2068                                     &m_backend_type, &m_backend_charset,
2069                                     &m_usemarcon_ini_stage1,
2070                                     &m_usemarcon_ini_stage2);
2071         if (stylesheet_name)
2072         {
2073             m_parent->low_socket_close();
2074
2075 #if HAVE_XSLT
2076             if (m_stylesheet_xsp)
2077                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2078             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2079                                                        stylesheet_name);
2080 #endif
2081             m_stylesheet_offset = 0;
2082             xfree(stylesheet_name);
2083
2084             m_parent->low_socket_open();
2085         }
2086         if (err == -1)
2087         {
2088             sr->smallSetElementSetNames = 0;
2089             sr->mediumSetElementSetNames = 0;
2090             m_marcxml_mode = marcxml;
2091             if (m_backend_type)
2092             {
2093
2094                 sr->preferredRecordSyntax =
2095                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2096                                         m_backend_type);
2097             }
2098             else
2099                 sr->preferredRecordSyntax =
2100                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2101                                            VAL_USMARC);
2102         }
2103         else if (err)
2104         {
2105             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2106
2107             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2108             new_apdu->u.searchResponse->records =
2109                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2110             *new_apdu->u.searchResponse->searchStatus = 0;
2111
2112             send_to_client(new_apdu);
2113
2114             return 0;
2115         }
2116         else if (m_backend_type)
2117         {
2118             sr->preferredRecordSyntax =
2119                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2120         }
2121     }
2122     else if (apdu->which == Z_APDU_presentRequest)
2123     {
2124         Z_PresentRequest *pr = apdu->u.presentRequest;
2125         int err = 0;
2126         char *addinfo = 0;
2127         Yaz_ProxyConfig *cfg = check_reconfigure();
2128
2129         if (pr->preferredRecordSyntax)
2130         {
2131             struct oident *ent;
2132             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2133             m_frontend_type = ent->value;
2134         }
2135         else
2136             m_frontend_type = VAL_NONE;
2137
2138         char *stylesheet_name = 0;
2139         if (cfg)
2140             err = cfg->check_syntax(odr_encode(), m_default_target,
2141                                     pr->preferredRecordSyntax,
2142                                     pr->recordComposition,
2143                                     &addinfo, &stylesheet_name, &m_schema,
2144                                     &m_backend_type, &m_backend_charset,
2145                                     &m_usemarcon_ini_stage1,
2146                                     &m_usemarcon_ini_stage2
2147                                     );
2148         if (stylesheet_name)
2149         {
2150             m_parent->low_socket_close();
2151
2152 #if HAVE_XSLT
2153             if (m_stylesheet_xsp)
2154                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2155             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2156                                                        stylesheet_name);
2157 #endif
2158             m_stylesheet_offset = 0;
2159             xfree(stylesheet_name);
2160
2161             m_parent->low_socket_open();
2162         }
2163         if (err == -1)
2164         {
2165             pr->recordComposition = 0;
2166             m_marcxml_mode = marcxml;
2167             if (m_backend_type)
2168             {
2169
2170                 pr->preferredRecordSyntax =
2171                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2172                                         m_backend_type);
2173             }
2174             else
2175                 pr->preferredRecordSyntax =
2176                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2177                                            VAL_USMARC);
2178         }
2179         else if (err)
2180         {
2181             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2182
2183             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2184             new_apdu->u.presentResponse->records =
2185                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2186             *new_apdu->u.presentResponse->presentStatus =
2187                 Z_PresentStatus_failure;
2188
2189             send_to_client(new_apdu);
2190
2191             return 0;
2192         }
2193         else if (m_backend_type)
2194         {
2195             pr->preferredRecordSyntax =
2196                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2197         }
2198     }
2199     return apdu;
2200 }
2201
2202 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2203 {
2204     if (!schema)
2205         return 0;
2206     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2207         odr_malloc(o, sizeof(Z_ElementSetNames));
2208     esn->which = Z_ElementSetNames_generic;
2209     esn->u.generic = odr_strdup(o, schema);
2210     return esn;
2211 }
2212
2213 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2214 {
2215     const char *t = 0;
2216     Yaz_ProxyConfig *cfg = check_reconfigure();
2217     if (cfg)
2218         t = cfg->get_explain_name(db, backend_db);
2219
2220     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2221     {
2222         releaseClient();
2223     }
2224
2225     if (t)
2226     {
2227         xfree(m_default_target);
2228         m_default_target = xstrdup(t);
2229     }
2230 }
2231
2232 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2233 {
2234     struct stat sbuf;
2235     if (strcmp(hreq->method, "GET"))
2236         return 0;
2237     if (hreq->path[0] != '/')
2238     {
2239         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2240         return 0;
2241     }
2242     const char *cp = hreq->path;
2243     while (*cp)
2244     {
2245         if (*cp == '/' && strchr("/.", cp[1]))
2246         {
2247             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2248             return 0;
2249         }
2250         cp++;
2251     }
2252     const char *fname = hreq->path+1;
2253     if (stat(fname, &sbuf))
2254     {
2255         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2256         return 0;
2257     }
2258     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2259     {
2260         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2261         return 0;
2262     }
2263     if (sbuf.st_size > (off_t) 1000000)
2264     {
2265         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2266         return 0;
2267     }
2268
2269     ODR o = odr_encode();
2270     Yaz_ProxyConfig *cfg = check_reconfigure();
2271     const char *ctype = cfg->check_mime_type(fname);
2272     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2273     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2274     if (m_http_version)
2275         hres->version = odr_strdup(o, m_http_version);
2276     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2277     if (m_http_keepalive)
2278         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2279     else
2280         timeout(0);
2281
2282     hres->content_len = sbuf.st_size;
2283     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2284     FILE *f = fopen(fname, "rb");
2285     if (f)
2286     {
2287         fread(hres->content_buf, 1, hres->content_len, f);
2288         fclose(f);
2289     }
2290     else
2291     {
2292         return 0;
2293     }
2294     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2295     {
2296         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2297                  fname);
2298     }
2299     int len;
2300     send_GDU(gdu, &len);
2301     return 1;
2302 }
2303
2304 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2305 {
2306     if (m_s2z_odr_init)
2307     {
2308         odr_destroy(m_s2z_odr_init);
2309         m_s2z_odr_init = 0;
2310     }
2311     if (m_s2z_odr_search)
2312     {
2313         odr_destroy(m_s2z_odr_search);
2314         m_s2z_odr_search = 0;
2315     }
2316
2317     m_http_keepalive = 0;
2318     m_http_version = 0;
2319     if (!strcmp(hreq->version, "1.0"))
2320     {
2321         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2322         if (v && !strcmp(v, "Keep-Alive"))
2323             m_http_keepalive = 1;
2324         else
2325             m_http_keepalive = 0;
2326         m_http_version = "1.0";
2327     }
2328     else
2329     {
2330         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2331         if (v && !strcmp(v, "close"))
2332             m_http_keepalive = 0;
2333         else
2334             m_http_keepalive = 1;
2335         m_http_version = "1.1";
2336     }
2337
2338     Z_SRW_PDU *srw_pdu = 0;
2339     Z_SOAP *soap_package = 0;
2340     char *charset = 0;
2341     Z_SRW_diagnostic *diagnostic = 0;
2342     int num_diagnostic = 0;
2343
2344     if (file_access(hreq))
2345     {
2346         return;
2347     }
2348     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2349                             &charset) == 0
2350              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2351                                &charset, &diagnostic, &num_diagnostic) == 0)
2352     {
2353         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2354         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2355         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2356         m_s2z_init_apdu = 0;
2357         m_s2z_search_apdu = 0;
2358         m_s2z_present_apdu = 0;
2359
2360         m_s2z_stylesheet = 0;
2361
2362         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2363         {
2364             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2365
2366             const char *backend_db = srw_req->database;
2367             srw_get_client(srw_req->database, &backend_db);
2368
2369             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2370             // recordXPath unsupported.
2371             if (srw_req->recordXPath)
2372             {
2373                 yaz_add_srw_diagnostic(odr_decode(),
2374                                        &diagnostic, &num_diagnostic,
2375                                        72, 0);
2376             }
2377             // sort unsupported
2378             if (srw_req->sort_type != Z_SRW_sort_type_none)
2379             {
2380                 yaz_add_srw_diagnostic(odr_decode(),
2381                                        &diagnostic, &num_diagnostic,
2382                                        80, 0);
2383             }
2384             // save stylesheet
2385             if (srw_req->stylesheet)
2386                 m_s2z_stylesheet =
2387                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2388
2389             // set packing for response records ..
2390             if (srw_req->recordPacking &&
2391                 !strcmp(srw_req->recordPacking, "xml"))
2392                 m_s2z_packing = Z_SRW_recordPacking_XML;
2393             else
2394                 m_s2z_packing = Z_SRW_recordPacking_string;
2395
2396             if (num_diagnostic)
2397             {
2398                 Z_SRW_PDU *srw_pdu =
2399                     yaz_srw_get(odr_encode(),
2400                                 Z_SRW_searchRetrieve_response);
2401                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2402
2403                 srw_res->diagnostics = diagnostic;
2404                 srw_res->num_diagnostics = num_diagnostic;
2405                 send_srw_response(srw_pdu);
2406                 return;
2407             }
2408
2409             // prepare search PDU
2410             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2411                                           Z_APDU_searchRequest);
2412             Z_SearchRequest *z_searchRequest =
2413                 m_s2z_search_apdu->u.searchRequest;
2414
2415             z_searchRequest->num_databaseNames = 1;
2416             z_searchRequest->databaseNames = (char**)
2417                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2418             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2419                                                            backend_db);
2420
2421             // query transformation
2422             Z_Query *query = (Z_Query *)
2423                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2424             z_searchRequest->query = query;
2425
2426             if (srw_req->query_type == Z_SRW_query_type_cql)
2427             {
2428                 Z_External *ext = (Z_External *)
2429                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2430                 ext->direct_reference =
2431                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2432                 ext->indirect_reference = 0;
2433                 ext->descriptor = 0;
2434                 ext->which = Z_External_CQL;
2435                 ext->u.cql = srw_req->query.cql;
2436
2437                 query->which = Z_Query_type_104;
2438                 query->u.type_104 =  ext;
2439             }
2440             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2441             {
2442                 Z_RPNQuery *RPNquery;
2443                 YAZ_PQF_Parser pqf_parser;
2444
2445                 pqf_parser = yaz_pqf_create ();
2446
2447                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2448                                           srw_req->query.pqf);
2449                 if (!RPNquery)
2450                 {
2451                     const char *pqf_msg;
2452                     size_t off;
2453                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2454                     yaz_log(YLOG_LOG, "%*s^\n", off+4, "");
2455                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2456
2457                     send_to_srw_client_error(10, 0);
2458                     return;
2459                 }
2460                 query->which = Z_Query_type_1;
2461                 query->u.type_1 =  RPNquery;
2462
2463                 yaz_pqf_destroy (pqf_parser);
2464             }
2465             else
2466             {
2467                 send_to_srw_client_error(7, "query");
2468                 return;
2469             }
2470
2471             // present
2472             m_s2z_present_apdu = 0;
2473             int max = 0;
2474             if (srw_req->maximumRecords)
2475                 max = *srw_req->maximumRecords;
2476             int start = 1;
2477             if (srw_req->startRecord)
2478                 start = *srw_req->startRecord;
2479             if (max > 0)
2480             {
2481                 // Some backend, such as Voyager doesn't honor piggyback
2482                 // So we use present always (0 &&).
2483                 if (0 && start <= 1)  // Z39.50 piggyback
2484                 {
2485                     *z_searchRequest->smallSetUpperBound = max;
2486                     *z_searchRequest->mediumSetPresentNumber = max;
2487                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2488
2489                     z_searchRequest->preferredRecordSyntax =
2490                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2491                                                VAL_TEXT_XML);
2492                     if (srw_req->recordSchema)
2493                     {
2494                         z_searchRequest->smallSetElementSetNames =
2495                             z_searchRequest->mediumSetElementSetNames =
2496                             mk_esn_from_schema(m_s2z_odr_search,
2497                                                srw_req->recordSchema);
2498                     }
2499                 }
2500                 else   // Z39.50 present
2501                 {
2502                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2503                                                    Z_APDU_presentRequest);
2504                     Z_PresentRequest *z_presentRequest =
2505                         m_s2z_present_apdu->u.presentRequest;
2506                     *z_presentRequest->resultSetStartPoint = start;
2507                     *z_presentRequest->numberOfRecordsRequested = max;
2508                     z_presentRequest->preferredRecordSyntax =
2509                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2510                                                VAL_TEXT_XML);
2511                     if (srw_req->recordSchema)
2512                     {
2513                         z_presentRequest->recordComposition =
2514                             (Z_RecordComposition *)
2515                             odr_malloc(m_s2z_odr_search,
2516                                        sizeof(Z_RecordComposition));
2517                         z_presentRequest->recordComposition->which =
2518                             Z_RecordComp_simple;
2519                         z_presentRequest->recordComposition->u.simple =
2520                             mk_esn_from_schema(m_s2z_odr_search,
2521                                                srw_req->recordSchema);
2522                     }
2523                 }
2524             }
2525             if (!m_client)
2526             {
2527                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2528                                             Z_APDU_initRequest);
2529
2530                 // prevent m_initRequest_apdu memory from being grabbed
2531                 // in Yaz_Proxy::handle_incoming_Z_PDU
2532                 m_initRequest_apdu = m_s2z_init_apdu;
2533                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2534                 return;
2535             }
2536             else
2537             {
2538                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2539                 return;
2540             }
2541         }
2542         else if (srw_pdu->which == Z_SRW_explain_request)
2543         {
2544             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2545
2546             const char *backend_db = srw_req->database;
2547             srw_get_client(srw_req->database, &backend_db);
2548
2549             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2550
2551             // save stylesheet
2552             if (srw_req->stylesheet)
2553                 m_s2z_stylesheet =
2554                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2555
2556             if (srw_req->recordPacking &&
2557                 !strcmp(srw_req->recordPacking, "xml"))
2558                 m_s2z_packing = Z_SRW_recordPacking_XML;
2559             else
2560                 m_s2z_packing = Z_SRW_recordPacking_string;
2561
2562             if (num_diagnostic)
2563             {
2564                 send_srw_explain_response(diagnostic, num_diagnostic);
2565                 return;
2566             }
2567
2568             if (!m_client)
2569             {
2570                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2571                                             Z_APDU_initRequest);
2572
2573                 // prevent m_initRequest_apdu memory from being grabbed
2574                 // in Yaz_Proxy::handle_incoming_Z_PDU
2575                 m_initRequest_apdu = m_s2z_init_apdu;
2576                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2577             }
2578             else
2579                 send_srw_explain_response(0, 0);
2580             return;
2581         }
2582         else if (srw_pdu->which == Z_SRW_scan_request)
2583         {
2584             m_s2z_database = odr_strdup(m_s2z_odr_init,
2585                                         srw_pdu->u.scan_request->database);
2586
2587             yaz_add_srw_diagnostic(odr_decode(),
2588                                    &diagnostic, &num_diagnostic,
2589                                    4, "scan");
2590             Z_SRW_PDU *srw_pdu =
2591                 yaz_srw_get(odr_encode(),
2592                             Z_SRW_scan_response);
2593             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2594
2595             srw_res->diagnostics = diagnostic;
2596             srw_res->num_diagnostics = num_diagnostic;
2597             send_srw_response(srw_pdu);
2598             return;
2599         }
2600         else
2601         {
2602             m_s2z_database = 0;
2603
2604             send_to_srw_client_error(4, 0);
2605         }
2606     }
2607     send_http_response(400);
2608 }
2609
2610 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2611 {
2612     Z_ReferenceId **refid = get_referenceIdP(apdu);
2613     nmem_reset(m_referenceId_mem);
2614     if (refid && *refid)
2615     {
2616         m_referenceId = (Z_ReferenceId *)
2617             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2618         m_referenceId->len = m_referenceId->size = (*refid)->len;
2619         m_referenceId->buf = (unsigned char *)
2620             nmem_malloc(m_referenceId_mem, (*refid)->len);
2621         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2622     }
2623     else
2624         m_referenceId = 0;
2625
2626     if (!m_client && m_invalid_session)
2627     {
2628         m_apdu_invalid_session = apdu;
2629         m_mem_invalid_session = odr_extract_mem(odr_decode());
2630         apdu = m_initRequest_apdu;
2631     }
2632
2633     // Determine our client.
2634     Z_OtherInformation **oi;
2635     get_otherInfoAPDU(apdu, &oi);
2636     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2637     if (!m_client)
2638     {
2639         if (m_http_version)
2640         {
2641             send_http_response(404);
2642             return;
2643         }
2644         else
2645         {
2646             delete this;
2647             return;
2648         }
2649     }
2650
2651     m_client->m_server = this;
2652
2653     if (apdu->which == Z_APDU_initRequest)
2654     {
2655         if (apdu->u.initRequest->implementationId)
2656             yaz_log(YLOG_LOG, "%simplementationId: %s",
2657                     m_session_str, apdu->u.initRequest->implementationId);
2658         if (apdu->u.initRequest->implementationName)
2659             yaz_log(YLOG_LOG, "%simplementationName: %s",
2660                     m_session_str, apdu->u.initRequest->implementationName);
2661         if (apdu->u.initRequest->implementationVersion)
2662             yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2663                     m_session_str, apdu->u.initRequest->implementationVersion);
2664         if (m_initRequest_apdu == 0)
2665         {
2666             if (m_initRequest_mem)
2667                 nmem_destroy(m_initRequest_mem);
2668
2669             m_initRequest_apdu = apdu;
2670             m_initRequest_mem = odr_extract_mem(odr_decode());
2671
2672             m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2673                 preferredMessageSize;
2674             *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2675             m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2676                 maximumRecordSize;
2677             *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2678
2679             // Save proposal charsets and langs.
2680             if (ODR_MASK_GET(apdu->u.initRequest->options,
2681                 Z_Options_negotiationModel))
2682             {
2683                 Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2684                     yaz_get_charneg_record(*oi);
2685
2686                 yaz_get_proposal_charneg(m_referenceId_mem,
2687                     charSetandLangRecord,
2688                     &m_initRequest_oi_negotiation_charsets,
2689                     &m_initRequest_oi_negotiation_num_charsets,
2690                     &m_initRequest_oi_negotiation_langs,
2691                     &m_initRequest_oi_negotiation_num_langs,
2692                     &m_initRequest_oi_negotiation_selected);
2693
2694                 for (int i=0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2695                 {
2696                     yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2697                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2698                         m_initRequest_oi_negotiation_charsets[i]:"none");
2699                 }
2700                 for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2701                 {
2702                     yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2703                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2704                         m_initRequest_oi_negotiation_langs[i]:"none");
2705                 }
2706                 yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2707                     m_session_str, m_initRequest_oi_negotiation_selected);
2708             }
2709             // save init options for the response..
2710             m_initRequest_options = apdu->u.initRequest->options;
2711
2712             apdu->u.initRequest->options =
2713                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2714                                            sizeof(Odr_bitmask));
2715             ODR_MASK_ZERO(apdu->u.initRequest->options);
2716             int i;
2717             for (i = 0; i<= 24; i++)
2718                 ODR_MASK_SET(apdu->u.initRequest->options, i);
2719             // check negotiation option
2720             if (!ODR_MASK_GET(m_initRequest_options,
2721                 Z_Options_negotiationModel))
2722             {
2723                 ODR_MASK_CLEAR(apdu->u.initRequest->options,
2724                     Z_Options_negotiationModel);
2725             }
2726             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2727                 Z_Options_concurrentOperations);
2728             // make new version
2729             m_initRequest_version = apdu->u.initRequest->protocolVersion;
2730             apdu->u.initRequest->protocolVersion =
2731                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2732                                            sizeof(Odr_bitmask));
2733             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2734
2735             for (i = 0; i<= 8; i++)
2736                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2737         }
2738         if (m_client->m_init_flag)
2739         {
2740             if (handle_init_response_for_invalid_session(apdu))
2741                 return;
2742             if (m_client->m_initResponse)
2743             {
2744                 Z_APDU *apdu2 = m_client->m_initResponse;
2745                 apdu2->u.initResponse->otherInfo = 0;
2746                 if (m_client->m_cookie && *m_client->m_cookie)
2747                     set_otherInformationString(apdu2, VAL_COOKIE, 1,
2748                                                m_client->m_cookie);
2749                 apdu2->u.initResponse->referenceId =
2750                     apdu->u.initRequest->referenceId;
2751                 apdu2->u.initResponse->options = m_client->m_initResponse_options;
2752                 apdu2->u.initResponse->protocolVersion =
2753                     m_client->m_initResponse_version;
2754
2755                 send_to_client(apdu2);
2756                 return;
2757             }
2758         }
2759         m_client->m_init_flag = 1;
2760     }
2761
2762     if (!handle_authentication(apdu))
2763     {
2764         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
2765         *apdu_reject->u.initResponse->result = 0;
2766         send_to_client(apdu_reject);
2767
2768         shutdown();
2769         return;
2770     }
2771
2772     handle_max_record_retrieve(apdu);
2773
2774     if (apdu)
2775         apdu = handle_syntax_validation(apdu);
2776
2777     if (apdu)
2778         apdu = handle_query_transformation(apdu);
2779
2780     if (apdu)
2781         apdu = handle_target_charset_conversion(apdu);
2782
2783     if (apdu)
2784         apdu = handle_query_validation(apdu);
2785
2786     if (apdu)
2787         apdu = result_set_optimize(apdu);
2788
2789     if (!apdu)
2790     {
2791         m_client->timeout(m_target_idletime);  // mark it active even
2792         // though we didn't use it
2793         return;
2794     }
2795     // Add otherInformation entry in APDU if
2796     // negotiatoin in use.
2797     if (apdu)
2798         handle_charset_lang_negotiation(apdu);
2799
2800     // delete other info construct completely if 0 elements
2801     get_otherInfoAPDU(apdu, &oi);
2802     if (oi && *oi && (*oi)->num_elements == 0)
2803         *oi = 0;
2804
2805     if (apdu->which == Z_APDU_presentRequest &&
2806         m_client->m_resultSetStartPoint == 0)
2807     {
2808         Z_PresentRequest *pr = apdu->u.presentRequest;
2809         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2810         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2811     } else {
2812         m_client->m_resultSetStartPoint = 0;
2813     }
2814     if (m_client->send_to_target(apdu) < 0)
2815     {
2816         delete m_client;
2817         m_client = 0;
2818         delete this;
2819     }
2820     else
2821         m_client->m_waiting = 1;
2822 }
2823
2824 void Yaz_Proxy::connectNotify()
2825 {
2826 }
2827
2828 void Yaz_Proxy::releaseClient()
2829 {
2830     xfree(m_proxyTarget);
2831     m_proxyTarget = 0;
2832     m_invalid_session = 0;
2833     // only keep if keep_alive flag is set...
2834     if (m_client &&
2835         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2836         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2837         m_client->m_waiting == 0)
2838     {
2839         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2840                  m_session_str,
2841                  m_client->get_hostname());
2842         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2843                 m_session_str, m_client->m_pdu_recv,
2844                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2845                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2846         assert (m_client->m_waiting != 2);
2847         // Tell client (if any) that no server connection is there..
2848         m_client->m_server = 0;
2849         m_client = 0;
2850     }
2851     else if (m_client)
2852     {
2853         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
2854                  m_session_str,
2855                  m_client->get_hostname());
2856         assert (m_client->m_waiting != 2);
2857         delete m_client;
2858         m_client = 0;
2859     }
2860     else if (!m_parent)
2861     {
2862         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
2863                  m_session_str);
2864         assert (m_parent);
2865     }
2866     else
2867     {
2868         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
2869                  m_session_str);
2870     }
2871     if (m_parent)
2872         m_parent->pre_init();
2873 }
2874
2875 void Yaz_Proxy::shutdown()
2876 {
2877     releaseClient();
2878     delete this;
2879 }
2880
2881 const char *Yaz_ProxyClient::get_session_str()
2882 {
2883     if (!m_server)
2884         return "0 ";
2885     return m_server->get_session_str();
2886 }
2887
2888 void Yaz_ProxyClient::shutdown()
2889 {
2890     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2891              get_hostname());
2892     delete m_server;
2893     delete this;
2894 }
2895
2896 void Yaz_Proxy::failNotify()
2897 {
2898     inc_request_no();
2899     yaz_log (YLOG_LOG, "%sConnection closed by client",
2900              get_session_str());
2901     shutdown();
2902 }
2903
2904 void Yaz_ProxyClient::failNotify()
2905 {
2906     if (m_server)
2907         m_server->inc_request_no();
2908     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
2909              get_session_str(), get_hostname());
2910     shutdown();
2911 }
2912
2913 void Yaz_ProxyClient::connectNotify()
2914 {
2915     const char *s = get_session_str();
2916     const char *h = get_hostname();
2917     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2918              m_target_idletime);
2919     timeout(m_target_idletime);
2920     if (!m_server)
2921         pre_init_client();
2922 }
2923
2924 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2925                                                   *the_PDU_Observable, int fd)
2926 {
2927     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2928 }
2929
2930 Yaz_ProxyClient::~Yaz_ProxyClient()
2931 {
2932     if (m_prev)
2933         *m_prev = m_next;
2934     if (m_next)
2935         m_next->m_prev = m_prev;
2936     m_waiting = 2;     // for debugging purposes only.
2937     odr_destroy(m_init_odr);
2938     delete m_last_query;
2939     xfree (m_last_resultSetId);
2940     xfree (m_cookie);
2941 }
2942
2943 void Yaz_ProxyClient::pre_init_client()
2944 {
2945     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2946     Z_InitRequest *req = apdu->u.initRequest;
2947
2948     int i;
2949     for (i = 0; i<= 24; i++)
2950         ODR_MASK_SET(req->options, i);
2951     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2952                    Z_Options_negotiationModel);
2953     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2954                    Z_Options_concurrentOperations);
2955     for (i = 0; i<= 10; i++)
2956         ODR_MASK_SET(req->protocolVersion, i);
2957
2958     if (send_to_target(apdu) < 0)
2959     {
2960         delete this;
2961     }
2962     else
2963     {
2964         m_waiting = 1;
2965         m_init_flag = 1;
2966     }
2967 }
2968
2969 void Yaz_Proxy::pre_init()
2970 {
2971     int i;
2972     const char *name = 0;
2973     const char *zurl_in_use[MAX_ZURL_PLEX];
2974     int limit_bw, limit_pdu, limit_req;
2975     int target_idletime, client_idletime;
2976     int max_clients;
2977     int keepalive_limit_bw, keepalive_limit_pdu;
2978     int pre_init;
2979     const char *cql2rpn = 0;
2980     const char *authentication = 0;
2981     const char *negotiation_charset = 0;
2982     const char *negotiation_lang = 0;
2983
2984     Yaz_ProxyConfig *cfg = check_reconfigure();
2985
2986     zurl_in_use[0] = 0;
2987
2988     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2989         set_APDU_yazlog(1);
2990     else
2991         set_APDU_yazlog(0);
2992
2993     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2994                                           &limit_bw, &limit_pdu, &limit_req,
2995                                           &target_idletime, &client_idletime,
2996                                           &max_clients,
2997                                           &keepalive_limit_bw,
2998                                           &keepalive_limit_pdu,
2999                                           &pre_init,
3000                                           &cql2rpn,
3001                                           &authentication,
3002                                           &negotiation_charset,
3003                                           &negotiation_lang,
3004                                           0) ; i++)
3005     {
3006         if (pre_init)
3007         {
3008             int j;
3009             for (j = 0; zurl_in_use[j]; j++)
3010             {
3011                 Yaz_ProxyClient *c;
3012                 int spare = 0;
3013                 int spare_waiting = 0;
3014                 int in_use = 0;
3015                 int other = 0;
3016                 for (c = m_clientPool; c; c = c->m_next)
3017                 {
3018                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3019                     {
3020                         if (c->m_cookie == 0)
3021                         {
3022                             if (c->m_server == 0)
3023                                 if (c->m_waiting)
3024                                     spare_waiting++;
3025                                 else
3026                                     spare++;
3027                             else
3028                                 in_use++;
3029                         }
3030                         else
3031                             other++;
3032                     }
3033                 }
3034                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3035                         "sparew=%d preinit=%d",m_session_str,
3036                         name, zurl_in_use[j], in_use, other,
3037                         spare, spare_waiting, pre_init);
3038                 if (spare + spare_waiting < pre_init)
3039                 {
3040                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3041                     c->m_next = m_clientPool;
3042                     if (c->m_next)
3043                         c->m_next->m_prev = &c->m_next;
3044                     m_clientPool = c;
3045                     c->m_prev = &m_clientPool;
3046
3047                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3048                         c->set_APDU_yazlog(1);
3049                     else
3050                         c->set_APDU_yazlog(0);
3051
3052                     if (c->client(zurl_in_use[j]))
3053                     {
3054                         timeout(60);
3055                         delete c;
3056                         return;
3057                     }
3058                     c->timeout(30);
3059                     c->m_waiting = 1;
3060                     c->m_target_idletime = target_idletime;
3061                     c->m_seqno = m_seqno++;
3062                 }
3063             }
3064         }
3065     }
3066 }
3067
3068 void Yaz_Proxy::timeoutNotify()
3069 {
3070     if (m_parent)
3071     {
3072         if (m_bw_hold_PDU)
3073         {
3074             timeout(m_client_idletime);
3075             Z_GDU *apdu = m_bw_hold_PDU;
3076             m_bw_hold_PDU = 0;
3077
3078             if (apdu->which == Z_GDU_Z3950)
3079                 handle_incoming_Z_PDU(apdu->u.z3950);
3080             else if (apdu->which == Z_GDU_HTTP_Request)
3081                 handle_incoming_HTTP(apdu->u.HTTP_Request);
3082         }
3083         else if (m_stylesheet_nprl)
3084             convert_xsl_delay();
3085         else
3086         {
3087             inc_request_no();
3088
3089             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3090             shutdown();
3091         }
3092     }
3093     else
3094     {
3095         timeout(600);
3096         pre_init();
3097     }
3098 }
3099
3100 void Yaz_Proxy::markInvalid()
3101 {
3102     m_client = 0;
3103     m_invalid_session = 1;
3104 }
3105
3106 void Yaz_ProxyClient::timeoutNotify()
3107 {
3108     if (m_server)
3109         m_server->inc_request_no();
3110
3111     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3112              get_hostname());
3113     m_waiting = 1;
3114     m_root->pre_init();
3115     if (m_server && m_init_flag)
3116     {
3117         // target timed out in a session that was properly initialized
3118         // server object stay alive but we mark it as invalid so it
3119         // gets initialized again
3120         m_server->markInvalid();
3121         m_server = 0;
3122     }
3123     shutdown();
3124 }
3125
3126 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
3127                                  Yaz_Proxy *parent) :
3128     Yaz_Z_Assoc (the_PDU_Observable)
3129 {
3130     m_cookie = 0;
3131     m_next = 0;
3132     m_prev = 0;
3133     m_init_flag = 0;
3134     m_last_query = 0;
3135     m_last_resultSetId = 0;
3136     m_last_resultCount = 0;
3137     m_last_ok = 0;
3138     m_sr_transform = 0;
3139     m_waiting = 0;
3140     m_init_odr = odr_createmem (ODR_DECODE);
3141     m_initResponse = 0;
3142     m_initResponse_options = 0;
3143     m_initResponse_version = 0;
3144     m_initResponse_preferredMessageSize = 0;
3145     m_initResponse_maximumRecordSize = 0;
3146     m_resultSetStartPoint = 0;
3147     m_bytes_sent = m_bytes_recv = 0;
3148     m_pdu_recv = 0;
3149     m_server = 0;
3150     m_seqno = 0;
3151     m_target_idletime = 600;
3152     m_root = parent;
3153 }
3154
3155 const char *Yaz_Proxy::option(const char *name, const char *value)
3156 {
3157     if (!strcmp (name, "optimize")) {
3158         if (value) {
3159             xfree (m_optimize);
3160             m_optimize = xstrdup (value);
3161         }
3162         return m_optimize;
3163     }
3164     return 0;
3165 }
3166
3167 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
3168 {
3169
3170 }
3171
3172 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
3173 {
3174     if (apdu->which == Z_GDU_Z3950)
3175         recv_Z_PDU(apdu->u.z3950, len);
3176     else if (apdu->which == Z_GDU_HTTP_Response)
3177         recv_HTTP_response(apdu->u.HTTP_Response, len);
3178     else
3179         shutdown();
3180 }
3181
3182 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
3183 {
3184     if (!m_invalid_session)
3185         return 0;
3186     m_invalid_session = 0;
3187     handle_incoming_Z_PDU(m_apdu_invalid_session);
3188     assert (m_mem_invalid_session);
3189     nmem_destroy(m_mem_invalid_session);
3190     m_mem_invalid_session = 0;
3191     return 1;
3192 }
3193
3194 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
3195 {
3196     m_bytes_recv += len;
3197
3198     m_pdu_recv++;
3199     m_waiting = 0;
3200     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
3201         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
3202                  apdu_name(apdu), get_hostname(), len);
3203     if (apdu->which == Z_APDU_initResponse)
3204     {
3205         if (!m_server)  // if this is a pre init session , check for more
3206             m_root->pre_init();
3207         NMEM nmem = odr_extract_mem (odr_decode());
3208         odr_reset (m_init_odr);
3209         nmem_transfer (m_init_odr->mem, nmem);
3210         m_initResponse = apdu;
3211         m_initResponse_options = apdu->u.initResponse->options;
3212         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3213         m_initResponse_preferredMessageSize =
3214             *apdu->u.initResponse->preferredMessageSize;
3215         m_initResponse_maximumRecordSize =
3216             *apdu->u.initResponse->maximumRecordSize;
3217
3218         Z_InitResponse *ir = apdu->u.initResponse;
3219         char *im0 = ir->implementationName;
3220
3221         char *im1 = (char*)
3222             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3223         *im1 = '\0';
3224         if (im0)
3225         {
3226             strcat(im1, im0);
3227             strcat(im1, " ");
3228         }
3229         strcat(im1, "(YAZ Proxy)");
3230         ir->implementationName = im1;
3231
3232         nmem_destroy (nmem);
3233
3234         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3235             return;
3236     }
3237     if (apdu->which == Z_APDU_searchResponse)
3238     {
3239         Z_SearchResponse *sr = apdu->u.searchResponse;
3240         m_last_resultCount = *sr->resultCount;
3241         int status = *sr->searchStatus;
3242         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3243         {
3244             m_last_ok = 1;
3245
3246             if (sr->records && sr->records->which == Z_Records_DBOSD)
3247             {
3248                 m_cache.add(odr_decode(),
3249                             sr->records->u.databaseOrSurDiagnostics, 1,
3250                             *sr->resultCount);
3251             }
3252         }
3253     }
3254     if (apdu->which == Z_APDU_presentResponse)
3255     {
3256         Z_PresentResponse *pr = apdu->u.presentResponse;
3257         if (m_sr_transform)
3258         {
3259             m_sr_transform = 0;
3260             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3261             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3262             sr->referenceId = pr->referenceId;
3263             *sr->resultCount = m_last_resultCount;
3264             sr->records = pr->records;
3265             sr->nextResultSetPosition = pr->nextResultSetPosition;
3266             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3267             apdu = new_apdu;
3268         }
3269         if (pr->records &&
3270             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3271         {
3272             m_cache.add(odr_decode(),
3273                         pr->records->u.databaseOrSurDiagnostics,
3274                         m_resultSetStartPoint, -1);
3275             m_resultSetStartPoint = 0;
3276         }
3277     }
3278     if (m_cookie)
3279         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3280     if (m_server)
3281     {
3282         m_server->send_to_client(apdu);
3283     }
3284     if (apdu->which == Z_APDU_close)
3285     {
3286         shutdown();
3287     }
3288 }
3289
3290 void Yaz_Proxy::low_socket_close()
3291 {
3292 #if WIN32
3293 #else
3294     int i;
3295     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3296         if  (m_lo_fd[i] >= 0)
3297             ::close(m_lo_fd[i]);
3298 #endif
3299 }
3300
3301 void Yaz_Proxy::low_socket_open()
3302 {
3303 #if WIN32
3304 #else
3305     int i;
3306     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3307         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3308 #endif
3309 }
3310
3311 int Yaz_Proxy::server(const char *addr)
3312 {
3313     int r = Yaz_Z_Assoc::server(addr);
3314     if (!r)
3315     {
3316         yaz_log(YLOG_LOG, "%sStarted proxy "
3317 #ifdef VERSION
3318             VERSION
3319 #endif
3320             " on %s", m_session_str, addr);
3321         timeout(1);
3322     }
3323     return r;
3324 }
3325