src/yaz-proxy.cpp

   1 /*
   2  * Copyright (c) 1998-2004, Index Data.
   3  * See the file LICENSE for details.
   4  *
   5  * $Id: yaz-proxy.cpp,v 1.93 2004-01-30 00:38:28 adam Exp $
   6  */
   7
   8 #include <assert.h>
   9 #include <time.h>
  10
  11 #include <yaz/srw.h>
  12 #include <yaz/marcdisp.h>
  13 #include <yaz/yaz-iconv.h>
  14 #include <yaz/log.h>
  15 #include <yaz/diagbib1.h>
  16 #include <yaz++/proxy.h>
  17 #include <yaz/pquery.h>
  18
  19 #if HAVE_XSLT
  20 #include <libxslt/xsltutils.h>
  21 #include <libxslt/transform.h>
  22 #endif
  23
  24 static const char *apdu_name(Z_APDU *apdu)
  25 {
  26     switch (apdu->which)
  27     {
  28     case Z_APDU_initRequest:
  29         return "initRequest";
  30     case Z_APDU_initResponse:
  31         return "initResponse";
  32     case Z_APDU_searchRequest:
  33         return "searchRequest";
  34     case Z_APDU_searchResponse:
  35         return "searchResponse";
  36     case Z_APDU_presentRequest:
  37         return "presentRequest";
  38     case Z_APDU_presentResponse:
  39         return "presentResponse";
  40     case Z_APDU_deleteResultSetRequest:
  41         return "deleteResultSetRequest";
  42     case Z_APDU_deleteResultSetResponse:
  43         return "deleteResultSetResponse";
  44     case Z_APDU_scanRequest:
  45         return "scanRequest";
  46     case Z_APDU_scanResponse:
  47         return "scanResponse";
  48     case Z_APDU_sortRequest:
  49         return "sortRequest";
  50     case Z_APDU_sortResponse:
  51         return "sortResponse";
  52     case Z_APDU_extendedServicesRequest:
  53         return "extendedServicesRequest";
  54     case Z_APDU_extendedServicesResponse:
  55         return "extendedServicesResponse";
  56     case Z_APDU_close:
  57         return "close";
  58     }
  59     return "other";
  60 }
  61
  62 static const char *gdu_name(Z_GDU *gdu)
  63 {
  64     switch(gdu->which)
  65     {
  66     case Z_GDU_Z3950:
  67         return apdu_name(gdu->u.z3950);
  68     case Z_GDU_HTTP_Request:
  69         return "HTTP Request";
  70     case Z_GDU_HTTP_Response:
  71         return "HTTP Response";
  72     }
  73     return "Unknown request/response";
  74 }
  75
  76 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
  77                      Yaz_Proxy *parent) :
  78     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
  79 {
  80     m_PDU_Observable = the_PDU_Observable;
  81     m_client = 0;
  82     m_parent = parent;
  83     m_clientPool = 0;
  84     m_seqno = 1;
  85     m_keepalive_limit_bw = 500000;
  86     m_keepalive_limit_pdu = 1000;
  87     m_proxyTarget = 0;
  88     m_default_target = 0;
  89     m_proxy_authentication = 0;
  90     m_max_clients = 150;
  91     m_log_mask = 0;
  92     m_seed = time(0);
  93     m_client_idletime = 600;
  94     m_target_idletime = 600;
  95     m_optimize = xstrdup ("1");
  96     strcpy(m_session_str, "0 ");
  97     m_session_no=0;
  98     m_bytes_sent = m_bytes_recv = 0;
  99     m_bw_hold_PDU = 0;
 100     m_bw_max = 0;
 101     m_pdu_max = 0;
 102     m_max_record_retrieve = 0;
 103     m_reconfig_flag = 0;
 104     m_config_fname = 0;
 105     m_request_no = 0;
 106     m_invalid_session = 0;
 107     m_config = 0;
 108     m_marcxml_flag = 0;
 109     m_stylesheet_schema = 0;
 110     m_s2z_stylesheet = 0;
 111     m_s2z_database = 0;
 112     m_schema = 0;
 113     m_initRequest_apdu = 0;
 114     m_initRequest_mem = 0;
 115     m_initRequest_options = 0;
 116     m_initRequest_version = 0;
 117     m_apdu_invalid_session = 0;
 118     m_mem_invalid_session = 0;
 119     m_s2z_odr_init = 0;
 120     m_s2z_odr_search = 0;
 121     m_s2z_init_apdu = 0;
 122     m_s2z_search_apdu = 0;
 123     m_s2z_present_apdu = 0;
 124     m_http_keepalive = 0;
 125     m_http_version = 0;
 126     m_soap_ns = 0;
 127     m_s2z_packing = Z_SRW_recordPacking_string;
 128     m_time_tv.tv_sec = 0;
 129     m_time_tv.tv_usec = 0;
 130 }
 131
 132 Yaz_Proxy::~Yaz_Proxy()
 133 {
 134     yaz_log(LOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 135             m_bytes_sent, m_bytes_recv);
 136     nmem_destroy(m_initRequest_mem);
 137     nmem_destroy(m_mem_invalid_session);
 138     xfree (m_proxyTarget);
 139     xfree (m_default_target);
 140     xfree (m_proxy_authentication);
 141     xfree (m_optimize);
 142     xfree (m_stylesheet_schema);
 143     xfree (m_schema);
 144     if (m_s2z_odr_init)
 145         odr_destroy(m_s2z_odr_init);
 146     if (m_s2z_odr_search)
 147         odr_destroy(m_s2z_odr_search);
 148     delete m_config;
 149 }
 150
 151 int Yaz_Proxy::set_config(const char *config)
 152 {
 153     delete m_config;
 154     m_config = new Yaz_ProxyConfig();
 155     xfree(m_config_fname);
 156     m_config_fname = xstrdup(config);
 157     int r = m_config->read_xml(config);
 158     if (!r)
 159         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 160     return r;
 161 }
 162
 163 void Yaz_Proxy::set_default_target(const char *target)
 164 {
 165     xfree (m_default_target);
 166     m_default_target = 0;
 167     if (target)
 168         m_default_target = (char *) xstrdup (target);
 169 }
 170
 171 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 172 {
 173     xfree (m_proxy_authentication);
 174     m_proxy_authentication = 0;
 175     if (auth)
 176         m_proxy_authentication = (char *) xstrdup (auth);
 177 }
 178
 179 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 180 {
 181     if (m_parent)
 182         return m_parent->check_reconfigure();
 183
 184     Yaz_ProxyConfig *cfg = m_config;
 185     if (m_reconfig_flag)
 186     {
 187         yaz_log(LOG_LOG, "reconfigure");
 188         yaz_log_reopen();
 189         if (m_config_fname && cfg)
 190         {
 191             yaz_log(LOG_LOG, "reconfigure config %s", m_config_fname);
 192             int r = cfg->read_xml(m_config_fname);
 193             if (r)
 194                 yaz_log(LOG_WARN, "reconfigure failed");
 195             else
 196             {
 197                 m_log_mask = 0;
 198                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 199             }
 200         }
 201         else
 202             yaz_log(LOG_LOG, "reconfigure");
 203         m_reconfig_flag = 0;
 204     }
 205     return cfg;
 206 }
 207
 208 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 209                                             *the_PDU_Observable, int fd)
 210 {
 211     check_reconfigure();
 212     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 213     new_proxy->m_config = 0;
 214     new_proxy->m_config_fname = 0;
 215     new_proxy->timeout(m_client_idletime);
 216     new_proxy->m_target_idletime = m_target_idletime;
 217     new_proxy->set_default_target(m_default_target);
 218     new_proxy->m_max_clients = m_max_clients;
 219     new_proxy->m_log_mask = m_log_mask;
 220     new_proxy->set_APDU_log(get_APDU_log());
 221     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 222         new_proxy->set_APDU_yazlog(1);
 223     else
 224         new_proxy->set_APDU_yazlog(0);
 225     new_proxy->set_proxy_authentication(m_proxy_authentication);
 226     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 227     m_session_no++;
 228     yaz_log (LOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 229              the_PDU_Observable->getpeername());
 230     return new_proxy;
 231 }
 232
 233 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 234 {
 235     int oid[OID_SIZE];
 236     Z_OtherInformationUnit *oi;
 237     struct oident ent;
 238     ent.proto = PROTO_Z3950;
 239     ent.oclass = CLASS_USERINFO;
 240     ent.value = (oid_value) VAL_COOKIE;
 241     assert (oid_ent_to_oid (&ent, oid));
 242
 243     if (oid_ent_to_oid (&ent, oid) &&
 244         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 245         oi->which == Z_OtherInfo_characterInfo)
 246         return oi->information.characterInfo;
 247     return 0;
 248 }
 249
 250 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 251 {
 252     int oid[OID_SIZE];
 253     Z_OtherInformationUnit *oi;
 254     struct oident ent;
 255     ent.proto = PROTO_Z3950;
 256     ent.oclass = CLASS_USERINFO;
 257     ent.value = (oid_value) VAL_PROXY;
 258     if (oid_ent_to_oid (&ent, oid) &&
 259         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 260         oi->which == Z_OtherInfo_characterInfo)
 261         return oi->information.characterInfo;
 262     return 0;
 263 }
 264
 265 const char *Yaz_Proxy::load_balance(const char **url)
 266 {
 267     int zurl_in_use[MAX_ZURL_PLEX];
 268     Yaz_ProxyClient *c;
 269     int i;
 270
 271     for (i = 0; i<MAX_ZURL_PLEX; i++)
 272         zurl_in_use[i] = 0;
 273     for (c = m_parent->m_clientPool; c; c = c->m_next)
 274     {
 275         for (i = 0; url[i]; i++)
 276             if (!strcmp(url[i], c->get_hostname()))
 277                 zurl_in_use[i]++;
 278     }
 279     int min = 100000;
 280     const char *ret = 0;
 281     for (i = 0; url[i]; i++)
 282     {
 283         yaz_log(LOG_DEBUG, "%szurl=%s use=%d",
 284                 m_session_str, url[i], zurl_in_use[i]);
 285         if (min > zurl_in_use[i])
 286         {
 287             ret = url[i];
 288             min = zurl_in_use[i];
 289         }
 290     }
 291     return ret;
 292 }
 293
 294 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 295                                        const char *proxy_host)
 296 {
 297     assert (m_parent);
 298     Yaz_Proxy *parent = m_parent;
 299     Yaz_ProxyClient *c = m_client;
 300
 301     if (!m_proxyTarget)
 302     {
 303         const char *url[MAX_ZURL_PLEX];
 304         Yaz_ProxyConfig *cfg = check_reconfigure();
 305         if (proxy_host)
 306         {
 307 #if 1
 308 /* only to be enabled for debugging... */
 309             if (!strcmp(proxy_host, "stop"))
 310                 exit(0);
 311 #endif
 312             xfree(m_default_target);
 313             m_default_target = xstrdup(proxy_host);
 314             proxy_host = m_default_target;
 315         }
 316         int client_idletime = -1;
 317         const char *cql2rpn_fname = 0;
 318         url[0] = m_default_target;
 319         url[1] = 0;
 320         if (cfg)
 321         {
 322             int pre_init = 0;
 323             cfg->get_target_info(proxy_host, url, &m_bw_max,
 324                                  &m_pdu_max, &m_max_record_retrieve,
 325                                  &m_target_idletime, &client_idletime,
 326                                  &parent->m_max_clients,
 327                                  &m_keepalive_limit_bw,
 328                                  &m_keepalive_limit_pdu,
 329                                  &pre_init,
 330                                  &cql2rpn_fname);
 331         }
 332         if (client_idletime != -1)
 333         {
 334             m_client_idletime = client_idletime;
 335             timeout(m_client_idletime);
 336         }
 337         if (cql2rpn_fname)
 338             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 339         if (!url[0])
 340         {
 341             yaz_log(LOG_LOG, "%sNo default target", m_session_str);
 342             return 0;
 343         }
 344         // we don't handle multiplexing for cookie session, so we just
 345         // pick the first one in this case (anonymous users will be able
 346         // to use any backend)
 347         if (cookie && *cookie)
 348             m_proxyTarget = (char*) xstrdup(url[0]);
 349         else
 350             m_proxyTarget = (char*) xstrdup(load_balance(url));
 351     }
 352     if (cookie && *cookie)
 353     {   // search in sessions with a cookie
 354         for (c = parent->m_clientPool; c; c = c->m_next)
 355         {
 356             assert (c->m_prev);
 357             assert (*c->m_prev == c);
 358             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 359                 !strcmp(m_proxyTarget, c->get_hostname()))
 360             {
 361                 // Found it in cache
 362                 // The following handles "cancel"
 363                 // If connection is busy (waiting for PDU) and
 364                 // we have an initRequest we can safely do re-open
 365                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 366                 {
 367                     yaz_log (LOG_LOG, "%s REOPEN target=%s", m_session_str,
 368                              c->get_hostname());
 369                     c->close();
 370                     c->m_init_flag = 0;
 371
 372                     c->m_last_ok = 0;
 373                     c->m_cache.clear();
 374                     c->m_last_resultCount = 0;
 375                     c->m_sr_transform = 0;
 376                     c->m_waiting = 0;
 377                     c->m_resultSetStartPoint = 0;
 378                     c->m_target_idletime = m_target_idletime;
 379                     if (c->client(m_proxyTarget))
 380                     {
 381                         delete c;
 382                         return 0;
 383                     }
 384                     c->timeout(30);
 385                 }
 386                 c->m_seqno = parent->m_seqno;
 387                 if (c->m_server && c->m_server != this)
 388                     c->m_server->m_client = 0;
 389                 c->m_server = this;
 390                 (parent->m_seqno)++;
 391                 yaz_log (LOG_DEBUG, "get_client 1 %p %p", this, c);
 392                 return c;
 393             }
 394         }
 395     }
 396     else if (!c)
 397     {
 398         // don't have a client session yet. Search in session w/o cookie
 399         for (c = parent->m_clientPool; c; c = c->m_next)
 400         {
 401             assert (c->m_prev);
 402             assert (*c->m_prev == c);
 403             if (c->m_server == 0 && c->m_cookie == 0 &&
 404                 c->m_waiting == 0 &&
 405                 !strcmp(m_proxyTarget, c->get_hostname()))
 406             {
 407                 // found it in cache
 408                 yaz_log (LOG_LOG, "%sREUSE %s",
 409                          m_session_str, c->get_hostname());
 410
 411                 c->m_seqno = parent->m_seqno;
 412                 assert(c->m_server == 0);
 413                 c->m_server = this;
 414
 415                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 416                     c->set_APDU_yazlog(1);
 417                 else
 418                     c->set_APDU_yazlog(0);
 419
 420                 (parent->m_seqno)++;
 421
 422                 parent->pre_init();
 423
 424                 return c;
 425             }
 426         }
 427     }
 428     if (!m_client)
 429     {
 430         if (apdu->which != Z_APDU_initRequest)
 431         {
 432             yaz_log (LOG_LOG, "%sno init request as first PDU", m_session_str);
 433             return 0;
 434         }
 435         Z_InitRequest *initRequest = apdu->u.initRequest;
 436
 437         if (!initRequest->idAuthentication)
 438         {
 439             if (m_proxy_authentication)
 440             {
 441                 initRequest->idAuthentication =
 442                     (Z_IdAuthentication *)
 443                     odr_malloc (odr_encode(),
 444                                 sizeof(*initRequest->idAuthentication));
 445                 initRequest->idAuthentication->which =
 446                     Z_IdAuthentication_open;
 447                 initRequest->idAuthentication->u.open =
 448                     odr_strdup (odr_encode(), m_proxy_authentication);
 449             }
 450         }
 451         // go through list of clients - and find the lowest/oldest one.
 452         Yaz_ProxyClient *c_min = 0;
 453         int min_seq = -1;
 454         int no_of_clients = 0;
 455         if (parent->m_clientPool)
 456             yaz_log (LOG_DEBUG, "Existing sessions");
 457         for (c = parent->m_clientPool; c; c = c->m_next)
 458         {
 459             yaz_log (LOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 460                                c->m_waiting, c->get_hostname(),
 461                                c->m_cookie ? c->m_cookie : "");
 462             no_of_clients++;
 463             if (min_seq < 0 || c->m_seqno < min_seq)
 464             {
 465                 min_seq = c->m_seqno;
 466                 c_min = c;
 467             }
 468         }
 469         if (no_of_clients >= parent->m_max_clients)
 470         {
 471             c = c_min;
 472             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 473             {
 474                 yaz_log (LOG_LOG, "%sMAXCLIENTS Destroy %d",
 475                          m_session_str, c->m_seqno);
 476                 if (c->m_server && c->m_server != this)
 477                     delete c->m_server;
 478                 c->m_server = 0;
 479             }
 480             else
 481             {
 482                 yaz_log (LOG_LOG, "%sMAXCLIENTS Reuse %d %d %s",
 483                          m_session_str,
 484                          c->m_seqno, parent->m_seqno, c->get_hostname());
 485                 xfree (c->m_cookie);
 486                 c->m_cookie = 0;
 487                 if (cookie)
 488                     c->m_cookie = xstrdup(cookie);
 489                 c->m_seqno = parent->m_seqno;
 490                 if (c->m_server && c->m_server != this)
 491                 {
 492                     c->m_server->m_client = 0;
 493                     delete c->m_server;
 494                 }
 495                 (parent->m_seqno)++;
 496                 c->m_target_idletime = m_target_idletime;
 497                 c->timeout(m_target_idletime);
 498
 499                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 500                     c->set_APDU_yazlog(1);
 501                 else
 502                     c->set_APDU_yazlog(0);
 503
 504                 return c;
 505             }
 506         }
 507         else
 508         {
 509             yaz_log (LOG_LOG, "%sNEW %d %s",
 510                      m_session_str, parent->m_seqno, m_proxyTarget);
 511             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 512             c->m_next = parent->m_clientPool;
 513             if (c->m_next)
 514                 c->m_next->m_prev = &c->m_next;
 515             parent->m_clientPool = c;
 516             c->m_prev = &parent->m_clientPool;
 517         }
 518
 519         xfree (c->m_cookie);
 520         c->m_cookie = 0;
 521         if (cookie)
 522             c->m_cookie = xstrdup(cookie);
 523
 524         c->m_seqno = parent->m_seqno;
 525         c->m_init_flag = 0;
 526         c->m_last_resultCount = 0;
 527         c->m_last_ok = 0;
 528         c->m_cache.clear();
 529         c->m_sr_transform = 0;
 530         c->m_waiting = 0;
 531         c->m_resultSetStartPoint = 0;
 532         (parent->m_seqno)++;
 533         if (c->client(m_proxyTarget))
 534         {
 535             delete c;
 536             return 0;
 537         }
 538         c->m_target_idletime = m_target_idletime;
 539         c->timeout(30);
 540
 541         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 542             c->set_APDU_yazlog(1);
 543         else
 544             c->set_APDU_yazlog(0);
 545     }
 546     yaz_log (LOG_DEBUG, "get_client 3 %p %p", this, c);
 547     return c;
 548 }
 549
 550 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 551 {
 552     int i;
 553     for (i = 0; i<num; i++)
 554     {
 555         oident *ent;
 556         Z_DefaultDiagFormat *r;
 557         Z_DiagRec *p = pp[i];
 558         if (p->which != Z_DiagRec_defaultFormat)
 559         {
 560             yaz_log(LOG_LOG, "%sError no diagnostics", m_session_str);
 561             return;
 562         }
 563         else
 564             r = p->u.defaultFormat;
 565         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 566             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 567             yaz_log(LOG_LOG, "%sError unknown diagnostic set", m_session_str);
 568         switch (r->which)
 569         {
 570         case Z_DefaultDiagFormat_v2Addinfo:
 571             yaz_log(LOG_LOG, "%sError %d %s:%s",
 572                     m_session_str,
 573                     *r->condition, diagbib1_str(*r->condition),
 574                     r->u.v2Addinfo);
 575             break;
 576         case Z_DefaultDiagFormat_v3Addinfo:
 577             yaz_log(LOG_LOG, "%sError %d %s:%s",
 578                     m_session_str,
 579                     *r->condition, diagbib1_str(*r->condition),
 580                     r->u.v3Addinfo);
 581             break;
 582         }
 583     }
 584 }
 585
 586 void Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p)
 587 {
 588     if (!m_stylesheet_schema)
 589         return;
 590     xsltStylesheetPtr xsp;
 591
 592     xsp = xsltParseStylesheetFile((const xmlChar *) m_stylesheet_schema);
 593
 594     int i;
 595     for (i = 0; i < p->num_records; i++)
 596     {
 597         Z_NamePlusRecord *npr = p->records[i];
 598         if (npr->which == Z_NamePlusRecord_databaseRecord)
 599         {
 600             Z_External *r = npr->u.databaseRecord;
 601             if (r->which == Z_External_octet)
 602             {
 603                 xmlDocPtr res, doc = xmlParseMemory(
 604                     (char*) r->u.octet_aligned->buf,
 605                     r->u.octet_aligned->len);
 606
 607                 res = xsltApplyStylesheet(xsp, doc, 0);
 608
 609                 xmlChar *out_buf;
 610                 int out_len;
 611                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 612
 613                 p->records[i]->u.databaseRecord =
 614                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 615                                  (char*) out_buf, out_len);
 616                 xmlFree(out_buf);
 617                 xmlFreeDoc(doc);
 618                 xmlFreeDoc(res);
 619             }
 620         }
 621     }
 622     xsltFreeStylesheet(xsp);
 623 }
 624
 625 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p)
 626 {
 627     int i;
 628
 629     yaz_marc_t mt = yaz_marc_create();
 630     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 631     for (i = 0; i < p->num_records; i++)
 632     {
 633         Z_NamePlusRecord *npr = p->records[i];
 634         if (npr->which == Z_NamePlusRecord_databaseRecord)
 635         {
 636             Z_External *r = npr->u.databaseRecord;
 637             if (r->which == Z_External_octet)
 638             {
 639                 int rlen;
 640                 char *result;
 641                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 642                                         r->u.octet_aligned->len,
 643                                         &result, &rlen))
 644                 {
 645                     yaz_iconv_t cd = yaz_iconv_open("UTF-8", "MARC-8");
 646                     WRBUF wrbuf = wrbuf_alloc();
 647
 648                     char outbuf[120];
 649                     size_t inbytesleft = rlen;
 650                     const char *inp = result;
 651                     while (cd && inbytesleft)
 652                     {
 653                         size_t outbytesleft = sizeof(outbuf);
 654                         char *outp = outbuf;
 655                         size_t r;
 656
 657                         r = yaz_iconv (cd, (char**) &inp,
 658                                        &inbytesleft,
 659                                        &outp, &outbytesleft);
 660                         if (r == (size_t) (-1))
 661                         {
 662                             int e = yaz_iconv_error(cd);
 663                             if (e != YAZ_ICONV_E2BIG)
 664                             {
 665                                 yaz_log(LOG_WARN, "conversion failure");
 666                                 break;
 667                             }
 668                         }
 669                         wrbuf_write(wrbuf, outbuf, outp - outbuf);
 670                     }
 671                     if (cd)
 672                         yaz_iconv_close(cd);
 673
 674                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 675                                                          VAL_TEXT_XML,
 676                                                          wrbuf_buf(wrbuf),
 677                                                          wrbuf_len(wrbuf));
 678                     wrbuf_free(wrbuf, 1);
 679                 }
 680             }
 681         }
 682     }
 683     yaz_marc_destroy(mt);
 684 }
 685
 686 void Yaz_Proxy::logtime()
 687 {
 688     if (m_time_tv.tv_sec)
 689     {
 690         struct timeval tv;
 691         gettimeofday(&tv, 0);
 692         long diff = (tv.tv_sec - m_time_tv.tv_sec)*1000000 +
 693             (tv.tv_usec - m_time_tv.tv_usec);
 694         if (diff >= 0)
 695             yaz_log(LOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 696                     diff/1000000, (diff/1000)%1000);
 697     }
 698     m_time_tv.tv_sec = 0;
 699     m_time_tv.tv_usec = 0;
 700 }
 701
 702 int Yaz_Proxy::send_http_response(int code)
 703 {
 704     ODR o = odr_encode();
 705     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 706     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 707     if (m_http_version)
 708         hres->version = odr_strdup(o, m_http_version);
 709     if (m_http_keepalive)
 710         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 711     else
 712         timeout(0);
 713
 714     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 715     {
 716         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 717                  gdu_name(gdu));
 718     }
 719     int len;
 720     int r = send_GDU(gdu, &len);
 721     logtime();
 722     return r;
 723 }
 724
 725 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 726 {
 727     ODR o = odr_encode();
 728     const char *ctype = "text/xml";
 729     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 730     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 731     if (m_http_version)
 732         hres->version = odr_strdup(o, m_http_version);
 733     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 734     if (m_http_keepalive)
 735         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 736     else
 737         timeout(0);
 738
 739     static Z_SOAP_Handler soap_handlers[2] = {
 740 #if HAVE_XSLT
 741         {"http://www.loc.gov/zing/srw/", 0,
 742          (Z_SOAP_fun) yaz_srw_codec},
 743 #endif
 744         {0, 0, 0}
 745     };
 746
 747     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 748     soap_package->which = Z_SOAP_generic;
 749     soap_package->u.generic =
 750         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 751     soap_package->u.generic->no = 0;
 752     soap_package->u.generic->ns = soap_handlers[0].ns;
 753     soap_package->u.generic->p = (void *) srw_pdu;
 754     soap_package->ns = m_soap_ns;
 755     z_soap_codec_enc_xsl(o, &soap_package,
 756                          &hres->content_buf, &hres->content_len,
 757                          soap_handlers, 0, m_s2z_stylesheet);
 758     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 759     {
 760         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 761                  gdu_name(gdu));
 762     }
 763     int len;
 764     int r = send_GDU(gdu, &len);
 765     logtime();
 766     return r;
 767 }
 768
 769 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
 770 {
 771     ODR o = odr_encode();
 772     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 773     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 774
 775     srw_res->num_diagnostics = 1;
 776     srw_res->diagnostics = (Z_SRW_diagnostic *)
 777         odr_malloc(o, sizeof(*srw_res->diagnostics));
 778     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
 779     return send_srw_response(srw_pdu);
 780 }
 781
 782 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 783                              Z_DefaultDiagFormat *ddf)
 784 {
 785     int bib1_code = *ddf->condition;
 786     if (bib1_code == 109)
 787         return 404;
 788     srw_res->num_diagnostics = 1;
 789     srw_res->diagnostics = (Z_SRW_diagnostic *)
 790         odr_malloc(o, sizeof(*srw_res->diagnostics));
 791     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
 792                           yaz_diag_bib1_to_srw(*ddf->condition),
 793                           ddf->u.v2Addinfo);
 794     return 0;
 795 }
 796
 797 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
 798 {
 799     ODR o = odr_encode();
 800     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 801     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 802
 803     srw_res->numberOfRecords = odr_intdup (o, hits);
 804     if (records && records->which == Z_Records_DBOSD)
 805     {
 806         srw_res->num_records =
 807             records->u.databaseOrSurDiagnostics->num_records;
 808         int i;
 809         srw_res->records = (Z_SRW_record *)
 810             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
 811         for (i = 0; i < srw_res->num_records; i++)
 812         {
 813             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
 814             if (npr->which != Z_NamePlusRecord_databaseRecord)
 815             {
 816                 srw_res->records[i].recordSchema = "diagnostic";
 817                 srw_res->records[i].recordPacking = m_s2z_packing;
 818                 srw_res->records[i].recordData_buf = "67";
 819                 srw_res->records[i].recordData_len = 2;
 820                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 821                 continue;
 822             }
 823             Z_External *r = npr->u.databaseRecord;
 824             oident *ent = oid_getentbyoid(r->direct_reference);
 825             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
 826             {
 827                 srw_res->records[i].recordSchema = m_schema;
 828                 srw_res->records[i].recordPacking = m_s2z_packing;
 829                 srw_res->records[i].recordData_buf = (char*)
 830                     r->u.octet_aligned->buf;
 831                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
 832                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 833             }
 834             else
 835             {
 836                 srw_res->records[i].recordSchema = "diagnostic";
 837                 srw_res->records[i].recordPacking = m_s2z_packing;
 838                 srw_res->records[i].recordData_buf = "67";
 839                 srw_res->records[i].recordData_len = 2;
 840                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 841             }
 842         }
 843     }
 844     if (records && records->which == Z_Records_NSD)
 845     {
 846         int http_code;
 847         http_code = z_to_srw_diag(odr_encode(), srw_res,
 848                                    records->u.nonSurrogateDiagnostic);
 849         if (http_code)
 850             return send_http_response(http_code);
 851     }
 852     return send_srw_response(srw_pdu);
 853
 854 }
 855
 856 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
 857                                         int num_diagnostics)
 858 {
 859     Yaz_ProxyConfig *cfg = check_reconfigure();
 860     if (cfg)
 861     {
 862         int len;
 863         char *b = cfg->get_explain(odr_encode(), 0 /* target */,
 864                                    m_s2z_database, &len);
 865         if (b)
 866         {
 867             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
 868             Z_SRW_explainResponse *er = res->u.explain_response;
 869
 870             er->record.recordData_buf = b;
 871             er->record.recordData_len = len;
 872             er->record.recordPacking = m_s2z_packing;
 873
 874             er->diagnostics = diagnostics;
 875             er->num_diagnostics = num_diagnostics;
 876             return send_srw_response(res);
 877         }
 878     }
 879     return send_http_response(404);
 880 }
 881
 882 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu, int *len)
 883 {
 884     if (m_http_version)
 885     {
 886         if (apdu->which == Z_APDU_initResponse)
 887         {
 888             Z_InitResponse *res = apdu->u.initResponse;
 889             if (*res->result == 0)
 890             {
 891                 send_to_srw_client_error(3, 0);
 892             }
 893             else if (!m_s2z_search_apdu)
 894             {
 895                 send_srw_explain_response(0, 0);
 896             }
 897             else
 898             {
 899                 handle_incoming_Z_PDU(m_s2z_search_apdu);
 900             }
 901         }
 902         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
 903         {
 904             m_s2z_search_apdu = 0;
 905             Z_SearchResponse *res = apdu->u.searchResponse;
 906             m_s2z_hit_count = *res->resultCount;
 907             if (res->records && res->records->which == Z_Records_NSD)
 908             {
 909                 send_to_srw_client_ok(0, res->records, 1);
 910             }
 911             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
 912             {
 913                 // adjust
 914                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
 915
 916                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
 917                 {
 918                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
 919                         > m_s2z_hit_count)
 920                         *pr->numberOfRecordsRequested =
 921                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
 922                 }
 923                 handle_incoming_Z_PDU(m_s2z_present_apdu);
 924             }
 925             else
 926             {
 927                 m_s2z_present_apdu = 0;
 928                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
 929             }
 930         }
 931         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
 932         {
 933             int start =
 934                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
 935
 936             m_s2z_present_apdu = 0;
 937             Z_PresentResponse *res = apdu->u.presentResponse;
 938             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
 939         }
 940     }
 941     else
 942     {
 943         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 944             yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 945                      apdu_name(apdu));
 946         int r = send_Z_PDU(apdu, len);
 947         logtime();
 948         return r;
 949     }
 950     return 0;
 951 }
 952
 953 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 954 {
 955     int len = 0;
 956     int kill_session = 0;
 957     if (apdu->which == Z_APDU_searchResponse)
 958     {
 959         Z_SearchResponse *sr = apdu->u.searchResponse;
 960         Z_Records *p = sr->records;
 961         if (p && p->which == Z_Records_NSD)
 962         {
 963             Z_DiagRec dr, *dr_p = &dr;
 964             dr.which = Z_DiagRec_defaultFormat;
 965             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 966
 967             *sr->searchStatus = 0;
 968             display_diagrecs(&dr_p, 1);
 969         }
 970         else
 971         {
 972             if (p && p->which == Z_Records_DBOSD)
 973             {
 974                 if (m_marcxml_flag)
 975                     convert_to_marcxml(p->u.databaseOrSurDiagnostics);
 976                 convert_xsl(p->u.databaseOrSurDiagnostics);
 977             }
 978             if (sr->resultCount)
 979             {
 980                 yaz_log(LOG_LOG, "%s%d hits", m_session_str,
 981                         *sr->resultCount);
 982                 if (*sr->resultCount < 0)
 983                 {
 984                     m_invalid_session = 1;
 985                     kill_session = 1;
 986
 987                     *sr->searchStatus = 0;
 988                     sr->records =
 989                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
 990                     *sr->resultCount = 0;
 991                 }
 992             }
 993         }
 994     }
 995     else if (apdu->which == Z_APDU_presentResponse)
 996     {
 997         Z_PresentResponse *sr = apdu->u.presentResponse;
 998         Z_Records *p = sr->records;
 999         if (p && p->which == Z_Records_NSD)
1000         {
1001             Z_DiagRec dr, *dr_p = &dr;
1002             dr.which = Z_DiagRec_defaultFormat;
1003             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1004             if (*sr->presentStatus == Z_PresentStatus_success)
1005                 *sr->presentStatus = Z_PresentStatus_failure;
1006             display_diagrecs(&dr_p, 1);
1007         }
1008         if (p && p->which == Z_Records_DBOSD)
1009         {
1010             if (m_marcxml_flag)
1011                 convert_to_marcxml(p->u.databaseOrSurDiagnostics);
1012             convert_xsl(p->u.databaseOrSurDiagnostics);
1013         }
1014     }
1015     else if (apdu->which == Z_APDU_initResponse)
1016     {
1017         if (m_initRequest_options)
1018         {
1019             Z_Options *nopt =
1020                 (Odr_bitmask *)odr_malloc(odr_encode(),
1021                                           sizeof(Odr_bitmask));
1022             ODR_MASK_ZERO(nopt);
1023
1024             int i;
1025             for (i = 0; i<24; i++)
1026                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1027                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1028                     ODR_MASK_SET(nopt, i);
1029             apdu->u.initResponse->options = nopt;
1030         }
1031         if (m_initRequest_version)
1032         {
1033             Z_ProtocolVersion *nopt =
1034                 (Odr_bitmask *)odr_malloc(odr_encode(),
1035                                           sizeof(Odr_bitmask));
1036             ODR_MASK_ZERO(nopt);
1037
1038             int i;
1039             for (i = 0; i<8; i++)
1040                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1041                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1042                     ODR_MASK_SET(nopt, i);
1043             apdu->u.initResponse->protocolVersion = nopt;
1044         }
1045     }
1046     int r = send_PDU_convert(apdu, &len);
1047     if (r)
1048         return r;
1049     m_bytes_sent += len;
1050     m_bw_stat.add_bytes(len);
1051     if (kill_session)
1052     {
1053         delete m_client;
1054         m_client = 0;
1055         m_parent->pre_init();
1056     }
1057     return r;
1058 }
1059
1060 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1061 {
1062     int len = 0;
1063     const char *apdu_name_tmp = apdu_name(apdu);
1064     int r = send_Z_PDU(apdu, &len);
1065     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1066         yaz_log (LOG_LOG, "%sSending %s to %s %d bytes",
1067                  get_session_str(),
1068                  apdu_name_tmp, get_hostname(), len);
1069     m_bytes_sent += len;
1070     return r;
1071 }
1072
1073 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1074 {
1075     if (apdu->which == Z_APDU_presentRequest)
1076     {
1077         Z_PresentRequest *pr = apdu->u.presentRequest;
1078         int toget = *pr->numberOfRecordsRequested;
1079         int start = *pr->resultSetStartPoint;
1080
1081         yaz_log(LOG_LOG, "%sPresent %s %d+%d", m_session_str,
1082                 pr->resultSetId, start, toget);
1083
1084         if (*m_parent->m_optimize == '0')
1085             return apdu;
1086
1087         if (!m_client->m_last_resultSetId)
1088         {
1089             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1090             new_apdu->u.presentResponse->records =
1091                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1092                                                pr->resultSetId);
1093             send_to_client(new_apdu);
1094             return 0;
1095         }
1096         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1097         {
1098             if (start+toget-1 > m_client->m_last_resultCount)
1099             {
1100                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1101                 new_apdu->u.presentResponse->records =
1102                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1103                 send_to_client(new_apdu);
1104                 return 0;
1105             }
1106             Z_NamePlusRecordList *npr;
1107             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1108                                           pr->preferredRecordSyntax,
1109                                           pr->recordComposition))
1110             {
1111                 yaz_log (LOG_LOG, "%sReturned cached records for present request",
1112                          m_session_str);
1113                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1114                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1115
1116                 new_apdu->u.presentResponse->numberOfRecordsReturned
1117                     = odr_intdup(odr_encode(), toget);
1118
1119                 new_apdu->u.presentResponse->records = (Z_Records*)
1120                     odr_malloc(odr_encode(), sizeof(Z_Records));
1121                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1122                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1123                 new_apdu->u.presentResponse->nextResultSetPosition =
1124                     odr_intdup(odr_encode(), start+toget);
1125
1126                 send_to_client(new_apdu);
1127                 return 0;
1128             }
1129         }
1130     }
1131
1132     if (apdu->which != Z_APDU_searchRequest)
1133         return apdu;
1134     Z_SearchRequest *sr = apdu->u.searchRequest;
1135     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1136     Yaz_Z_Databases this_databases;
1137
1138     this_databases.set(sr->num_databaseNames, (const char **)
1139                        sr->databaseNames);
1140
1141     this_query->set_Z_Query(sr->query);
1142
1143     char query_str[120];
1144     this_query->print(query_str, sizeof(query_str)-1);
1145     yaz_log(LOG_LOG, "%sSearch %s", m_session_str, query_str);
1146
1147     if (*m_parent->m_optimize != '0' &&
1148         m_client->m_last_ok && m_client->m_last_query &&
1149         m_client->m_last_query->match(this_query) &&
1150         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1151         m_client->m_last_databases.match(this_databases))
1152     {
1153         delete this_query;
1154         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1155             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1156         {
1157             Z_NamePlusRecordList *npr;
1158             int toget = *sr->mediumSetPresentNumber;
1159             Z_RecordComposition *comp = 0;
1160
1161             if (toget > m_client->m_last_resultCount)
1162                 toget = m_client->m_last_resultCount;
1163
1164             if (sr->mediumSetElementSetNames)
1165             {
1166                 comp = (Z_RecordComposition *)
1167                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1168                 comp->which = Z_RecordComp_simple;
1169                 comp->u.simple = sr->mediumSetElementSetNames;
1170             }
1171
1172             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1173                                           sr->preferredRecordSyntax, comp))
1174             {
1175                 yaz_log (LOG_LOG, "%sReturned cached records for medium set",
1176                          m_session_str);
1177                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1178                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1179                 new_apdu->u.searchResponse->resultCount =
1180                     &m_client->m_last_resultCount;
1181
1182                 new_apdu->u.searchResponse->numberOfRecordsReturned
1183                     = odr_intdup(odr_encode(), toget);
1184
1185                 new_apdu->u.searchResponse->presentStatus =
1186                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1187                 new_apdu->u.searchResponse->records = (Z_Records*)
1188                     odr_malloc(odr_encode(), sizeof(Z_Records));
1189                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1190                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1191                 new_apdu->u.searchResponse->nextResultSetPosition =
1192                     odr_intdup(odr_encode(), toget+1);
1193                 send_to_client(new_apdu);
1194                 return 0;
1195             }
1196             else
1197             {
1198                 // medium Set
1199                 // send present request (medium size)
1200                 yaz_log (LOG_LOG, "%sOptimizing search for medium set",
1201                          m_session_str);
1202
1203                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1204                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1205                 pr->referenceId = sr->referenceId;
1206                 pr->resultSetId = sr->resultSetName;
1207                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1208                 *pr->numberOfRecordsRequested = toget;
1209                 pr->recordComposition = comp;
1210                 m_client->m_sr_transform = 1;
1211                 return new_apdu;
1212             }
1213         }
1214         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1215             m_client->m_last_resultCount <= 0)
1216         {
1217             // large set. Return pseudo-search response immediately
1218             yaz_log (LOG_LOG, "%sOptimizing search for large set",
1219                      m_session_str);
1220             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1221             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1222             new_apdu->u.searchResponse->resultCount =
1223                 &m_client->m_last_resultCount;
1224             send_to_client(new_apdu);
1225             return 0;
1226         }
1227         else
1228         {
1229             Z_NamePlusRecordList *npr;
1230             int toget = m_client->m_last_resultCount;
1231             Z_RecordComposition *comp = 0;
1232             // small set
1233             // send a present request (small set)
1234
1235             if (sr->smallSetElementSetNames)
1236             {
1237                 comp = (Z_RecordComposition *)
1238                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1239                 comp->which = Z_RecordComp_simple;
1240                 comp->u.simple = sr->smallSetElementSetNames;
1241             }
1242
1243             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1244                                           sr->preferredRecordSyntax, comp))
1245             {
1246                 yaz_log (LOG_LOG, "%sReturned cached records for small set",
1247                          m_session_str);
1248                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1249                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1250                 new_apdu->u.searchResponse->resultCount =
1251                     &m_client->m_last_resultCount;
1252
1253                 new_apdu->u.searchResponse->numberOfRecordsReturned
1254                     = odr_intdup(odr_encode(), toget);
1255
1256                 new_apdu->u.searchResponse->presentStatus =
1257                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1258                 new_apdu->u.searchResponse->records = (Z_Records*)
1259                     odr_malloc(odr_encode(), sizeof(Z_Records));
1260                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1261                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1262                 new_apdu->u.searchResponse->nextResultSetPosition =
1263                     odr_intdup(odr_encode(), toget+1);
1264                 send_to_client(new_apdu);
1265                 return 0;
1266             }
1267             else
1268             {
1269                 yaz_log (LOG_LOG, "%sOptimizing search for small set",
1270                          m_session_str);
1271                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1272                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1273                 pr->referenceId = sr->referenceId;
1274                 pr->resultSetId = sr->resultSetName;
1275                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1276                 *pr->numberOfRecordsRequested = toget;
1277                 pr->recordComposition = comp;
1278                 m_client->m_sr_transform = 1;
1279                 return new_apdu;
1280             }
1281         }
1282     }
1283     else  // query doesn't match
1284     {
1285         delete m_client->m_last_query;
1286         m_client->m_last_query = this_query;
1287         m_client->m_last_ok = 0;
1288         m_client->m_cache.clear();
1289         m_client->m_resultSetStartPoint = 0;
1290
1291         xfree (m_client->m_last_resultSetId);
1292         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1293
1294         m_client->m_last_databases.set(sr->num_databaseNames,
1295                                        (const char **) sr->databaseNames);
1296     }
1297     return apdu;
1298 }
1299
1300
1301 void Yaz_Proxy::inc_request_no()
1302 {
1303     char *cp = strchr(m_session_str, ' ');
1304     m_request_no++;
1305     if (cp)
1306         sprintf(cp+1, "%d ", m_request_no);
1307 }
1308
1309 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1310 {
1311     inc_request_no();
1312
1313     m_bytes_recv += len;
1314
1315     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
1316         yaz_log (LOG_DEBUG, "%sReceiving %s from client %d bytes",
1317                  m_session_str, gdu_name(apdu), len);
1318
1319     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1320         shutdown();
1321
1322     m_bw_stat.add_bytes(len);
1323     m_pdu_stat.add_bytes(1);
1324
1325     gettimeofday(&m_time_tv, 0);
1326
1327     int bw_total = m_bw_stat.get_total();
1328     int pdu_total = m_pdu_stat.get_total();
1329
1330     int reduce = 0;
1331     if (m_bw_max)
1332     {
1333         if (bw_total > m_bw_max)
1334         {
1335             reduce = (bw_total/m_bw_max);
1336         }
1337     }
1338     if (m_pdu_max)
1339     {
1340         if (pdu_total > m_pdu_max)
1341         {
1342             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1343             reduce = (reduce > nreduce) ? reduce : nreduce;
1344         }
1345     }
1346     if (reduce)
1347     {
1348         yaz_log(LOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1349                 m_session_str, reduce, bw_total, pdu_total,
1350                 m_bw_max, m_pdu_max);
1351
1352         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1353         timeout(reduce);       // call us reduce seconds later
1354     }
1355     else if (apdu->which == Z_GDU_Z3950)
1356         handle_incoming_Z_PDU(apdu->u.z3950);
1357     else if (apdu->which == Z_GDU_HTTP_Request)
1358         handle_incoming_HTTP(apdu->u.HTTP_Request);
1359 }
1360
1361 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1362 {
1363     if (m_max_record_retrieve)
1364     {
1365         if (apdu->which == Z_APDU_presentRequest)
1366         {
1367             Z_PresentRequest *pr = apdu->u.presentRequest;
1368             if (pr->numberOfRecordsRequested &&
1369                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1370                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1371         }
1372     }
1373 }
1374
1375 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1376                                                      int error,
1377                                                      const char *addinfo)
1378 {
1379     Z_Records *rec = (Z_Records *)
1380         odr_malloc (odr, sizeof(*rec));
1381     int *err = (int *)
1382         odr_malloc (odr, sizeof(*err));
1383     Z_DiagRec *drec = (Z_DiagRec *)
1384         odr_malloc (odr, sizeof(*drec));
1385     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1386         odr_malloc (odr, sizeof(*dr));
1387     *err = error;
1388     rec->which = Z_Records_NSD;
1389     rec->u.nonSurrogateDiagnostic = dr;
1390     dr->diagnosticSetId =
1391         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1392     dr->condition = err;
1393     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1394     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1395     return rec;
1396 }
1397
1398 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1399 {
1400     if (apdu->which == Z_APDU_searchRequest &&
1401         apdu->u.searchRequest->query &&
1402         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1403         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1404     {
1405         Z_RPNQuery *rpnquery = 0;
1406         Z_SearchRequest *sr = apdu->u.searchRequest;
1407         char *addinfo = 0;
1408
1409         yaz_log(LOG_LOG, "%sCQL: %s", m_session_str,
1410                 sr->query->u.type_104->u.cql);
1411
1412         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1413                                           &rpnquery, odr_encode(),
1414                                           &addinfo);
1415         if (r == -3)
1416             yaz_log(LOG_LOG, "%sNo CQL to RPN table", m_session_str);
1417         else if (r)
1418         {
1419             yaz_log(LOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1420             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1421
1422             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1423             new_apdu->u.searchResponse->records =
1424                 create_nonSurrogateDiagnostics(odr_encode(),
1425                                                yaz_diag_srw_to_bib1(r),
1426                                                addinfo);
1427             *new_apdu->u.searchResponse->searchStatus = 0;
1428
1429             send_to_client(new_apdu);
1430
1431             return 0;
1432         }
1433         else
1434         {
1435             sr->query->which = Z_Query_type_1;
1436             sr->query->u.type_1 = rpnquery;
1437         }
1438         return apdu;
1439     }
1440     return apdu;
1441 }
1442
1443 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1444 {
1445     if (apdu->which == Z_APDU_searchRequest)
1446     {
1447         Z_SearchRequest *sr = apdu->u.searchRequest;
1448         int err = 0;
1449         char *addinfo = 0;
1450
1451         Yaz_ProxyConfig *cfg = check_reconfigure();
1452         if (cfg)
1453             err = cfg->check_query(odr_encode(), m_default_target,
1454                                    sr->query, &addinfo);
1455         if (err)
1456         {
1457             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1458
1459             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1460             new_apdu->u.searchResponse->records =
1461                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1462             *new_apdu->u.searchResponse->searchStatus = 0;
1463
1464             send_to_client(new_apdu);
1465
1466             return 0;
1467         }
1468     }
1469     return apdu;
1470 }
1471
1472 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1473 {
1474     m_marcxml_flag = 0;
1475     if (apdu->which == Z_APDU_searchRequest)
1476     {
1477         Z_SearchRequest *sr = apdu->u.searchRequest;
1478         int err = 0;
1479         char *addinfo = 0;
1480         Yaz_ProxyConfig *cfg = check_reconfigure();
1481
1482         Z_RecordComposition rc_temp, *rc = 0;
1483         if (sr->smallSetElementSetNames)
1484         {
1485             rc_temp.which = Z_RecordComp_simple;
1486             rc_temp.u.simple = sr->smallSetElementSetNames;
1487             rc = &rc_temp;
1488         }
1489
1490         if (cfg)
1491             err = cfg->check_syntax(odr_encode(),
1492                                     m_default_target,
1493                                     sr->preferredRecordSyntax, rc,
1494                                     &addinfo, &m_stylesheet_schema, &m_schema);
1495         if (err == -1)
1496         {
1497             sr->preferredRecordSyntax =
1498                 yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN, VAL_USMARC);
1499             m_marcxml_flag = 1;
1500         }
1501         else if (err)
1502         {
1503             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1504
1505             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1506             new_apdu->u.searchResponse->records =
1507                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1508             *new_apdu->u.searchResponse->searchStatus = 0;
1509
1510             send_to_client(new_apdu);
1511
1512             return 0;
1513         }
1514     }
1515     else if (apdu->which == Z_APDU_presentRequest)
1516     {
1517         Z_PresentRequest *pr = apdu->u.presentRequest;
1518         int err = 0;
1519         char *addinfo = 0;
1520         Yaz_ProxyConfig *cfg = check_reconfigure();
1521
1522         if (cfg)
1523             err = cfg->check_syntax(odr_encode(), m_default_target,
1524                                     pr->preferredRecordSyntax,
1525                                     pr->recordComposition,
1526                                     &addinfo, &m_stylesheet_schema, &m_schema);
1527         if (err == -1)
1528         {
1529             pr->preferredRecordSyntax =
1530                 yaz_oidval_to_z3950oid(odr_decode(), CLASS_RECSYN, VAL_USMARC);
1531             m_marcxml_flag = 1;
1532         }
1533         else if (err)
1534         {
1535             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1536
1537             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1538             new_apdu->u.presentResponse->records =
1539                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1540             *new_apdu->u.presentResponse->presentStatus =
1541                 Z_PresentStatus_failure;
1542
1543             send_to_client(new_apdu);
1544
1545             return 0;
1546         }
1547     }
1548     return apdu;
1549 }
1550
1551 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
1552 {
1553     if (!schema)
1554         return 0;
1555     Z_ElementSetNames *esn = (Z_ElementSetNames *)
1556         odr_malloc(o, sizeof(Z_ElementSetNames));
1557     esn->which = Z_ElementSetNames_generic;
1558     esn->u.generic = odr_strdup(o, schema);
1559     return esn;
1560 }
1561
1562 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1563 {
1564     if (m_s2z_odr_init)
1565     {
1566         odr_destroy(m_s2z_odr_init);
1567         m_s2z_odr_init = 0;
1568     }
1569     if (m_s2z_odr_search)
1570     {
1571         odr_destroy(m_s2z_odr_search);
1572         m_s2z_odr_search = 0;
1573     }
1574
1575     m_http_keepalive = 0;
1576     m_http_version = 0;
1577     if (!strcmp(hreq->version, "1.0"))
1578     {
1579         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1580         if (v && !strcmp(v, "Keep-Alive"))
1581             m_http_keepalive = 1;
1582         else
1583             m_http_keepalive = 0;
1584         m_http_version = "1.0";
1585     }
1586     else
1587     {
1588         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1589         if (v && !strcmp(v, "close"))
1590             m_http_keepalive = 0;
1591         else
1592             m_http_keepalive = 1;
1593         m_http_version = "1.1";
1594     }
1595
1596     Z_SRW_PDU *srw_pdu = 0;
1597     Z_SOAP *soap_package = 0;
1598     char *charset = 0;
1599     Z_SRW_diagnostic *diagnostic = 0;
1600     int num_diagnostic = 0;
1601     if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1602                        &charset) == 0
1603         || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1604                           &charset, &diagnostic, &num_diagnostic) == 0)
1605     {
1606         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
1607         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
1608         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
1609         m_s2z_init_apdu = 0;
1610         m_s2z_search_apdu = 0;
1611         m_s2z_present_apdu = 0;
1612
1613         m_s2z_stylesheet = 0;
1614
1615         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
1616         {
1617             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
1618
1619             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1620             // recordXPath unsupported.
1621             if (srw_req->recordXPath)
1622             {
1623                 yaz_add_srw_diagnostic(odr_decode(),
1624                                        &diagnostic, &num_diagnostic,
1625                                        72, 0);
1626             }
1627             // must have a query
1628             if (!srw_req->query.cql)
1629             {
1630                 yaz_add_srw_diagnostic(odr_decode(),
1631                                        &diagnostic, &num_diagnostic,
1632                                        7, "query");
1633             }
1634             // sort unsupported
1635             if (srw_req->sort_type != Z_SRW_sort_type_none)
1636             {
1637                 yaz_add_srw_diagnostic(odr_decode(),
1638                                        &diagnostic, &num_diagnostic,
1639                                        80, 0);
1640             }
1641             // save stylesheet
1642             if (srw_req->stylesheet)
1643                 m_s2z_stylesheet =
1644                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1645
1646             // set packing for response records ..
1647             if (srw_req->recordPacking &&
1648                 !strcmp(srw_req->recordPacking, "xml"))
1649                 m_s2z_packing = Z_SRW_recordPacking_XML;
1650             else
1651                 m_s2z_packing = Z_SRW_recordPacking_string;
1652
1653             if (num_diagnostic)
1654             {
1655                 Z_SRW_PDU *srw_pdu =
1656                     yaz_srw_get(odr_encode(),
1657                                 Z_SRW_searchRetrieve_response);
1658                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1659
1660                 srw_res->diagnostics = diagnostic;
1661                 srw_res->num_diagnostics = num_diagnostic;
1662                 send_srw_response(srw_pdu);
1663                 return;
1664             }
1665
1666             // prepare search PDU
1667             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
1668                                           Z_APDU_searchRequest);
1669             Z_SearchRequest *z_searchRequest =
1670                 m_s2z_search_apdu->u.searchRequest;
1671
1672             z_searchRequest->num_databaseNames = 1;
1673             z_searchRequest->databaseNames = (char**)
1674                 odr_malloc(m_s2z_odr_search, sizeof(char *));
1675             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
1676                                                            srw_req->database);
1677
1678             // query transformation
1679             Z_Query *query = (Z_Query *)
1680                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
1681             z_searchRequest->query = query;
1682
1683             if (srw_req->query_type == Z_SRW_query_type_cql)
1684             {
1685                 Z_External *ext = (Z_External *)
1686                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
1687                 ext->direct_reference =
1688                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
1689                 ext->indirect_reference = 0;
1690                 ext->descriptor = 0;
1691                 ext->which = Z_External_CQL;
1692                 ext->u.cql = srw_req->query.cql;
1693
1694                 query->which = Z_Query_type_104;
1695                 query->u.type_104 =  ext;
1696             }
1697             else if (srw_req->query_type == Z_SRW_query_type_pqf)
1698             {
1699                 Z_RPNQuery *RPNquery;
1700                 YAZ_PQF_Parser pqf_parser;
1701
1702                 pqf_parser = yaz_pqf_create ();
1703
1704                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
1705                                           srw_req->query.pqf);
1706                 if (!RPNquery)
1707                 {
1708                     const char *pqf_msg;
1709                     size_t off;
1710                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
1711                     yaz_log(LOG_LOG, "%*s^\n", off+4, "");
1712                     yaz_log(LOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
1713
1714                     send_to_srw_client_error(10, 0);
1715                     return;
1716                 }
1717                 query->which = Z_Query_type_1;
1718                 query->u.type_1 =  RPNquery;
1719
1720                 yaz_pqf_destroy (pqf_parser);
1721             }
1722             else
1723             {
1724                 send_to_srw_client_error(7, "query");
1725                 return;
1726             }
1727
1728             // present
1729             m_s2z_present_apdu = 0;
1730             int max = 0;
1731             if (srw_req->maximumRecords)
1732                 max = *srw_req->maximumRecords;
1733             int start = 1;
1734             if (srw_req->startRecord)
1735                 start = *srw_req->startRecord;
1736             if (max > 0)
1737             {
1738                 // Some backend, such as Voyager doesn't honor piggyback
1739                 // So we use present always (0 &&).
1740                 if (0 && start <= 1)  // Z39.50 piggyback
1741                 {
1742                     *z_searchRequest->smallSetUpperBound = max;
1743                     *z_searchRequest->mediumSetPresentNumber = max;
1744                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
1745
1746                     z_searchRequest->preferredRecordSyntax =
1747                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1748                                                VAL_TEXT_XML);
1749                     if (srw_req->recordSchema)
1750                     {
1751                         z_searchRequest->smallSetElementSetNames =
1752                             z_searchRequest->mediumSetElementSetNames =
1753                             mk_esn_from_schema(m_s2z_odr_search,
1754                                                srw_req->recordSchema);
1755                     }
1756                 }
1757                 else   // Z39.50 present
1758                 {
1759                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
1760                                                    Z_APDU_presentRequest);
1761                     Z_PresentRequest *z_presentRequest =
1762                         m_s2z_present_apdu->u.presentRequest;
1763                     *z_presentRequest->resultSetStartPoint = start;
1764                     *z_presentRequest->numberOfRecordsRequested = max;
1765                     z_presentRequest->preferredRecordSyntax =
1766                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1767                                                VAL_TEXT_XML);
1768                     z_presentRequest->recordComposition =
1769                         (Z_RecordComposition *)
1770                         odr_malloc(m_s2z_odr_search,
1771                                    sizeof(Z_RecordComposition));
1772                     if (srw_req->recordSchema)
1773                     {
1774                         z_presentRequest->recordComposition->which =
1775                             Z_RecordComp_simple;
1776                         z_presentRequest->recordComposition->u.simple =
1777                             mk_esn_from_schema(m_s2z_odr_search,
1778                                                srw_req->recordSchema);
1779                     }
1780                 }
1781             }
1782             if (!m_client)
1783             {
1784                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1785                                             Z_APDU_initRequest);
1786
1787                 // prevent m_initRequest_apdu memory from being grabbed
1788                 // in Yaz_Proxy::handle_incoming_Z_PDU
1789                 m_initRequest_apdu = m_s2z_init_apdu;
1790                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1791                 return;
1792             }
1793             else
1794             {
1795                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1796                 return;
1797             }
1798         }
1799         else if (srw_pdu->which == Z_SRW_explain_request)
1800         {
1801             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
1802
1803             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1804
1805             // save stylesheet
1806             if (srw_req->stylesheet)
1807                 m_s2z_stylesheet =
1808                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1809
1810             if (srw_req->recordPacking &&
1811                 !strcmp(srw_req->recordPacking, "xml"))
1812                 m_s2z_packing = Z_SRW_recordPacking_XML;
1813             else
1814                 m_s2z_packing = Z_SRW_recordPacking_string;
1815
1816             if (num_diagnostic)
1817             {
1818                 send_srw_explain_response(diagnostic, num_diagnostic);
1819                 return;
1820             }
1821
1822             if (!m_client)
1823             {
1824                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1825                                             Z_APDU_initRequest);
1826
1827                 // prevent m_initRequest_apdu memory from being grabbed
1828                 // in Yaz_Proxy::handle_incoming_Z_PDU
1829                 m_initRequest_apdu = m_s2z_init_apdu;
1830                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1831             }
1832             else
1833                 send_srw_explain_response(0, 0);
1834             return;
1835         }
1836         else if (srw_pdu->which == Z_SRW_scan_request)
1837         {
1838             m_s2z_database = odr_strdup(m_s2z_odr_init,
1839                                         srw_pdu->u.scan_request->database);
1840
1841             yaz_add_srw_diagnostic(odr_decode(),
1842                                    &diagnostic, &num_diagnostic,
1843                                    4, "scan");
1844             Z_SRW_PDU *srw_pdu =
1845                 yaz_srw_get(odr_encode(),
1846                             Z_SRW_scan_response);
1847             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1848
1849             srw_res->diagnostics = diagnostic;
1850             srw_res->num_diagnostics = num_diagnostic;
1851             send_srw_response(srw_pdu);
1852             return;
1853         }
1854         else
1855         {
1856             m_s2z_database = 0;
1857
1858             send_to_srw_client_error(4, 0);
1859         }
1860     }
1861     int len = 0;
1862     Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
1863     timeout(0);
1864     send_GDU(p, &len);
1865 }
1866
1867 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
1868 {
1869     if (!m_client && m_invalid_session)
1870     {
1871         m_apdu_invalid_session = apdu;
1872         m_mem_invalid_session = odr_extract_mem(odr_decode());
1873         apdu = m_initRequest_apdu;
1874     }
1875
1876     // Determine our client.
1877     Z_OtherInformation **oi;
1878     get_otherInfoAPDU(apdu, &oi);
1879     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
1880     if (!m_client)
1881     {
1882         delete this;
1883         return;
1884     }
1885     m_client->m_server = this;
1886
1887     if (apdu->which == Z_APDU_initRequest)
1888     {
1889         if (apdu->u.initRequest->implementationId)
1890             yaz_log(LOG_LOG, "%simplementationId: %s",
1891                     m_session_str, apdu->u.initRequest->implementationId);
1892         if (apdu->u.initRequest->implementationName)
1893             yaz_log(LOG_LOG, "%simplementationName: %s",
1894                     m_session_str, apdu->u.initRequest->implementationName);
1895         if (apdu->u.initRequest->implementationVersion)
1896             yaz_log(LOG_LOG, "%simplementationVersion: %s",
1897                     m_session_str, apdu->u.initRequest->implementationVersion);
1898         if (m_initRequest_apdu == 0)
1899         {
1900             if (m_initRequest_mem)
1901                 nmem_destroy(m_initRequest_mem);
1902             m_initRequest_apdu = apdu;
1903             m_initRequest_mem = odr_extract_mem(odr_decode());
1904
1905             // save init options for the response..
1906             m_initRequest_options = apdu->u.initRequest->options;
1907
1908             apdu->u.initRequest->options =
1909                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
1910                                            sizeof(Odr_bitmask));
1911             ODR_MASK_ZERO(apdu->u.initRequest->options);
1912             int i;
1913             for (i = 0; i<= 24; i++)
1914                 ODR_MASK_SET(apdu->u.initRequest->options, i);
1915             ODR_MASK_CLEAR(apdu->u.initRequest->options,
1916                            Z_Options_negotiationModel);
1917
1918             // make new version
1919             m_initRequest_version = apdu->u.initRequest->protocolVersion;
1920             apdu->u.initRequest->protocolVersion =
1921                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
1922                                            sizeof(Odr_bitmask));
1923             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
1924
1925             for (i = 0; i<= 8; i++)
1926                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
1927         }
1928         if (m_client->m_init_flag)
1929         {
1930             if (handle_init_response_for_invalid_session(apdu))
1931                 return;
1932             Z_APDU *apdu2 = m_client->m_initResponse;
1933             apdu2->u.initResponse->otherInfo = 0;
1934             if (m_client->m_cookie && *m_client->m_cookie)
1935                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
1936                                            m_client->m_cookie);
1937             apdu2->u.initResponse->referenceId =
1938                 apdu->u.initRequest->referenceId;
1939             apdu2->u.initResponse->options = m_client->m_initResponse_options;
1940             apdu2->u.initResponse->protocolVersion =
1941                 m_client->m_initResponse_version;
1942
1943             send_to_client(apdu2);
1944             return;
1945         }
1946         m_client->m_init_flag = 1;
1947     }
1948     handle_max_record_retrieve(apdu);
1949
1950     if (apdu)
1951         apdu = handle_syntax_validation(apdu);
1952
1953     if (apdu)
1954         apdu = handle_query_transformation(apdu);
1955
1956     if (apdu)
1957         apdu = handle_query_validation(apdu);
1958
1959     if (apdu)
1960         apdu = result_set_optimize(apdu);
1961     if (!apdu)
1962     {
1963         m_client->timeout(m_target_idletime);  // mark it active even
1964         // though we didn't use it
1965         return;
1966     }
1967
1968     // delete other info part from PDU before sending to target
1969     get_otherInfoAPDU(apdu, &oi);
1970     if (oi)
1971         *oi = 0;
1972
1973     if (apdu->which == Z_APDU_presentRequest &&
1974         m_client->m_resultSetStartPoint == 0)
1975     {
1976         Z_PresentRequest *pr = apdu->u.presentRequest;
1977         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
1978         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
1979     } else {
1980         m_client->m_resultSetStartPoint = 0;
1981     }
1982     if (m_client->send_to_target(apdu) < 0)
1983     {
1984         delete m_client;
1985         m_client = 0;
1986         delete this;
1987     }
1988     else
1989         m_client->m_waiting = 1;
1990 }
1991
1992 void Yaz_Proxy::connectNotify()
1993 {
1994 }
1995
1996 void Yaz_Proxy::shutdown()
1997 {
1998     m_invalid_session = 0;
1999     // only keep if keep_alive flag is set...
2000     if (m_client &&
2001         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2002         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2003         m_client->m_waiting == 0)
2004     {
2005         yaz_log(LOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2006                  m_session_str,
2007                  m_client->get_hostname());
2008         yaz_log(LOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2009                 m_session_str, m_client->m_pdu_recv,
2010                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2011                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2012         assert (m_client->m_waiting != 2);
2013         // Tell client (if any) that no server connection is there..
2014         m_client->m_server = 0;
2015         m_invalid_session = 0;
2016     }
2017     else if (m_client)
2018     {
2019         yaz_log (LOG_LOG, "%sShutdown (client to proxy) close %s",
2020                  m_session_str,
2021                  m_client->get_hostname());
2022         assert (m_client->m_waiting != 2);
2023         delete m_client;
2024     }
2025     else if (!m_parent)
2026     {
2027         yaz_log (LOG_LOG, "%sshutdown (client to proxy) bad state",
2028                  m_session_str);
2029         assert (m_parent);
2030     }
2031     else
2032     {
2033         yaz_log (LOG_LOG, "%sShutdown (client to proxy)",
2034                  m_session_str);
2035     }
2036     if (m_parent)
2037         m_parent->pre_init();
2038     delete this;
2039 }
2040
2041 const char *Yaz_ProxyClient::get_session_str()
2042 {
2043     if (!m_server)
2044         return "0 ";
2045     return m_server->get_session_str();
2046 }
2047
2048 void Yaz_ProxyClient::shutdown()
2049 {
2050     yaz_log (LOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2051              get_hostname());
2052     delete m_server;
2053     delete this;
2054 }
2055
2056 void Yaz_Proxy::failNotify()
2057 {
2058     inc_request_no();
2059     yaz_log (LOG_LOG, "%sConnection closed by client",
2060              get_session_str());
2061     shutdown();
2062 }
2063
2064 void Yaz_ProxyClient::failNotify()
2065 {
2066     if (m_server)
2067         m_server->inc_request_no();
2068     yaz_log (LOG_LOG, "%sConnection closed by target %s",
2069              get_session_str(), get_hostname());
2070     shutdown();
2071 }
2072
2073 void Yaz_ProxyClient::connectNotify()
2074 {
2075     const char *s = get_session_str();
2076     const char *h = get_hostname();
2077     yaz_log (LOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2078              m_target_idletime);
2079     timeout(m_target_idletime);
2080     if (!m_server)
2081         pre_init_client();
2082 }
2083
2084 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2085                                                   *the_PDU_Observable, int fd)
2086 {
2087     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2088 }
2089
2090 Yaz_ProxyClient::~Yaz_ProxyClient()
2091 {
2092     if (m_prev)
2093         *m_prev = m_next;
2094     if (m_next)
2095         m_next->m_prev = m_prev;
2096     m_waiting = 2;     // for debugging purposes only.
2097     odr_destroy(m_init_odr);
2098     delete m_last_query;
2099     xfree (m_last_resultSetId);
2100     xfree (m_cookie);
2101 }
2102
2103 void Yaz_ProxyClient::pre_init_client()
2104 {
2105     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2106     Z_InitRequest *req = apdu->u.initRequest;
2107
2108     int i;
2109     for (i = 0; i<= 24; i++)
2110         ODR_MASK_SET(req->options, i);
2111     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2112                    Z_Options_negotiationModel);
2113     for (i = 0; i<= 10; i++)
2114         ODR_MASK_SET(req->protocolVersion, i);
2115
2116     if (send_to_target(apdu) < 0)
2117     {
2118         delete this;
2119     }
2120     else
2121     {
2122         m_waiting = 1;
2123         m_init_flag = 1;
2124     }
2125 }
2126
2127 void Yaz_Proxy::pre_init()
2128 {
2129     int i;
2130     const char *name = 0;
2131     const char *zurl_in_use[MAX_ZURL_PLEX];
2132     int limit_bw, limit_pdu, limit_req;
2133     int target_idletime, client_idletime;
2134     int max_clients;
2135     int keepalive_limit_bw, keepalive_limit_pdu;
2136     int pre_init;
2137     const char *cql2rpn = 0;
2138
2139     Yaz_ProxyConfig *cfg = check_reconfigure();
2140
2141     zurl_in_use[0] = 0;
2142
2143     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2144         set_APDU_yazlog(1);
2145     else
2146         set_APDU_yazlog(0);
2147
2148     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2149                                           &limit_bw, &limit_pdu, &limit_req,
2150                                           &target_idletime, &client_idletime,
2151                                           &max_clients,
2152                                           &keepalive_limit_bw,
2153                                           &keepalive_limit_pdu,
2154                                           &pre_init,
2155                                           &cql2rpn) ; i++)
2156     {
2157         if (pre_init)
2158         {
2159             int j;
2160             for (j = 0; zurl_in_use[j]; j++)
2161             {
2162                 Yaz_ProxyClient *c;
2163                 int spare = 0;
2164                 int in_use = 0;
2165                 int other = 0;
2166                 for (c = m_clientPool; c; c = c->m_next)
2167                 {
2168                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
2169                     {
2170                         if (c->m_cookie == 0)
2171                         {
2172                             if (c->m_server == 0)
2173                                 spare++;
2174                             else
2175                                 in_use++;
2176                         }
2177                         else
2178                             other++;
2179                     }
2180                 }
2181                 yaz_log(LOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2182                         "preinit=%d",m_session_str,
2183                         name, zurl_in_use[j], in_use, other, spare, pre_init);
2184                 if (spare < pre_init)
2185                 {
2186                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2187                     c->m_next = m_clientPool;
2188                     if (c->m_next)
2189                         c->m_next->m_prev = &c->m_next;
2190                     m_clientPool = c;
2191                     c->m_prev = &m_clientPool;
2192
2193                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
2194                         c->set_APDU_yazlog(1);
2195                     else
2196                         c->set_APDU_yazlog(0);
2197
2198                     if (c->client(zurl_in_use[j]))
2199                     {
2200                         timeout(60);
2201                         delete c;
2202                         return;
2203                     }
2204                     c->timeout(30);
2205                     c->m_waiting = 1;
2206                     c->m_target_idletime = target_idletime;
2207                     c->m_seqno = m_seqno++;
2208                 }
2209             }
2210         }
2211     }
2212 }
2213
2214 void Yaz_Proxy::timeoutNotify()
2215 {
2216     if (m_parent)
2217     {
2218         if (m_bw_hold_PDU)
2219         {
2220             timeout(m_client_idletime);
2221             Z_GDU *apdu = m_bw_hold_PDU;
2222             m_bw_hold_PDU = 0;
2223
2224             if (apdu->which == Z_GDU_Z3950)
2225                 handle_incoming_Z_PDU(apdu->u.z3950);
2226             else if (apdu->which == Z_GDU_HTTP_Request)
2227                 handle_incoming_HTTP(apdu->u.HTTP_Request);
2228         }
2229         else
2230         {
2231             inc_request_no();
2232
2233             yaz_log (LOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2234             shutdown();
2235         }
2236     }
2237     else
2238     {
2239         timeout(600);
2240         pre_init();
2241     }
2242 }
2243
2244 void Yaz_Proxy::markInvalid()
2245 {
2246     m_client = 0;
2247     m_invalid_session = 1;
2248 }
2249
2250 void Yaz_ProxyClient::timeoutNotify()
2251 {
2252     if (m_server)
2253         m_server->inc_request_no();
2254
2255     yaz_log (LOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2256              get_hostname());
2257     m_waiting = 1;
2258     m_root->pre_init();
2259     if (m_server && m_init_flag)
2260     {
2261         // target timed out in a session that was properly initialized
2262         // server object stay alive but we mark it as invalid so it
2263         // gets initialized again
2264         m_server->markInvalid();
2265         m_server = 0;
2266     }
2267     shutdown();
2268 }
2269
2270 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2271                                  Yaz_Proxy *parent) :
2272     Yaz_Z_Assoc (the_PDU_Observable)
2273 {
2274     m_cookie = 0;
2275     m_next = 0;
2276     m_prev = 0;
2277     m_init_flag = 0;
2278     m_last_query = 0;
2279     m_last_resultSetId = 0;
2280     m_last_resultCount = 0;
2281     m_last_ok = 0;
2282     m_sr_transform = 0;
2283     m_waiting = 0;
2284     m_init_odr = odr_createmem (ODR_DECODE);
2285     m_initResponse = 0;
2286     m_initResponse_options = 0;
2287     m_initResponse_version = 0;
2288     m_resultSetStartPoint = 0;
2289     m_bytes_sent = m_bytes_recv = 0;
2290     m_pdu_recv = 0;
2291     m_server = 0;
2292     m_seqno = 0;
2293     m_target_idletime = 600;
2294     m_root = parent;
2295 }
2296
2297 const char *Yaz_Proxy::option(const char *name, const char *value)
2298 {
2299     if (!strcmp (name, "optimize")) {
2300         if (value) {
2301             xfree (m_optimize);
2302             m_optimize = xstrdup (value);
2303         }
2304         return m_optimize;
2305     }
2306     return 0;
2307 }
2308
2309 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2310 {
2311
2312 }
2313
2314 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2315 {
2316     if (apdu->which == Z_GDU_Z3950)
2317         recv_Z_PDU(apdu->u.z3950, len);
2318     else if (apdu->which == Z_GDU_HTTP_Response)
2319         recv_HTTP_response(apdu->u.HTTP_Response, len);
2320     else
2321         shutdown();
2322 }
2323
2324 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2325 {
2326     if (!m_invalid_session)
2327         return 0;
2328     m_invalid_session = 0;
2329     handle_incoming_Z_PDU(m_apdu_invalid_session);
2330     assert (m_mem_invalid_session);
2331     nmem_destroy(m_mem_invalid_session);
2332     m_mem_invalid_session = 0;
2333     return 1;
2334 }
2335
2336 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2337 {
2338     m_bytes_recv += len;
2339     m_pdu_recv++;
2340     m_waiting = 0;
2341     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2342         yaz_log (LOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2343                  apdu_name(apdu), get_hostname(), len);
2344     if (apdu->which == Z_APDU_initResponse)
2345     {
2346         if (!m_server)  // if this is a pre init session , check for more
2347             m_root->pre_init();
2348         NMEM nmem = odr_extract_mem (odr_decode());
2349         odr_reset (m_init_odr);
2350         nmem_transfer (m_init_odr->mem, nmem);
2351         m_initResponse = apdu;
2352         m_initResponse_options = apdu->u.initResponse->options;
2353         m_initResponse_version = apdu->u.initResponse->protocolVersion;
2354
2355         Z_InitResponse *ir = apdu->u.initResponse;
2356         char *im0 = ir->implementationName;
2357
2358         char *im1 = (char*)
2359             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2360         *im1 = '\0';
2361         if (im0)
2362         {
2363             strcat(im1, im0);
2364             strcat(im1, " ");
2365         }
2366         strcat(im1, "(YAZ Proxy)");
2367         ir->implementationName = im1;
2368
2369         nmem_destroy (nmem);
2370
2371         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2372             return;
2373     }
2374     if (apdu->which == Z_APDU_searchResponse)
2375     {
2376         Z_SearchResponse *sr = apdu->u.searchResponse;
2377         m_last_resultCount = *sr->resultCount;
2378         int status = *sr->searchStatus;
2379         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2380         {
2381             m_last_ok = 1;
2382
2383             if (sr->records && sr->records->which == Z_Records_DBOSD)
2384             {
2385                 m_cache.add(odr_decode(),
2386                             sr->records->u.databaseOrSurDiagnostics, 1,
2387                             *sr->resultCount);
2388             }
2389         }
2390     }
2391     if (apdu->which == Z_APDU_presentResponse)
2392     {
2393         Z_PresentResponse *pr = apdu->u.presentResponse;
2394         if (m_sr_transform)
2395         {
2396             m_sr_transform = 0;
2397             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2398             Z_SearchResponse *sr = new_apdu->u.searchResponse;
2399             sr->referenceId = pr->referenceId;
2400             *sr->resultCount = m_last_resultCount;
2401             sr->records = pr->records;
2402             sr->nextResultSetPosition = pr->nextResultSetPosition;
2403             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2404             apdu = new_apdu;
2405         }
2406         if (pr->records &&
2407             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2408         {
2409             m_cache.add(odr_decode(),
2410                         pr->records->u.databaseOrSurDiagnostics,
2411                         m_resultSetStartPoint, -1);
2412             m_resultSetStartPoint = 0;
2413         }
2414     }
2415     if (m_cookie)
2416         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2417     if (m_server)
2418     {
2419         m_server->send_to_client(apdu);
2420     }
2421     if (apdu->which == Z_APDU_close)
2422     {
2423         shutdown();
2424     }
2425 }
2426
2427 int Yaz_Proxy::server(const char *addr)
2428 {
2429     int r = Yaz_Z_Assoc::server(addr);
2430     if (!r)
2431     {
2432         yaz_log(LOG_LOG, "%sStarted proxy " VERSION " on %s", m_session_str, addr);
2433         timeout(1);
2434     }
2435     return r;
2436 }
2437