src/yaz-proxy.cpp

   1 /*
   2  * Copyright (c) 1998-2004, Index Data.
   3  * See the file LICENSE for details.
   4  *
   5  * $Id: yaz-proxy.cpp,v 1.96 2004-02-02 11:17:45 adam Exp $
   6  */
   7
   8 #include <assert.h>
   9 #include <time.h>
  10
  11 #include <yaz/srw.h>
  12 #include <yaz/marcdisp.h>
  13 #include <yaz/yaz-iconv.h>
  14 #include <yaz/log.h>
  15 #include <yaz/diagbib1.h>
  16 #include <yaz++/proxy.h>
  17 #include <yaz/pquery.h>
  18
  19 static const char *apdu_name(Z_APDU *apdu)
  20 {
  21     switch (apdu->which)
  22     {
  23     case Z_APDU_initRequest:
  24         return "initRequest";
  25     case Z_APDU_initResponse:
  26         return "initResponse";
  27     case Z_APDU_searchRequest:
  28         return "searchRequest";
  29     case Z_APDU_searchResponse:
  30         return "searchResponse";
  31     case Z_APDU_presentRequest:
  32         return "presentRequest";
  33     case Z_APDU_presentResponse:
  34         return "presentResponse";
  35     case Z_APDU_deleteResultSetRequest:
  36         return "deleteResultSetRequest";
  37     case Z_APDU_deleteResultSetResponse:
  38         return "deleteResultSetResponse";
  39     case Z_APDU_scanRequest:
  40         return "scanRequest";
  41     case Z_APDU_scanResponse:
  42         return "scanResponse";
  43     case Z_APDU_sortRequest:
  44         return "sortRequest";
  45     case Z_APDU_sortResponse:
  46         return "sortResponse";
  47     case Z_APDU_extendedServicesRequest:
  48         return "extendedServicesRequest";
  49     case Z_APDU_extendedServicesResponse:
  50         return "extendedServicesResponse";
  51     case Z_APDU_close:
  52         return "close";
  53     }
  54     return "other";
  55 }
  56
  57 static const char *gdu_name(Z_GDU *gdu)
  58 {
  59     switch(gdu->which)
  60     {
  61     case Z_GDU_Z3950:
  62         return apdu_name(gdu->u.z3950);
  63     case Z_GDU_HTTP_Request:
  64         return "HTTP Request";
  65     case Z_GDU_HTTP_Response:
  66         return "HTTP Response";
  67     }
  68     return "Unknown request/response";
  69 }
  70
  71 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
  72                      Yaz_Proxy *parent) :
  73     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
  74 {
  75     m_PDU_Observable = the_PDU_Observable;
  76     m_client = 0;
  77     m_parent = parent;
  78     m_clientPool = 0;
  79     m_seqno = 1;
  80     m_keepalive_limit_bw = 500000;
  81     m_keepalive_limit_pdu = 1000;
  82     m_proxyTarget = 0;
  83     m_default_target = 0;
  84     m_proxy_authentication = 0;
  85     m_max_clients = 150;
  86     m_log_mask = 0;
  87     m_seed = time(0);
  88     m_client_idletime = 600;
  89     m_target_idletime = 600;
  90     m_optimize = xstrdup ("1");
  91     strcpy(m_session_str, "0 ");
  92     m_session_no=0;
  93     m_bytes_sent = 0;
  94     m_bytes_recv = 0;
  95     m_bw_hold_PDU = 0;
  96     m_bw_max = 0;
  97     m_pdu_max = 0;
  98     m_max_record_retrieve = 0;
  99     m_reconfig_flag = 0;
 100     m_config_fname = 0;
 101     m_request_no = 0;
 102     m_invalid_session = 0;
 103     m_config = 0;
 104     m_marcxml_flag = 0;
 105     m_stylesheet_xsp = 0;
 106     m_stylesheet_nprl = 0;
 107     m_s2z_stylesheet = 0;
 108     m_s2z_database = 0;
 109     m_schema = 0;
 110     m_initRequest_apdu = 0;
 111     m_initRequest_mem = 0;
 112     m_initRequest_options = 0;
 113     m_initRequest_version = 0;
 114     m_apdu_invalid_session = 0;
 115     m_mem_invalid_session = 0;
 116     m_s2z_odr_init = 0;
 117     m_s2z_odr_search = 0;
 118     m_s2z_init_apdu = 0;
 119     m_s2z_search_apdu = 0;
 120     m_s2z_present_apdu = 0;
 121     m_http_keepalive = 0;
 122     m_http_version = 0;
 123     m_soap_ns = 0;
 124     m_s2z_packing = Z_SRW_recordPacking_string;
 125     m_time_tv.tv_sec = 0;
 126     m_time_tv.tv_usec = 0;
 127 }
 128
 129 Yaz_Proxy::~Yaz_Proxy()
 130 {
 131     yaz_log(LOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 132             m_bytes_sent, m_bytes_recv);
 133     nmem_destroy(m_initRequest_mem);
 134     nmem_destroy(m_mem_invalid_session);
 135     xfree (m_proxyTarget);
 136     xfree (m_default_target);
 137     xfree (m_proxy_authentication);
 138     xfree (m_optimize);
 139
 140     if (m_stylesheet_xsp)
 141         xsltFreeStylesheet(m_stylesheet_xsp);
 142
 143     xfree (m_schema);
 144     if (m_s2z_odr_init)
 145         odr_destroy(m_s2z_odr_init);
 146     if (m_s2z_odr_search)
 147         odr_destroy(m_s2z_odr_search);
 148     delete m_config;
 149 }
 150
 151 int Yaz_Proxy::set_config(const char *config)
 152 {
 153     delete m_config;
 154     m_config = new Yaz_ProxyConfig();
 155     xfree(m_config_fname);
 156     m_config_fname = xstrdup(config);
 157     int r = m_config->read_xml(config);
 158     if (!r)
 159         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 160     return r;
 161 }
 162
 163 void Yaz_Proxy::set_default_target(const char *target)
 164 {
 165     xfree (m_default_target);
 166     m_default_target = 0;
 167     if (target)
 168         m_default_target = (char *) xstrdup (target);
 169 }
 170
 171 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 172 {
 173     xfree (m_proxy_authentication);
 174     m_proxy_authentication = 0;
 175     if (auth)
 176         m_proxy_authentication = (char *) xstrdup (auth);
 177 }
 178
 179 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 180 {
 181     if (m_parent)
 182         return m_parent->check_reconfigure();
 183
 184     Yaz_ProxyConfig *cfg = m_config;
 185     if (m_reconfig_flag)
 186     {
 187         yaz_log(LOG_LOG, "reconfigure");
 188         yaz_log_reopen();
 189         if (m_config_fname && cfg)
 190         {
 191             yaz_log(LOG_LOG, "reconfigure config %s", m_config_fname);
 192             int r = cfg->read_xml(m_config_fname);
 193             if (r)
 194                 yaz_log(LOG_WARN, "reconfigure failed");
 195             else
 196             {
 197                 m_log_mask = 0;
 198                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 199             }
 200         }
 201         else
 202             yaz_log(LOG_LOG, "reconfigure");
 203         m_reconfig_flag = 0;
 204     }
 205     return cfg;
 206 }
 207
 208 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 209                                             *the_PDU_Observable, int fd)
 210 {
 211     check_reconfigure();
 212     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 213     new_proxy->m_config = 0;
 214     new_proxy->m_config_fname = 0;
 215     new_proxy->timeout(m_client_idletime);
 216     new_proxy->m_target_idletime = m_target_idletime;
 217     new_proxy->set_default_target(m_default_target);
 218     new_proxy->m_max_clients = m_max_clients;
 219     new_proxy->m_log_mask = m_log_mask;
 220     new_proxy->set_APDU_log(get_APDU_log());
 221     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 222         new_proxy->set_APDU_yazlog(1);
 223     else
 224         new_proxy->set_APDU_yazlog(0);
 225     new_proxy->set_proxy_authentication(m_proxy_authentication);
 226     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 227     m_session_no++;
 228     yaz_log (LOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 229              the_PDU_Observable->getpeername());
 230     return new_proxy;
 231 }
 232
 233 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 234 {
 235     int oid[OID_SIZE];
 236     Z_OtherInformationUnit *oi;
 237     struct oident ent;
 238     ent.proto = PROTO_Z3950;
 239     ent.oclass = CLASS_USERINFO;
 240     ent.value = (oid_value) VAL_COOKIE;
 241     assert (oid_ent_to_oid (&ent, oid));
 242
 243     if (oid_ent_to_oid (&ent, oid) &&
 244         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 245         oi->which == Z_OtherInfo_characterInfo)
 246         return oi->information.characterInfo;
 247     return 0;
 248 }
 249
 250 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 251 {
 252     int oid[OID_SIZE];
 253     Z_OtherInformationUnit *oi;
 254     struct oident ent;
 255     ent.proto = PROTO_Z3950;
 256     ent.oclass = CLASS_USERINFO;
 257     ent.value = (oid_value) VAL_PROXY;
 258     if (oid_ent_to_oid (&ent, oid) &&
 259         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 260         oi->which == Z_OtherInfo_characterInfo)
 261         return oi->information.characterInfo;
 262     return 0;
 263 }
 264
 265 const char *Yaz_Proxy::load_balance(const char **url)
 266 {
 267     int zurl_in_use[MAX_ZURL_PLEX];
 268     Yaz_ProxyClient *c;
 269     int i;
 270
 271     for (i = 0; i<MAX_ZURL_PLEX; i++)
 272         zurl_in_use[i] = 0;
 273     for (c = m_parent->m_clientPool; c; c = c->m_next)
 274     {
 275         for (i = 0; url[i]; i++)
 276             if (!strcmp(url[i], c->get_hostname()))
 277                 zurl_in_use[i]++;
 278     }
 279     int min = 100000;
 280     const char *ret = 0;
 281     for (i = 0; url[i]; i++)
 282     {
 283         yaz_log(LOG_DEBUG, "%szurl=%s use=%d",
 284                 m_session_str, url[i], zurl_in_use[i]);
 285         if (min > zurl_in_use[i])
 286         {
 287             ret = url[i];
 288             min = zurl_in_use[i];
 289         }
 290     }
 291     return ret;
 292 }
 293
 294 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 295                                        const char *proxy_host)
 296 {
 297     assert (m_parent);
 298     Yaz_Proxy *parent = m_parent;
 299     Yaz_ProxyClient *c = m_client;
 300
 301     if (!m_proxyTarget)
 302     {
 303         const char *url[MAX_ZURL_PLEX];
 304         Yaz_ProxyConfig *cfg = check_reconfigure();
 305         if (proxy_host)
 306         {
 307 #if 1
 308 /* only to be enabled for debugging... */
 309             if (!strcmp(proxy_host, "stop"))
 310                 exit(0);
 311 #endif
 312             xfree(m_default_target);
 313             m_default_target = xstrdup(proxy_host);
 314             proxy_host = m_default_target;
 315         }
 316         int client_idletime = -1;
 317         const char *cql2rpn_fname = 0;
 318         url[0] = m_default_target;
 319         url[1] = 0;
 320         if (cfg)
 321         {
 322             int pre_init = 0;
 323             cfg->get_target_info(proxy_host, url, &m_bw_max,
 324                                  &m_pdu_max, &m_max_record_retrieve,
 325                                  &m_target_idletime, &client_idletime,
 326                                  &parent->m_max_clients,
 327                                  &m_keepalive_limit_bw,
 328                                  &m_keepalive_limit_pdu,
 329                                  &pre_init,
 330                                  &cql2rpn_fname);
 331         }
 332         if (client_idletime != -1)
 333         {
 334             m_client_idletime = client_idletime;
 335             timeout(m_client_idletime);
 336         }
 337         if (cql2rpn_fname)
 338             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 339         if (!url[0])
 340         {
 341             yaz_log(LOG_LOG, "%sNo default target", m_session_str);
 342             return 0;
 343         }
 344         // we don't handle multiplexing for cookie session, so we just
 345         // pick the first one in this case (anonymous users will be able
 346         // to use any backend)
 347         if (cookie && *cookie)
 348             m_proxyTarget = (char*) xstrdup(url[0]);
 349         else
 350             m_proxyTarget = (char*) xstrdup(load_balance(url));
 351     }
 352     if (cookie && *cookie)
 353     {   // search in sessions with a cookie
 354         for (c = parent->m_clientPool; c; c = c->m_next)
 355         {
 356             assert (c->m_prev);
 357             assert (*c->m_prev == c);
 358             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 359                 !strcmp(m_proxyTarget, c->get_hostname()))
 360             {
 361                 // Found it in cache
 362                 // The following handles "cancel"
 363                 // If connection is busy (waiting for PDU) and
 364                 // we have an initRequest we can safely do re-open
 365                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 366                 {
 367                     yaz_log (LOG_LOG, "%s REOPEN target=%s", m_session_str,
 368                              c->get_hostname());
 369                     c->close();
 370                     c->m_init_flag = 0;
 371
 372                     c->m_last_ok = 0;
 373                     c->m_cache.clear();
 374                     c->m_last_resultCount = 0;
 375                     c->m_sr_transform = 0;
 376                     c->m_waiting = 0;
 377                     c->m_resultSetStartPoint = 0;
 378                     c->m_target_idletime = m_target_idletime;
 379                     if (c->client(m_proxyTarget))
 380                     {
 381                         delete c;
 382                         return 0;
 383                     }
 384                     c->timeout(30);
 385                 }
 386                 c->m_seqno = parent->m_seqno;
 387                 if (c->m_server && c->m_server != this)
 388                     c->m_server->m_client = 0;
 389                 c->m_server = this;
 390                 (parent->m_seqno)++;
 391                 yaz_log (LOG_DEBUG, "get_client 1 %p %p", this, c);
 392                 return c;
 393             }
 394         }
 395     }
 396     else if (!c)
 397     {
 398         // don't have a client session yet. Search in session w/o cookie
 399         for (c = parent->m_clientPool; c; c = c->m_next)
 400         {
 401             assert (c->m_prev);
 402             assert (*c->m_prev == c);
 403             if (c->m_server == 0 && c->m_cookie == 0 &&
 404                 c->m_waiting == 0 &&
 405                 !strcmp(m_proxyTarget, c->get_hostname()))
 406             {
 407                 // found it in cache
 408                 yaz_log (LOG_LOG, "%sREUSE %s",
 409                          m_session_str, c->get_hostname());
 410
 411                 c->m_seqno = parent->m_seqno;
 412                 assert(c->m_server == 0);
 413                 c->m_server = this;
 414
 415                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 416                     c->set_APDU_yazlog(1);
 417                 else
 418                     c->set_APDU_yazlog(0);
 419
 420                 (parent->m_seqno)++;
 421
 422                 parent->pre_init();
 423
 424                 return c;
 425             }
 426         }
 427     }
 428     if (!m_client)
 429     {
 430         if (apdu->which != Z_APDU_initRequest)
 431         {
 432             yaz_log (LOG_LOG, "%sno init request as first PDU", m_session_str);
 433             return 0;
 434         }
 435         Z_InitRequest *initRequest = apdu->u.initRequest;
 436
 437         if (!initRequest->idAuthentication)
 438         {
 439             if (m_proxy_authentication)
 440             {
 441                 initRequest->idAuthentication =
 442                     (Z_IdAuthentication *)
 443                     odr_malloc (odr_encode(),
 444                                 sizeof(*initRequest->idAuthentication));
 445                 initRequest->idAuthentication->which =
 446                     Z_IdAuthentication_open;
 447                 initRequest->idAuthentication->u.open =
 448                     odr_strdup (odr_encode(), m_proxy_authentication);
 449             }
 450         }
 451         // go through list of clients - and find the lowest/oldest one.
 452         Yaz_ProxyClient *c_min = 0;
 453         int min_seq = -1;
 454         int no_of_clients = 0;
 455         if (parent->m_clientPool)
 456             yaz_log (LOG_DEBUG, "Existing sessions");
 457         for (c = parent->m_clientPool; c; c = c->m_next)
 458         {
 459             yaz_log (LOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 460                                c->m_waiting, c->get_hostname(),
 461                                c->m_cookie ? c->m_cookie : "");
 462             no_of_clients++;
 463             if (min_seq < 0 || c->m_seqno < min_seq)
 464             {
 465                 min_seq = c->m_seqno;
 466                 c_min = c;
 467             }
 468         }
 469         if (no_of_clients >= parent->m_max_clients)
 470         {
 471             c = c_min;
 472             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 473             {
 474                 yaz_log (LOG_LOG, "%sMAXCLIENTS Destroy %d",
 475                          m_session_str, c->m_seqno);
 476                 if (c->m_server && c->m_server != this)
 477                     delete c->m_server;
 478                 c->m_server = 0;
 479             }
 480             else
 481             {
 482                 yaz_log (LOG_LOG, "%sMAXCLIENTS Reuse %d %d %s",
 483                          m_session_str,
 484                          c->m_seqno, parent->m_seqno, c->get_hostname());
 485                 xfree (c->m_cookie);
 486                 c->m_cookie = 0;
 487                 if (cookie)
 488                     c->m_cookie = xstrdup(cookie);
 489                 c->m_seqno = parent->m_seqno;
 490                 if (c->m_server && c->m_server != this)
 491                 {
 492                     c->m_server->m_client = 0;
 493                     delete c->m_server;
 494                 }
 495                 (parent->m_seqno)++;
 496                 c->m_target_idletime = m_target_idletime;
 497                 c->timeout(m_target_idletime);
 498
 499                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 500                     c->set_APDU_yazlog(1);
 501                 else
 502                     c->set_APDU_yazlog(0);
 503
 504                 return c;
 505             }
 506         }
 507         else
 508         {
 509             yaz_log (LOG_LOG, "%sNEW %d %s",
 510                      m_session_str, parent->m_seqno, m_proxyTarget);
 511             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 512             c->m_next = parent->m_clientPool;
 513             if (c->m_next)
 514                 c->m_next->m_prev = &c->m_next;
 515             parent->m_clientPool = c;
 516             c->m_prev = &parent->m_clientPool;
 517         }
 518
 519         xfree (c->m_cookie);
 520         c->m_cookie = 0;
 521         if (cookie)
 522             c->m_cookie = xstrdup(cookie);
 523
 524         c->m_seqno = parent->m_seqno;
 525         c->m_init_flag = 0;
 526         c->m_last_resultCount = 0;
 527         c->m_last_ok = 0;
 528         c->m_cache.clear();
 529         c->m_sr_transform = 0;
 530         c->m_waiting = 0;
 531         c->m_resultSetStartPoint = 0;
 532         (parent->m_seqno)++;
 533         if (c->client(m_proxyTarget))
 534         {
 535             delete c;
 536             return 0;
 537         }
 538         c->m_target_idletime = m_target_idletime;
 539         c->timeout(30);
 540
 541         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 542             c->set_APDU_yazlog(1);
 543         else
 544             c->set_APDU_yazlog(0);
 545     }
 546     yaz_log (LOG_DEBUG, "get_client 3 %p %p", this, c);
 547     return c;
 548 }
 549
 550 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 551 {
 552     int i;
 553     for (i = 0; i<num; i++)
 554     {
 555         oident *ent;
 556         Z_DefaultDiagFormat *r;
 557         Z_DiagRec *p = pp[i];
 558         if (p->which != Z_DiagRec_defaultFormat)
 559         {
 560             yaz_log(LOG_LOG, "%sError no diagnostics", m_session_str);
 561             return;
 562         }
 563         else
 564             r = p->u.defaultFormat;
 565         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 566             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 567             yaz_log(LOG_LOG, "%sError unknown diagnostic set", m_session_str);
 568         switch (r->which)
 569         {
 570         case Z_DefaultDiagFormat_v2Addinfo:
 571             yaz_log(LOG_LOG, "%sError %d %s:%s",
 572                     m_session_str,
 573                     *r->condition, diagbib1_str(*r->condition),
 574                     r->u.v2Addinfo);
 575             break;
 576         case Z_DefaultDiagFormat_v3Addinfo:
 577             yaz_log(LOG_LOG, "%sError %d %s:%s",
 578                     m_session_str,
 579                     *r->condition, diagbib1_str(*r->condition),
 580                     r->u.v3Addinfo);
 581             break;
 582         }
 583     }
 584 }
 585
 586 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 587 {
 588     if (!m_stylesheet_xsp || p->num_records <= 0)
 589         return 0;  /* no XSLT to be done ... */
 590
 591     m_stylesheet_offset = 0;
 592     m_stylesheet_nprl = p;
 593     m_stylesheet_apdu = apdu;
 594     timeout(0);
 595     return 1;
 596 }
 597
 598 void Yaz_Proxy::convert_xsl_delay()
 599 {
 600     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 601     if (npr->which == Z_NamePlusRecord_databaseRecord)
 602     {
 603         Z_External *r = npr->u.databaseRecord;
 604         if (r->which == Z_External_octet)
 605         {
 606             xmlDocPtr res, doc = xmlParseMemory(
 607                 (char*) r->u.octet_aligned->buf,
 608                 r->u.octet_aligned->len);
 609
 610             yaz_log(LOG_LOG, "%sXSLT convert %d",
 611                     m_session_str, m_stylesheet_offset);
 612             res = xsltApplyStylesheet(m_stylesheet_xsp, doc, 0);
 613             if (res)
 614             {
 615                 xmlChar *out_buf;
 616                 int out_len;
 617                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 618
 619                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 620                     u.databaseRecord =
 621                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 622                                  (char*) out_buf, out_len);
 623                 xmlFree(out_buf);
 624                 xmlFreeDoc(res);
 625             }
 626             xmlFreeDoc(doc);
 627         }
 628     }
 629     m_stylesheet_offset++;
 630     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 631     {
 632         m_stylesheet_nprl = 0;
 633         if (m_stylesheet_xsp)
 634             xsltFreeStylesheet(m_stylesheet_xsp);
 635         m_stylesheet_xsp = 0;
 636         timeout(m_client_idletime);
 637         int r = send_PDU_convert(m_stylesheet_apdu);
 638     }
 639     else
 640         timeout(0);
 641 }
 642
 643 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p)
 644 {
 645     int i;
 646
 647     yaz_marc_t mt = yaz_marc_create();
 648     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 649     for (i = 0; i < p->num_records; i++)
 650     {
 651         Z_NamePlusRecord *npr = p->records[i];
 652         if (npr->which == Z_NamePlusRecord_databaseRecord)
 653         {
 654             Z_External *r = npr->u.databaseRecord;
 655             if (r->which == Z_External_octet)
 656             {
 657                 int rlen;
 658                 char *result;
 659                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 660                                         r->u.octet_aligned->len,
 661                                         &result, &rlen))
 662                 {
 663                     yaz_iconv_t cd = yaz_iconv_open("UTF-8", "MARC-8");
 664                     WRBUF wrbuf = wrbuf_alloc();
 665
 666                     char outbuf[120];
 667                     size_t inbytesleft = rlen;
 668                     const char *inp = result;
 669                     while (cd && inbytesleft)
 670                     {
 671                         size_t outbytesleft = sizeof(outbuf);
 672                         char *outp = outbuf;
 673                         size_t r;
 674
 675                         r = yaz_iconv (cd, (char**) &inp,
 676                                        &inbytesleft,
 677                                        &outp, &outbytesleft);
 678                         if (r == (size_t) (-1))
 679                         {
 680                             int e = yaz_iconv_error(cd);
 681                             if (e != YAZ_ICONV_E2BIG)
 682                             {
 683                                 yaz_log(LOG_WARN, "conversion failure");
 684                                 break;
 685                             }
 686                         }
 687                         wrbuf_write(wrbuf, outbuf, outp - outbuf);
 688                     }
 689                     if (cd)
 690                         yaz_iconv_close(cd);
 691
 692                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 693                                                          VAL_TEXT_XML,
 694                                                          wrbuf_buf(wrbuf),
 695                                                          wrbuf_len(wrbuf));
 696                     wrbuf_free(wrbuf, 1);
 697                 }
 698             }
 699         }
 700     }
 701     yaz_marc_destroy(mt);
 702 }
 703
 704 void Yaz_Proxy::logtime()
 705 {
 706     if (m_time_tv.tv_sec)
 707     {
 708         struct timeval tv;
 709         gettimeofday(&tv, 0);
 710         long diff = (tv.tv_sec - m_time_tv.tv_sec)*1000000 +
 711             (tv.tv_usec - m_time_tv.tv_usec);
 712         if (diff >= 0)
 713             yaz_log(LOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 714                     diff/1000000, (diff/1000)%1000);
 715     }
 716     m_time_tv.tv_sec = 0;
 717     m_time_tv.tv_usec = 0;
 718 }
 719
 720 int Yaz_Proxy::send_http_response(int code)
 721 {
 722     ODR o = odr_encode();
 723     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 724     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 725     if (m_http_version)
 726         hres->version = odr_strdup(o, m_http_version);
 727     if (m_http_keepalive)
 728         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 729     else
 730         timeout(0);
 731
 732     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 733     {
 734         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 735                  gdu_name(gdu));
 736     }
 737     int len;
 738     int r = send_GDU(gdu, &len);
 739     m_bytes_sent += len;
 740     m_bw_stat.add_bytes(len);
 741     logtime();
 742     return r;
 743 }
 744
 745 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 746 {
 747     ODR o = odr_encode();
 748     const char *ctype = "text/xml";
 749     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 750     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 751     if (m_http_version)
 752         hres->version = odr_strdup(o, m_http_version);
 753     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 754     if (m_http_keepalive)
 755         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 756     else
 757         timeout(0);
 758
 759     static Z_SOAP_Handler soap_handlers[2] = {
 760 #if HAVE_XSLT
 761         {"http://www.loc.gov/zing/srw/", 0,
 762          (Z_SOAP_fun) yaz_srw_codec},
 763 #endif
 764         {0, 0, 0}
 765     };
 766
 767     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 768     soap_package->which = Z_SOAP_generic;
 769     soap_package->u.generic =
 770         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 771     soap_package->u.generic->no = 0;
 772     soap_package->u.generic->ns = soap_handlers[0].ns;
 773     soap_package->u.generic->p = (void *) srw_pdu;
 774     soap_package->ns = m_soap_ns;
 775     z_soap_codec_enc_xsl(o, &soap_package,
 776                          &hres->content_buf, &hres->content_len,
 777                          soap_handlers, 0, m_s2z_stylesheet);
 778     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 779     {
 780         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 781                  gdu_name(gdu));
 782     }
 783     int len;
 784     int r = send_GDU(gdu, &len);
 785     m_bytes_sent += len;
 786     m_bw_stat.add_bytes(len);
 787     logtime();
 788     return r;
 789 }
 790
 791 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
 792 {
 793     ODR o = odr_encode();
 794     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 795     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 796
 797     srw_res->num_diagnostics = 1;
 798     srw_res->diagnostics = (Z_SRW_diagnostic *)
 799         odr_malloc(o, sizeof(*srw_res->diagnostics));
 800     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
 801     return send_srw_response(srw_pdu);
 802 }
 803
 804 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 805                              Z_DefaultDiagFormat *ddf)
 806 {
 807     int bib1_code = *ddf->condition;
 808     if (bib1_code == 109)
 809         return 404;
 810     srw_res->num_diagnostics = 1;
 811     srw_res->diagnostics = (Z_SRW_diagnostic *)
 812         odr_malloc(o, sizeof(*srw_res->diagnostics));
 813     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
 814                           yaz_diag_bib1_to_srw(*ddf->condition),
 815                           ddf->u.v2Addinfo);
 816     return 0;
 817 }
 818
 819 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
 820 {
 821     ODR o = odr_encode();
 822     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 823     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 824
 825     srw_res->numberOfRecords = odr_intdup (o, hits);
 826     if (records && records->which == Z_Records_DBOSD)
 827     {
 828         srw_res->num_records =
 829             records->u.databaseOrSurDiagnostics->num_records;
 830         int i;
 831         srw_res->records = (Z_SRW_record *)
 832             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
 833         for (i = 0; i < srw_res->num_records; i++)
 834         {
 835             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
 836             if (npr->which != Z_NamePlusRecord_databaseRecord)
 837             {
 838                 srw_res->records[i].recordSchema = "diagnostic";
 839                 srw_res->records[i].recordPacking = m_s2z_packing;
 840                 srw_res->records[i].recordData_buf = "67";
 841                 srw_res->records[i].recordData_len = 2;
 842                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 843                 continue;
 844             }
 845             Z_External *r = npr->u.databaseRecord;
 846             oident *ent = oid_getentbyoid(r->direct_reference);
 847             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
 848             {
 849                 srw_res->records[i].recordSchema = m_schema;
 850                 srw_res->records[i].recordPacking = m_s2z_packing;
 851                 srw_res->records[i].recordData_buf = (char*)
 852                     r->u.octet_aligned->buf;
 853                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
 854                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 855             }
 856             else
 857             {
 858                 srw_res->records[i].recordSchema = "diagnostic";
 859                 srw_res->records[i].recordPacking = m_s2z_packing;
 860                 srw_res->records[i].recordData_buf = "67";
 861                 srw_res->records[i].recordData_len = 2;
 862                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 863             }
 864         }
 865     }
 866     if (records && records->which == Z_Records_NSD)
 867     {
 868         int http_code;
 869         http_code = z_to_srw_diag(odr_encode(), srw_res,
 870                                    records->u.nonSurrogateDiagnostic);
 871         if (http_code)
 872             return send_http_response(http_code);
 873     }
 874     return send_srw_response(srw_pdu);
 875
 876 }
 877
 878 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
 879                                         int num_diagnostics)
 880 {
 881     Yaz_ProxyConfig *cfg = check_reconfigure();
 882     if (cfg)
 883     {
 884         int len;
 885         char *b = cfg->get_explain(odr_encode(), 0 /* target */,
 886                                    m_s2z_database, &len);
 887         if (b)
 888         {
 889             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
 890             Z_SRW_explainResponse *er = res->u.explain_response;
 891
 892             er->record.recordData_buf = b;
 893             er->record.recordData_len = len;
 894             er->record.recordPacking = m_s2z_packing;
 895
 896             er->diagnostics = diagnostics;
 897             er->num_diagnostics = num_diagnostics;
 898             return send_srw_response(res);
 899         }
 900     }
 901     return send_http_response(404);
 902 }
 903
 904 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
 905 {
 906     if (m_http_version)
 907     {
 908         if (apdu->which == Z_APDU_initResponse)
 909         {
 910             Z_InitResponse *res = apdu->u.initResponse;
 911             if (*res->result == 0)
 912             {
 913                 send_to_srw_client_error(3, 0);
 914             }
 915             else if (!m_s2z_search_apdu)
 916             {
 917                 send_srw_explain_response(0, 0);
 918             }
 919             else
 920             {
 921                 handle_incoming_Z_PDU(m_s2z_search_apdu);
 922             }
 923         }
 924         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
 925         {
 926             m_s2z_search_apdu = 0;
 927             Z_SearchResponse *res = apdu->u.searchResponse;
 928             m_s2z_hit_count = *res->resultCount;
 929             if (res->records && res->records->which == Z_Records_NSD)
 930             {
 931                 send_to_srw_client_ok(0, res->records, 1);
 932             }
 933             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
 934             {
 935                 // adjust
 936                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
 937
 938                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
 939                 {
 940                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
 941                         > m_s2z_hit_count)
 942                         *pr->numberOfRecordsRequested =
 943                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
 944                 }
 945                 handle_incoming_Z_PDU(m_s2z_present_apdu);
 946             }
 947             else
 948             {
 949                 m_s2z_present_apdu = 0;
 950                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
 951             }
 952         }
 953         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
 954         {
 955             int start =
 956                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
 957
 958             m_s2z_present_apdu = 0;
 959             Z_PresentResponse *res = apdu->u.presentResponse;
 960             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
 961         }
 962     }
 963     else
 964     {
 965         int len = 0;
 966         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 967             yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 968                      apdu_name(apdu));
 969         int r = send_Z_PDU(apdu, &len);
 970         m_bytes_sent += len;
 971         m_bw_stat.add_bytes(len);
 972         logtime();
 973         return r;
 974     }
 975     return 0;
 976 }
 977
 978 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 979 {
 980     int kill_session = 0;
 981     if (apdu->which == Z_APDU_searchResponse)
 982     {
 983         Z_SearchResponse *sr = apdu->u.searchResponse;
 984         Z_Records *p = sr->records;
 985         if (p && p->which == Z_Records_NSD)
 986         {
 987             Z_DiagRec dr, *dr_p = &dr;
 988             dr.which = Z_DiagRec_defaultFormat;
 989             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 990
 991             *sr->searchStatus = 0;
 992             display_diagrecs(&dr_p, 1);
 993         }
 994         else
 995         {
 996             if (p && p->which == Z_Records_DBOSD)
 997             {
 998                 if (m_marcxml_flag)
 999                     convert_to_marcxml(p->u.databaseOrSurDiagnostics);
1000                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1001                     return 0;
1002
1003             }
1004             if (sr->resultCount)
1005             {
1006                 yaz_log(LOG_LOG, "%s%d hits", m_session_str,
1007                         *sr->resultCount);
1008                 if (*sr->resultCount < 0)
1009                 {
1010                     m_invalid_session = 1;
1011                     kill_session = 1;
1012
1013                     *sr->searchStatus = 0;
1014                     sr->records =
1015                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1016                     *sr->resultCount = 0;
1017                 }
1018             }
1019         }
1020     }
1021     else if (apdu->which == Z_APDU_presentResponse)
1022     {
1023         Z_PresentResponse *sr = apdu->u.presentResponse;
1024         Z_Records *p = sr->records;
1025         if (p && p->which == Z_Records_NSD)
1026         {
1027             Z_DiagRec dr, *dr_p = &dr;
1028             dr.which = Z_DiagRec_defaultFormat;
1029             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1030             if (*sr->presentStatus == Z_PresentStatus_success)
1031                 *sr->presentStatus = Z_PresentStatus_failure;
1032             display_diagrecs(&dr_p, 1);
1033         }
1034         if (p && p->which == Z_Records_DBOSD)
1035         {
1036             if (m_marcxml_flag)
1037                 convert_to_marcxml(p->u.databaseOrSurDiagnostics);
1038             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1039                 return 0;
1040         }
1041     }
1042     else if (apdu->which == Z_APDU_initResponse)
1043     {
1044         if (m_initRequest_options)
1045         {
1046             Z_Options *nopt =
1047                 (Odr_bitmask *)odr_malloc(odr_encode(),
1048                                           sizeof(Odr_bitmask));
1049             ODR_MASK_ZERO(nopt);
1050
1051             int i;
1052             for (i = 0; i<24; i++)
1053                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1054                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1055                     ODR_MASK_SET(nopt, i);
1056             apdu->u.initResponse->options = nopt;
1057         }
1058         if (m_initRequest_version)
1059         {
1060             Z_ProtocolVersion *nopt =
1061                 (Odr_bitmask *)odr_malloc(odr_encode(),
1062                                           sizeof(Odr_bitmask));
1063             ODR_MASK_ZERO(nopt);
1064
1065             int i;
1066             for (i = 0; i<8; i++)
1067                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1068                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1069                     ODR_MASK_SET(nopt, i);
1070             apdu->u.initResponse->protocolVersion = nopt;
1071         }
1072     }
1073     int r = send_PDU_convert(apdu);
1074     if (r)
1075         return r;
1076     if (kill_session)
1077     {
1078         delete m_client;
1079         m_client = 0;
1080         m_parent->pre_init();
1081     }
1082     return r;
1083 }
1084
1085 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1086 {
1087     int len = 0;
1088     const char *apdu_name_tmp = apdu_name(apdu);
1089     int r = send_Z_PDU(apdu, &len);
1090     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1091         yaz_log (LOG_LOG, "%sSending %s to %s %d bytes",
1092                  get_session_str(),
1093                  apdu_name_tmp, get_hostname(), len);
1094     m_bytes_sent += len;
1095     return r;
1096 }
1097
1098 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1099 {
1100     if (apdu->which == Z_APDU_presentRequest)
1101     {
1102         Z_PresentRequest *pr = apdu->u.presentRequest;
1103         int toget = *pr->numberOfRecordsRequested;
1104         int start = *pr->resultSetStartPoint;
1105
1106         yaz_log(LOG_LOG, "%sPresent %s %d+%d", m_session_str,
1107                 pr->resultSetId, start, toget);
1108
1109         if (*m_parent->m_optimize == '0')
1110             return apdu;
1111
1112         if (!m_client->m_last_resultSetId)
1113         {
1114             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1115             new_apdu->u.presentResponse->records =
1116                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1117                                                pr->resultSetId);
1118             send_to_client(new_apdu);
1119             return 0;
1120         }
1121         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1122         {
1123             if (start+toget-1 > m_client->m_last_resultCount)
1124             {
1125                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1126                 new_apdu->u.presentResponse->records =
1127                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1128                 send_to_client(new_apdu);
1129                 return 0;
1130             }
1131             Z_NamePlusRecordList *npr;
1132             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1133                                           pr->preferredRecordSyntax,
1134                                           pr->recordComposition))
1135             {
1136                 yaz_log (LOG_LOG, "%sReturned cached records for present request",
1137                          m_session_str);
1138                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1139                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1140
1141                 new_apdu->u.presentResponse->numberOfRecordsReturned
1142                     = odr_intdup(odr_encode(), toget);
1143
1144                 new_apdu->u.presentResponse->records = (Z_Records*)
1145                     odr_malloc(odr_encode(), sizeof(Z_Records));
1146                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1147                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1148                 new_apdu->u.presentResponse->nextResultSetPosition =
1149                     odr_intdup(odr_encode(), start+toget);
1150
1151                 send_to_client(new_apdu);
1152                 return 0;
1153             }
1154         }
1155     }
1156
1157     if (apdu->which != Z_APDU_searchRequest)
1158         return apdu;
1159     Z_SearchRequest *sr = apdu->u.searchRequest;
1160     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1161     Yaz_Z_Databases this_databases;
1162
1163     this_databases.set(sr->num_databaseNames, (const char **)
1164                        sr->databaseNames);
1165
1166     this_query->set_Z_Query(sr->query);
1167
1168     char query_str[120];
1169     this_query->print(query_str, sizeof(query_str)-1);
1170     yaz_log(LOG_LOG, "%sSearch %s", m_session_str, query_str);
1171
1172     if (*m_parent->m_optimize != '0' &&
1173         m_client->m_last_ok && m_client->m_last_query &&
1174         m_client->m_last_query->match(this_query) &&
1175         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1176         m_client->m_last_databases.match(this_databases))
1177     {
1178         delete this_query;
1179         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1180             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1181         {
1182             Z_NamePlusRecordList *npr;
1183             int toget = *sr->mediumSetPresentNumber;
1184             Z_RecordComposition *comp = 0;
1185
1186             if (toget > m_client->m_last_resultCount)
1187                 toget = m_client->m_last_resultCount;
1188
1189             if (sr->mediumSetElementSetNames)
1190             {
1191                 comp = (Z_RecordComposition *)
1192                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1193                 comp->which = Z_RecordComp_simple;
1194                 comp->u.simple = sr->mediumSetElementSetNames;
1195             }
1196
1197             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1198                                           sr->preferredRecordSyntax, comp))
1199             {
1200                 yaz_log (LOG_LOG, "%sReturned cached records for medium set",
1201                          m_session_str);
1202                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1203                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1204                 new_apdu->u.searchResponse->resultCount =
1205                     &m_client->m_last_resultCount;
1206
1207                 new_apdu->u.searchResponse->numberOfRecordsReturned
1208                     = odr_intdup(odr_encode(), toget);
1209
1210                 new_apdu->u.searchResponse->presentStatus =
1211                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1212                 new_apdu->u.searchResponse->records = (Z_Records*)
1213                     odr_malloc(odr_encode(), sizeof(Z_Records));
1214                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1215                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1216                 new_apdu->u.searchResponse->nextResultSetPosition =
1217                     odr_intdup(odr_encode(), toget+1);
1218                 send_to_client(new_apdu);
1219                 return 0;
1220             }
1221             else
1222             {
1223                 // medium Set
1224                 // send present request (medium size)
1225                 yaz_log (LOG_LOG, "%sOptimizing search for medium set",
1226                          m_session_str);
1227
1228                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1229                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1230                 pr->referenceId = sr->referenceId;
1231                 pr->resultSetId = sr->resultSetName;
1232                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1233                 *pr->numberOfRecordsRequested = toget;
1234                 pr->recordComposition = comp;
1235                 m_client->m_sr_transform = 1;
1236                 return new_apdu;
1237             }
1238         }
1239         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1240             m_client->m_last_resultCount <= 0)
1241         {
1242             // large set. Return pseudo-search response immediately
1243             yaz_log (LOG_LOG, "%sOptimizing search for large set",
1244                      m_session_str);
1245             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1246             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1247             new_apdu->u.searchResponse->resultCount =
1248                 &m_client->m_last_resultCount;
1249             send_to_client(new_apdu);
1250             return 0;
1251         }
1252         else
1253         {
1254             Z_NamePlusRecordList *npr;
1255             int toget = m_client->m_last_resultCount;
1256             Z_RecordComposition *comp = 0;
1257             // small set
1258             // send a present request (small set)
1259
1260             if (sr->smallSetElementSetNames)
1261             {
1262                 comp = (Z_RecordComposition *)
1263                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1264                 comp->which = Z_RecordComp_simple;
1265                 comp->u.simple = sr->smallSetElementSetNames;
1266             }
1267
1268             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1269                                           sr->preferredRecordSyntax, comp))
1270             {
1271                 yaz_log (LOG_LOG, "%sReturned cached records for small set",
1272                          m_session_str);
1273                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1274                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1275                 new_apdu->u.searchResponse->resultCount =
1276                     &m_client->m_last_resultCount;
1277
1278                 new_apdu->u.searchResponse->numberOfRecordsReturned
1279                     = odr_intdup(odr_encode(), toget);
1280
1281                 new_apdu->u.searchResponse->presentStatus =
1282                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1283                 new_apdu->u.searchResponse->records = (Z_Records*)
1284                     odr_malloc(odr_encode(), sizeof(Z_Records));
1285                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1286                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1287                 new_apdu->u.searchResponse->nextResultSetPosition =
1288                     odr_intdup(odr_encode(), toget+1);
1289                 send_to_client(new_apdu);
1290                 return 0;
1291             }
1292             else
1293             {
1294                 yaz_log (LOG_LOG, "%sOptimizing search for small set",
1295                          m_session_str);
1296                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1297                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1298                 pr->referenceId = sr->referenceId;
1299                 pr->resultSetId = sr->resultSetName;
1300                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1301                 *pr->numberOfRecordsRequested = toget;
1302                 pr->recordComposition = comp;
1303                 m_client->m_sr_transform = 1;
1304                 return new_apdu;
1305             }
1306         }
1307     }
1308     else  // query doesn't match
1309     {
1310         delete m_client->m_last_query;
1311         m_client->m_last_query = this_query;
1312         m_client->m_last_ok = 0;
1313         m_client->m_cache.clear();
1314         m_client->m_resultSetStartPoint = 0;
1315
1316         xfree (m_client->m_last_resultSetId);
1317         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1318
1319         m_client->m_last_databases.set(sr->num_databaseNames,
1320                                        (const char **) sr->databaseNames);
1321     }
1322     return apdu;
1323 }
1324
1325
1326 void Yaz_Proxy::inc_request_no()
1327 {
1328     char *cp = strchr(m_session_str, ' ');
1329     m_request_no++;
1330     if (cp)
1331         sprintf(cp+1, "%d ", m_request_no);
1332 }
1333
1334 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1335 {
1336     inc_request_no();
1337
1338     m_bytes_recv += len;
1339
1340     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
1341         yaz_log (LOG_DEBUG, "%sReceiving %s from client %d bytes",
1342                  m_session_str, gdu_name(apdu), len);
1343
1344     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1345         shutdown();
1346
1347     m_bw_stat.add_bytes(len);
1348     m_pdu_stat.add_bytes(1);
1349
1350     gettimeofday(&m_time_tv, 0);
1351
1352     int bw_total = m_bw_stat.get_total();
1353     int pdu_total = m_pdu_stat.get_total();
1354
1355     int reduce = 0;
1356     if (m_bw_max)
1357     {
1358         if (bw_total > m_bw_max)
1359         {
1360             reduce = (bw_total/m_bw_max);
1361         }
1362     }
1363     if (m_pdu_max)
1364     {
1365         if (pdu_total > m_pdu_max)
1366         {
1367             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1368             reduce = (reduce > nreduce) ? reduce : nreduce;
1369         }
1370     }
1371     if (reduce)
1372     {
1373         yaz_log(LOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1374                 m_session_str, reduce, bw_total, pdu_total,
1375                 m_bw_max, m_pdu_max);
1376
1377         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1378         timeout(reduce);       // call us reduce seconds later
1379     }
1380     else if (apdu->which == Z_GDU_Z3950)
1381         handle_incoming_Z_PDU(apdu->u.z3950);
1382     else if (apdu->which == Z_GDU_HTTP_Request)
1383         handle_incoming_HTTP(apdu->u.HTTP_Request);
1384 }
1385
1386 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1387 {
1388     if (m_max_record_retrieve)
1389     {
1390         if (apdu->which == Z_APDU_presentRequest)
1391         {
1392             Z_PresentRequest *pr = apdu->u.presentRequest;
1393             if (pr->numberOfRecordsRequested &&
1394                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1395                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1396         }
1397     }
1398 }
1399
1400 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1401                                                      int error,
1402                                                      const char *addinfo)
1403 {
1404     Z_Records *rec = (Z_Records *)
1405         odr_malloc (odr, sizeof(*rec));
1406     int *err = (int *)
1407         odr_malloc (odr, sizeof(*err));
1408     Z_DiagRec *drec = (Z_DiagRec *)
1409         odr_malloc (odr, sizeof(*drec));
1410     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1411         odr_malloc (odr, sizeof(*dr));
1412     *err = error;
1413     rec->which = Z_Records_NSD;
1414     rec->u.nonSurrogateDiagnostic = dr;
1415     dr->diagnosticSetId =
1416         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1417     dr->condition = err;
1418     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1419     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1420     return rec;
1421 }
1422
1423 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1424 {
1425     if (apdu->which == Z_APDU_searchRequest &&
1426         apdu->u.searchRequest->query &&
1427         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1428         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1429     {
1430         Z_RPNQuery *rpnquery = 0;
1431         Z_SearchRequest *sr = apdu->u.searchRequest;
1432         char *addinfo = 0;
1433
1434         yaz_log(LOG_LOG, "%sCQL: %s", m_session_str,
1435                 sr->query->u.type_104->u.cql);
1436
1437         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1438                                           &rpnquery, odr_encode(),
1439                                           &addinfo);
1440         if (r == -3)
1441             yaz_log(LOG_LOG, "%sNo CQL to RPN table", m_session_str);
1442         else if (r)
1443         {
1444             yaz_log(LOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1445             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1446
1447             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1448             new_apdu->u.searchResponse->records =
1449                 create_nonSurrogateDiagnostics(odr_encode(),
1450                                                yaz_diag_srw_to_bib1(r),
1451                                                addinfo);
1452             *new_apdu->u.searchResponse->searchStatus = 0;
1453
1454             send_to_client(new_apdu);
1455
1456             return 0;
1457         }
1458         else
1459         {
1460             sr->query->which = Z_Query_type_1;
1461             sr->query->u.type_1 = rpnquery;
1462         }
1463         return apdu;
1464     }
1465     return apdu;
1466 }
1467
1468 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1469 {
1470     if (apdu->which == Z_APDU_searchRequest)
1471     {
1472         Z_SearchRequest *sr = apdu->u.searchRequest;
1473         int err = 0;
1474         char *addinfo = 0;
1475
1476         Yaz_ProxyConfig *cfg = check_reconfigure();
1477         if (cfg)
1478             err = cfg->check_query(odr_encode(), m_default_target,
1479                                    sr->query, &addinfo);
1480         if (err)
1481         {
1482             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1483
1484             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1485             new_apdu->u.searchResponse->records =
1486                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1487             *new_apdu->u.searchResponse->searchStatus = 0;
1488
1489             send_to_client(new_apdu);
1490
1491             return 0;
1492         }
1493     }
1494     return apdu;
1495 }
1496
1497 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1498 {
1499     m_marcxml_flag = 0;
1500     if (apdu->which == Z_APDU_searchRequest)
1501     {
1502         Z_SearchRequest *sr = apdu->u.searchRequest;
1503         int err = 0;
1504         char *addinfo = 0;
1505         Yaz_ProxyConfig *cfg = check_reconfigure();
1506
1507         Z_RecordComposition rc_temp, *rc = 0;
1508         if (sr->smallSetElementSetNames)
1509         {
1510             rc_temp.which = Z_RecordComp_simple;
1511             rc_temp.u.simple = sr->smallSetElementSetNames;
1512             rc = &rc_temp;
1513         }
1514
1515         char *stylesheet_name = 0;
1516         if (cfg)
1517             err = cfg->check_syntax(odr_encode(),
1518                                     m_default_target,
1519                                     sr->preferredRecordSyntax, rc,
1520                                     &addinfo, &stylesheet_name, &m_schema);
1521         if (stylesheet_name)
1522         {
1523             if (m_stylesheet_xsp)
1524                 xsltFreeStylesheet(m_stylesheet_xsp);
1525             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1526                                                        stylesheet_name);
1527             m_stylesheet_offset = 0;
1528             xfree(stylesheet_name);
1529         }
1530         if (err == -1)
1531         {
1532             sr->preferredRecordSyntax =
1533                 yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN, VAL_USMARC);
1534             m_marcxml_flag = 1;
1535         }
1536         else if (err)
1537         {
1538             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1539
1540             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1541             new_apdu->u.searchResponse->records =
1542                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1543             *new_apdu->u.searchResponse->searchStatus = 0;
1544
1545             send_to_client(new_apdu);
1546
1547             return 0;
1548         }
1549     }
1550     else if (apdu->which == Z_APDU_presentRequest)
1551     {
1552         Z_PresentRequest *pr = apdu->u.presentRequest;
1553         int err = 0;
1554         char *addinfo = 0;
1555         Yaz_ProxyConfig *cfg = check_reconfigure();
1556
1557         char *stylesheet_name = 0;
1558         if (cfg)
1559             err = cfg->check_syntax(odr_encode(), m_default_target,
1560                                     pr->preferredRecordSyntax,
1561                                     pr->recordComposition,
1562                                     &addinfo, &stylesheet_name, &m_schema);
1563         if (stylesheet_name)
1564         {
1565             if (m_stylesheet_xsp)
1566                 xsltFreeStylesheet(m_stylesheet_xsp);
1567             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1568                                                        stylesheet_name);
1569             m_stylesheet_offset = 0;
1570             xfree(stylesheet_name);
1571         }
1572         if (err == -1)
1573         {
1574             pr->preferredRecordSyntax =
1575                 yaz_oidval_to_z3950oid(odr_decode(), CLASS_RECSYN, VAL_USMARC);
1576             m_marcxml_flag = 1;
1577         }
1578         else if (err)
1579         {
1580             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1581
1582             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1583             new_apdu->u.presentResponse->records =
1584                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1585             *new_apdu->u.presentResponse->presentStatus =
1586                 Z_PresentStatus_failure;
1587
1588             send_to_client(new_apdu);
1589
1590             return 0;
1591         }
1592     }
1593     return apdu;
1594 }
1595
1596 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
1597 {
1598     if (!schema)
1599         return 0;
1600     Z_ElementSetNames *esn = (Z_ElementSetNames *)
1601         odr_malloc(o, sizeof(Z_ElementSetNames));
1602     esn->which = Z_ElementSetNames_generic;
1603     esn->u.generic = odr_strdup(o, schema);
1604     return esn;
1605 }
1606
1607 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1608 {
1609     if (m_s2z_odr_init)
1610     {
1611         odr_destroy(m_s2z_odr_init);
1612         m_s2z_odr_init = 0;
1613     }
1614     if (m_s2z_odr_search)
1615     {
1616         odr_destroy(m_s2z_odr_search);
1617         m_s2z_odr_search = 0;
1618     }
1619
1620     m_http_keepalive = 0;
1621     m_http_version = 0;
1622     if (!strcmp(hreq->version, "1.0"))
1623     {
1624         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1625         if (v && !strcmp(v, "Keep-Alive"))
1626             m_http_keepalive = 1;
1627         else
1628             m_http_keepalive = 0;
1629         m_http_version = "1.0";
1630     }
1631     else
1632     {
1633         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1634         if (v && !strcmp(v, "close"))
1635             m_http_keepalive = 0;
1636         else
1637             m_http_keepalive = 1;
1638         m_http_version = "1.1";
1639     }
1640
1641     Z_SRW_PDU *srw_pdu = 0;
1642     Z_SOAP *soap_package = 0;
1643     char *charset = 0;
1644     Z_SRW_diagnostic *diagnostic = 0;
1645     int num_diagnostic = 0;
1646     if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1647                        &charset) == 0
1648         || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1649                           &charset, &diagnostic, &num_diagnostic) == 0)
1650     {
1651         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
1652         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
1653         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
1654         m_s2z_init_apdu = 0;
1655         m_s2z_search_apdu = 0;
1656         m_s2z_present_apdu = 0;
1657
1658         m_s2z_stylesheet = 0;
1659
1660         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
1661         {
1662             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
1663
1664             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1665             // recordXPath unsupported.
1666             if (srw_req->recordXPath)
1667             {
1668                 yaz_add_srw_diagnostic(odr_decode(),
1669                                        &diagnostic, &num_diagnostic,
1670                                        72, 0);
1671             }
1672             // must have a query
1673             if (!srw_req->query.cql)
1674             {
1675                 yaz_add_srw_diagnostic(odr_decode(),
1676                                        &diagnostic, &num_diagnostic,
1677                                        7, "query");
1678             }
1679             // sort unsupported
1680             if (srw_req->sort_type != Z_SRW_sort_type_none)
1681             {
1682                 yaz_add_srw_diagnostic(odr_decode(),
1683                                        &diagnostic, &num_diagnostic,
1684                                        80, 0);
1685             }
1686             // save stylesheet
1687             if (srw_req->stylesheet)
1688                 m_s2z_stylesheet =
1689                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1690
1691             // set packing for response records ..
1692             if (srw_req->recordPacking &&
1693                 !strcmp(srw_req->recordPacking, "xml"))
1694                 m_s2z_packing = Z_SRW_recordPacking_XML;
1695             else
1696                 m_s2z_packing = Z_SRW_recordPacking_string;
1697
1698             if (num_diagnostic)
1699             {
1700                 Z_SRW_PDU *srw_pdu =
1701                     yaz_srw_get(odr_encode(),
1702                                 Z_SRW_searchRetrieve_response);
1703                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1704
1705                 srw_res->diagnostics = diagnostic;
1706                 srw_res->num_diagnostics = num_diagnostic;
1707                 send_srw_response(srw_pdu);
1708                 return;
1709             }
1710
1711             // prepare search PDU
1712             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
1713                                           Z_APDU_searchRequest);
1714             Z_SearchRequest *z_searchRequest =
1715                 m_s2z_search_apdu->u.searchRequest;
1716
1717             z_searchRequest->num_databaseNames = 1;
1718             z_searchRequest->databaseNames = (char**)
1719                 odr_malloc(m_s2z_odr_search, sizeof(char *));
1720             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
1721                                                            srw_req->database);
1722
1723             // query transformation
1724             Z_Query *query = (Z_Query *)
1725                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
1726             z_searchRequest->query = query;
1727
1728             if (srw_req->query_type == Z_SRW_query_type_cql)
1729             {
1730                 Z_External *ext = (Z_External *)
1731                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
1732                 ext->direct_reference =
1733                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
1734                 ext->indirect_reference = 0;
1735                 ext->descriptor = 0;
1736                 ext->which = Z_External_CQL;
1737                 ext->u.cql = srw_req->query.cql;
1738
1739                 query->which = Z_Query_type_104;
1740                 query->u.type_104 =  ext;
1741             }
1742             else if (srw_req->query_type == Z_SRW_query_type_pqf)
1743             {
1744                 Z_RPNQuery *RPNquery;
1745                 YAZ_PQF_Parser pqf_parser;
1746
1747                 pqf_parser = yaz_pqf_create ();
1748
1749                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
1750                                           srw_req->query.pqf);
1751                 if (!RPNquery)
1752                 {
1753                     const char *pqf_msg;
1754                     size_t off;
1755                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
1756                     yaz_log(LOG_LOG, "%*s^\n", off+4, "");
1757                     yaz_log(LOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
1758
1759                     send_to_srw_client_error(10, 0);
1760                     return;
1761                 }
1762                 query->which = Z_Query_type_1;
1763                 query->u.type_1 =  RPNquery;
1764
1765                 yaz_pqf_destroy (pqf_parser);
1766             }
1767             else
1768             {
1769                 send_to_srw_client_error(7, "query");
1770                 return;
1771             }
1772
1773             // present
1774             m_s2z_present_apdu = 0;
1775             int max = 0;
1776             if (srw_req->maximumRecords)
1777                 max = *srw_req->maximumRecords;
1778             int start = 1;
1779             if (srw_req->startRecord)
1780                 start = *srw_req->startRecord;
1781             if (max > 0)
1782             {
1783                 // Some backend, such as Voyager doesn't honor piggyback
1784                 // So we use present always (0 &&).
1785                 if (0 && start <= 1)  // Z39.50 piggyback
1786                 {
1787                     *z_searchRequest->smallSetUpperBound = max;
1788                     *z_searchRequest->mediumSetPresentNumber = max;
1789                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
1790
1791                     z_searchRequest->preferredRecordSyntax =
1792                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1793                                                VAL_TEXT_XML);
1794                     if (srw_req->recordSchema)
1795                     {
1796                         z_searchRequest->smallSetElementSetNames =
1797                             z_searchRequest->mediumSetElementSetNames =
1798                             mk_esn_from_schema(m_s2z_odr_search,
1799                                                srw_req->recordSchema);
1800                     }
1801                 }
1802                 else   // Z39.50 present
1803                 {
1804                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
1805                                                    Z_APDU_presentRequest);
1806                     Z_PresentRequest *z_presentRequest =
1807                         m_s2z_present_apdu->u.presentRequest;
1808                     *z_presentRequest->resultSetStartPoint = start;
1809                     *z_presentRequest->numberOfRecordsRequested = max;
1810                     z_presentRequest->preferredRecordSyntax =
1811                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1812                                                VAL_TEXT_XML);
1813                     if (srw_req->recordSchema)
1814                     {
1815                         z_presentRequest->recordComposition =
1816                             (Z_RecordComposition *)
1817                             odr_malloc(m_s2z_odr_search,
1818                                        sizeof(Z_RecordComposition));
1819                         z_presentRequest->recordComposition->which =
1820                             Z_RecordComp_simple;
1821                         z_presentRequest->recordComposition->u.simple =
1822                             mk_esn_from_schema(m_s2z_odr_search,
1823                                                srw_req->recordSchema);
1824                     }
1825                 }
1826             }
1827             if (!m_client)
1828             {
1829                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1830                                             Z_APDU_initRequest);
1831
1832                 // prevent m_initRequest_apdu memory from being grabbed
1833                 // in Yaz_Proxy::handle_incoming_Z_PDU
1834                 m_initRequest_apdu = m_s2z_init_apdu;
1835                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1836                 return;
1837             }
1838             else
1839             {
1840                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1841                 return;
1842             }
1843         }
1844         else if (srw_pdu->which == Z_SRW_explain_request)
1845         {
1846             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
1847
1848             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1849
1850             // save stylesheet
1851             if (srw_req->stylesheet)
1852                 m_s2z_stylesheet =
1853                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1854
1855             if (srw_req->recordPacking &&
1856                 !strcmp(srw_req->recordPacking, "xml"))
1857                 m_s2z_packing = Z_SRW_recordPacking_XML;
1858             else
1859                 m_s2z_packing = Z_SRW_recordPacking_string;
1860
1861             if (num_diagnostic)
1862             {
1863                 send_srw_explain_response(diagnostic, num_diagnostic);
1864                 return;
1865             }
1866
1867             if (!m_client)
1868             {
1869                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1870                                             Z_APDU_initRequest);
1871
1872                 // prevent m_initRequest_apdu memory from being grabbed
1873                 // in Yaz_Proxy::handle_incoming_Z_PDU
1874                 m_initRequest_apdu = m_s2z_init_apdu;
1875                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1876             }
1877             else
1878                 send_srw_explain_response(0, 0);
1879             return;
1880         }
1881         else if (srw_pdu->which == Z_SRW_scan_request)
1882         {
1883             m_s2z_database = odr_strdup(m_s2z_odr_init,
1884                                         srw_pdu->u.scan_request->database);
1885
1886             yaz_add_srw_diagnostic(odr_decode(),
1887                                    &diagnostic, &num_diagnostic,
1888                                    4, "scan");
1889             Z_SRW_PDU *srw_pdu =
1890                 yaz_srw_get(odr_encode(),
1891                             Z_SRW_scan_response);
1892             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1893
1894             srw_res->diagnostics = diagnostic;
1895             srw_res->num_diagnostics = num_diagnostic;
1896             send_srw_response(srw_pdu);
1897             return;
1898         }
1899         else
1900         {
1901             m_s2z_database = 0;
1902
1903             send_to_srw_client_error(4, 0);
1904         }
1905     }
1906     int len = 0;
1907     Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
1908     timeout(0);
1909     send_GDU(p, &len);
1910 }
1911
1912 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
1913 {
1914     if (!m_client && m_invalid_session)
1915     {
1916         m_apdu_invalid_session = apdu;
1917         m_mem_invalid_session = odr_extract_mem(odr_decode());
1918         apdu = m_initRequest_apdu;
1919     }
1920
1921     // Determine our client.
1922     Z_OtherInformation **oi;
1923     get_otherInfoAPDU(apdu, &oi);
1924     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
1925     if (!m_client)
1926     {
1927         delete this;
1928         return;
1929     }
1930     m_client->m_server = this;
1931
1932     if (apdu->which == Z_APDU_initRequest)
1933     {
1934         if (apdu->u.initRequest->implementationId)
1935             yaz_log(LOG_LOG, "%simplementationId: %s",
1936                     m_session_str, apdu->u.initRequest->implementationId);
1937         if (apdu->u.initRequest->implementationName)
1938             yaz_log(LOG_LOG, "%simplementationName: %s",
1939                     m_session_str, apdu->u.initRequest->implementationName);
1940         if (apdu->u.initRequest->implementationVersion)
1941             yaz_log(LOG_LOG, "%simplementationVersion: %s",
1942                     m_session_str, apdu->u.initRequest->implementationVersion);
1943         if (m_initRequest_apdu == 0)
1944         {
1945             if (m_initRequest_mem)
1946                 nmem_destroy(m_initRequest_mem);
1947             m_initRequest_apdu = apdu;
1948             m_initRequest_mem = odr_extract_mem(odr_decode());
1949
1950             // save init options for the response..
1951             m_initRequest_options = apdu->u.initRequest->options;
1952
1953             apdu->u.initRequest->options =
1954                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
1955                                            sizeof(Odr_bitmask));
1956             ODR_MASK_ZERO(apdu->u.initRequest->options);
1957             int i;
1958             for (i = 0; i<= 24; i++)
1959                 ODR_MASK_SET(apdu->u.initRequest->options, i);
1960             ODR_MASK_CLEAR(apdu->u.initRequest->options,
1961                            Z_Options_negotiationModel);
1962
1963             // make new version
1964             m_initRequest_version = apdu->u.initRequest->protocolVersion;
1965             apdu->u.initRequest->protocolVersion =
1966                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
1967                                            sizeof(Odr_bitmask));
1968             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
1969
1970             for (i = 0; i<= 8; i++)
1971                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
1972         }
1973         if (m_client->m_init_flag)
1974         {
1975             if (handle_init_response_for_invalid_session(apdu))
1976                 return;
1977             Z_APDU *apdu2 = m_client->m_initResponse;
1978             apdu2->u.initResponse->otherInfo = 0;
1979             if (m_client->m_cookie && *m_client->m_cookie)
1980                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
1981                                            m_client->m_cookie);
1982             apdu2->u.initResponse->referenceId =
1983                 apdu->u.initRequest->referenceId;
1984             apdu2->u.initResponse->options = m_client->m_initResponse_options;
1985             apdu2->u.initResponse->protocolVersion =
1986                 m_client->m_initResponse_version;
1987
1988             send_to_client(apdu2);
1989             return;
1990         }
1991         m_client->m_init_flag = 1;
1992     }
1993     handle_max_record_retrieve(apdu);
1994
1995     if (apdu)
1996         apdu = handle_syntax_validation(apdu);
1997
1998     if (apdu)
1999         apdu = handle_query_transformation(apdu);
2000
2001     if (apdu)
2002         apdu = handle_query_validation(apdu);
2003
2004     if (apdu)
2005         apdu = result_set_optimize(apdu);
2006     if (!apdu)
2007     {
2008         m_client->timeout(m_target_idletime);  // mark it active even
2009         // though we didn't use it
2010         return;
2011     }
2012
2013     // delete other info part from PDU before sending to target
2014     get_otherInfoAPDU(apdu, &oi);
2015     if (oi)
2016         *oi = 0;
2017
2018     if (apdu->which == Z_APDU_presentRequest &&
2019         m_client->m_resultSetStartPoint == 0)
2020     {
2021         Z_PresentRequest *pr = apdu->u.presentRequest;
2022         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2023         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2024     } else {
2025         m_client->m_resultSetStartPoint = 0;
2026     }
2027     if (m_client->send_to_target(apdu) < 0)
2028     {
2029         delete m_client;
2030         m_client = 0;
2031         delete this;
2032     }
2033     else
2034         m_client->m_waiting = 1;
2035 }
2036
2037 void Yaz_Proxy::connectNotify()
2038 {
2039 }
2040
2041 void Yaz_Proxy::shutdown()
2042 {
2043     m_invalid_session = 0;
2044     // only keep if keep_alive flag is set...
2045     if (m_client &&
2046         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2047         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2048         m_client->m_waiting == 0)
2049     {
2050         yaz_log(LOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2051                  m_session_str,
2052                  m_client->get_hostname());
2053         yaz_log(LOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2054                 m_session_str, m_client->m_pdu_recv,
2055                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2056                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2057         assert (m_client->m_waiting != 2);
2058         // Tell client (if any) that no server connection is there..
2059         m_client->m_server = 0;
2060         m_invalid_session = 0;
2061     }
2062     else if (m_client)
2063     {
2064         yaz_log (LOG_LOG, "%sShutdown (client to proxy) close %s",
2065                  m_session_str,
2066                  m_client->get_hostname());
2067         assert (m_client->m_waiting != 2);
2068         delete m_client;
2069     }
2070     else if (!m_parent)
2071     {
2072         yaz_log (LOG_LOG, "%sshutdown (client to proxy) bad state",
2073                  m_session_str);
2074         assert (m_parent);
2075     }
2076     else
2077     {
2078         yaz_log (LOG_LOG, "%sShutdown (client to proxy)",
2079                  m_session_str);
2080     }
2081     if (m_parent)
2082         m_parent->pre_init();
2083     delete this;
2084 }
2085
2086 const char *Yaz_ProxyClient::get_session_str()
2087 {
2088     if (!m_server)
2089         return "0 ";
2090     return m_server->get_session_str();
2091 }
2092
2093 void Yaz_ProxyClient::shutdown()
2094 {
2095     yaz_log (LOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2096              get_hostname());
2097     delete m_server;
2098     delete this;
2099 }
2100
2101 void Yaz_Proxy::failNotify()
2102 {
2103     inc_request_no();
2104     yaz_log (LOG_LOG, "%sConnection closed by client",
2105              get_session_str());
2106     shutdown();
2107 }
2108
2109 void Yaz_ProxyClient::failNotify()
2110 {
2111     if (m_server)
2112         m_server->inc_request_no();
2113     yaz_log (LOG_LOG, "%sConnection closed by target %s",
2114              get_session_str(), get_hostname());
2115     shutdown();
2116 }
2117
2118 void Yaz_ProxyClient::connectNotify()
2119 {
2120     const char *s = get_session_str();
2121     const char *h = get_hostname();
2122     yaz_log (LOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2123              m_target_idletime);
2124     timeout(m_target_idletime);
2125     if (!m_server)
2126         pre_init_client();
2127 }
2128
2129 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2130                                                   *the_PDU_Observable, int fd)
2131 {
2132     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2133 }
2134
2135 Yaz_ProxyClient::~Yaz_ProxyClient()
2136 {
2137     if (m_prev)
2138         *m_prev = m_next;
2139     if (m_next)
2140         m_next->m_prev = m_prev;
2141     m_waiting = 2;     // for debugging purposes only.
2142     odr_destroy(m_init_odr);
2143     delete m_last_query;
2144     xfree (m_last_resultSetId);
2145     xfree (m_cookie);
2146 }
2147
2148 void Yaz_ProxyClient::pre_init_client()
2149 {
2150     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2151     Z_InitRequest *req = apdu->u.initRequest;
2152
2153     int i;
2154     for (i = 0; i<= 24; i++)
2155         ODR_MASK_SET(req->options, i);
2156     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2157                    Z_Options_negotiationModel);
2158     for (i = 0; i<= 10; i++)
2159         ODR_MASK_SET(req->protocolVersion, i);
2160
2161     if (send_to_target(apdu) < 0)
2162     {
2163         delete this;
2164     }
2165     else
2166     {
2167         m_waiting = 1;
2168         m_init_flag = 1;
2169     }
2170 }
2171
2172 void Yaz_Proxy::pre_init()
2173 {
2174     int i;
2175     const char *name = 0;
2176     const char *zurl_in_use[MAX_ZURL_PLEX];
2177     int limit_bw, limit_pdu, limit_req;
2178     int target_idletime, client_idletime;
2179     int max_clients;
2180     int keepalive_limit_bw, keepalive_limit_pdu;
2181     int pre_init;
2182     const char *cql2rpn = 0;
2183
2184     Yaz_ProxyConfig *cfg = check_reconfigure();
2185
2186     zurl_in_use[0] = 0;
2187
2188     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2189         set_APDU_yazlog(1);
2190     else
2191         set_APDU_yazlog(0);
2192
2193     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2194                                           &limit_bw, &limit_pdu, &limit_req,
2195                                           &target_idletime, &client_idletime,
2196                                           &max_clients,
2197                                           &keepalive_limit_bw,
2198                                           &keepalive_limit_pdu,
2199                                           &pre_init,
2200                                           &cql2rpn) ; i++)
2201     {
2202         if (pre_init)
2203         {
2204             int j;
2205             for (j = 0; zurl_in_use[j]; j++)
2206             {
2207                 Yaz_ProxyClient *c;
2208                 int spare = 0;
2209                 int in_use = 0;
2210                 int other = 0;
2211                 for (c = m_clientPool; c; c = c->m_next)
2212                 {
2213                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
2214                     {
2215                         if (c->m_cookie == 0)
2216                         {
2217                             if (c->m_server == 0)
2218                                 spare++;
2219                             else
2220                                 in_use++;
2221                         }
2222                         else
2223                             other++;
2224                     }
2225                 }
2226                 yaz_log(LOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2227                         "preinit=%d",m_session_str,
2228                         name, zurl_in_use[j], in_use, other, spare, pre_init);
2229                 if (spare < pre_init)
2230                 {
2231                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2232                     c->m_next = m_clientPool;
2233                     if (c->m_next)
2234                         c->m_next->m_prev = &c->m_next;
2235                     m_clientPool = c;
2236                     c->m_prev = &m_clientPool;
2237
2238                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
2239                         c->set_APDU_yazlog(1);
2240                     else
2241                         c->set_APDU_yazlog(0);
2242
2243                     if (c->client(zurl_in_use[j]))
2244                     {
2245                         timeout(60);
2246                         delete c;
2247                         return;
2248                     }
2249                     c->timeout(30);
2250                     c->m_waiting = 1;
2251                     c->m_target_idletime = target_idletime;
2252                     c->m_seqno = m_seqno++;
2253                 }
2254             }
2255         }
2256     }
2257 }
2258
2259 void Yaz_Proxy::timeoutNotify()
2260 {
2261     if (m_parent)
2262     {
2263         if (m_bw_hold_PDU)
2264         {
2265             timeout(m_client_idletime);
2266             Z_GDU *apdu = m_bw_hold_PDU;
2267             m_bw_hold_PDU = 0;
2268
2269             if (apdu->which == Z_GDU_Z3950)
2270                 handle_incoming_Z_PDU(apdu->u.z3950);
2271             else if (apdu->which == Z_GDU_HTTP_Request)
2272                 handle_incoming_HTTP(apdu->u.HTTP_Request);
2273         }
2274         else if (m_stylesheet_nprl)
2275             convert_xsl_delay();
2276         else
2277         {
2278             inc_request_no();
2279
2280             yaz_log (LOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2281             shutdown();
2282         }
2283     }
2284     else
2285     {
2286         timeout(600);
2287         pre_init();
2288     }
2289 }
2290
2291 void Yaz_Proxy::markInvalid()
2292 {
2293     m_client = 0;
2294     m_invalid_session = 1;
2295 }
2296
2297 void Yaz_ProxyClient::timeoutNotify()
2298 {
2299     if (m_server)
2300         m_server->inc_request_no();
2301
2302     yaz_log (LOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2303              get_hostname());
2304     m_waiting = 1;
2305     m_root->pre_init();
2306     if (m_server && m_init_flag)
2307     {
2308         // target timed out in a session that was properly initialized
2309         // server object stay alive but we mark it as invalid so it
2310         // gets initialized again
2311         m_server->markInvalid();
2312         m_server = 0;
2313     }
2314     shutdown();
2315 }
2316
2317 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2318                                  Yaz_Proxy *parent) :
2319     Yaz_Z_Assoc (the_PDU_Observable)
2320 {
2321     m_cookie = 0;
2322     m_next = 0;
2323     m_prev = 0;
2324     m_init_flag = 0;
2325     m_last_query = 0;
2326     m_last_resultSetId = 0;
2327     m_last_resultCount = 0;
2328     m_last_ok = 0;
2329     m_sr_transform = 0;
2330     m_waiting = 0;
2331     m_init_odr = odr_createmem (ODR_DECODE);
2332     m_initResponse = 0;
2333     m_initResponse_options = 0;
2334     m_initResponse_version = 0;
2335     m_resultSetStartPoint = 0;
2336     m_bytes_sent = m_bytes_recv = 0;
2337     m_pdu_recv = 0;
2338     m_server = 0;
2339     m_seqno = 0;
2340     m_target_idletime = 600;
2341     m_root = parent;
2342 }
2343
2344 const char *Yaz_Proxy::option(const char *name, const char *value)
2345 {
2346     if (!strcmp (name, "optimize")) {
2347         if (value) {
2348             xfree (m_optimize);
2349             m_optimize = xstrdup (value);
2350         }
2351         return m_optimize;
2352     }
2353     return 0;
2354 }
2355
2356 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2357 {
2358
2359 }
2360
2361 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2362 {
2363     if (apdu->which == Z_GDU_Z3950)
2364         recv_Z_PDU(apdu->u.z3950, len);
2365     else if (apdu->which == Z_GDU_HTTP_Response)
2366         recv_HTTP_response(apdu->u.HTTP_Response, len);
2367     else
2368         shutdown();
2369 }
2370
2371 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2372 {
2373     if (!m_invalid_session)
2374         return 0;
2375     m_invalid_session = 0;
2376     handle_incoming_Z_PDU(m_apdu_invalid_session);
2377     assert (m_mem_invalid_session);
2378     nmem_destroy(m_mem_invalid_session);
2379     m_mem_invalid_session = 0;
2380     return 1;
2381 }
2382
2383 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2384 {
2385     m_bytes_recv += len;
2386
2387     m_pdu_recv++;
2388     m_waiting = 0;
2389     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2390         yaz_log (LOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2391                  apdu_name(apdu), get_hostname(), len);
2392     if (apdu->which == Z_APDU_initResponse)
2393     {
2394         if (!m_server)  // if this is a pre init session , check for more
2395             m_root->pre_init();
2396         NMEM nmem = odr_extract_mem (odr_decode());
2397         odr_reset (m_init_odr);
2398         nmem_transfer (m_init_odr->mem, nmem);
2399         m_initResponse = apdu;
2400         m_initResponse_options = apdu->u.initResponse->options;
2401         m_initResponse_version = apdu->u.initResponse->protocolVersion;
2402
2403         Z_InitResponse *ir = apdu->u.initResponse;
2404         char *im0 = ir->implementationName;
2405
2406         char *im1 = (char*)
2407             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2408         *im1 = '\0';
2409         if (im0)
2410         {
2411             strcat(im1, im0);
2412             strcat(im1, " ");
2413         }
2414         strcat(im1, "(YAZ Proxy)");
2415         ir->implementationName = im1;
2416
2417         nmem_destroy (nmem);
2418
2419         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2420             return;
2421     }
2422     if (apdu->which == Z_APDU_searchResponse)
2423     {
2424         Z_SearchResponse *sr = apdu->u.searchResponse;
2425         m_last_resultCount = *sr->resultCount;
2426         int status = *sr->searchStatus;
2427         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2428         {
2429             m_last_ok = 1;
2430
2431             if (sr->records && sr->records->which == Z_Records_DBOSD)
2432             {
2433                 m_cache.add(odr_decode(),
2434                             sr->records->u.databaseOrSurDiagnostics, 1,
2435                             *sr->resultCount);
2436             }
2437         }
2438     }
2439     if (apdu->which == Z_APDU_presentResponse)
2440     {
2441         Z_PresentResponse *pr = apdu->u.presentResponse;
2442         if (m_sr_transform)
2443         {
2444             m_sr_transform = 0;
2445             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2446             Z_SearchResponse *sr = new_apdu->u.searchResponse;
2447             sr->referenceId = pr->referenceId;
2448             *sr->resultCount = m_last_resultCount;
2449             sr->records = pr->records;
2450             sr->nextResultSetPosition = pr->nextResultSetPosition;
2451             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2452             apdu = new_apdu;
2453         }
2454         if (pr->records &&
2455             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2456         {
2457             m_cache.add(odr_decode(),
2458                         pr->records->u.databaseOrSurDiagnostics,
2459                         m_resultSetStartPoint, -1);
2460             m_resultSetStartPoint = 0;
2461         }
2462     }
2463     if (m_cookie)
2464         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2465     if (m_server)
2466     {
2467         m_server->send_to_client(apdu);
2468     }
2469     if (apdu->which == Z_APDU_close)
2470     {
2471         shutdown();
2472     }
2473 }
2474
2475 int Yaz_Proxy::server(const char *addr)
2476 {
2477     int r = Yaz_Z_Assoc::server(addr);
2478     if (!r)
2479     {
2480         yaz_log(LOG_LOG, "%sStarted proxy " VERSION " on %s", m_session_str, addr);
2481         timeout(1);
2482     }
2483     return r;
2484 }
2485