src/yaz-proxy.cpp

   1 /*
   2  * Copyright (c) 1998-2004, Index Data.
   3  * See the file LICENSE for details.
   4  *
   5  * $Id: yaz-proxy.cpp,v 1.92 2004-01-29 20:53:34 adam Exp $
   6  */
   7
   8 #include <assert.h>
   9 #include <time.h>
  10
  11 #include <yaz/srw.h>
  12 #include <yaz/marcdisp.h>
  13 #include <yaz/yaz-iconv.h>
  14 #include <yaz/log.h>
  15 #include <yaz/diagbib1.h>
  16 #include <yaz++/proxy.h>
  17 #include <yaz/pquery.h>
  18
  19 #if HAVE_XSLT
  20 #include <libxslt/xsltutils.h>
  21 #include <libxslt/transform.h>
  22 #endif
  23
  24 static const char *apdu_name(Z_APDU *apdu)
  25 {
  26     switch (apdu->which)
  27     {
  28     case Z_APDU_initRequest:
  29         return "initRequest";
  30     case Z_APDU_initResponse:
  31         return "initResponse";
  32     case Z_APDU_searchRequest:
  33         return "searchRequest";
  34     case Z_APDU_searchResponse:
  35         return "searchResponse";
  36     case Z_APDU_presentRequest:
  37         return "presentRequest";
  38     case Z_APDU_presentResponse:
  39         return "presentResponse";
  40     case Z_APDU_deleteResultSetRequest:
  41         return "deleteResultSetRequest";
  42     case Z_APDU_deleteResultSetResponse:
  43         return "deleteResultSetResponse";
  44     case Z_APDU_scanRequest:
  45         return "scanRequest";
  46     case Z_APDU_scanResponse:
  47         return "scanResponse";
  48     case Z_APDU_sortRequest:
  49         return "sortRequest";
  50     case Z_APDU_sortResponse:
  51         return "sortResponse";
  52     case Z_APDU_extendedServicesRequest:
  53         return "extendedServicesRequest";
  54     case Z_APDU_extendedServicesResponse:
  55         return "extendedServicesResponse";
  56     case Z_APDU_close:
  57         return "close";
  58     }
  59     return "other";
  60 }
  61
  62 static const char *gdu_name(Z_GDU *gdu)
  63 {
  64     switch(gdu->which)
  65     {
  66     case Z_GDU_Z3950:
  67         return apdu_name(gdu->u.z3950);
  68     case Z_GDU_HTTP_Request:
  69         return "HTTP Request";
  70     case Z_GDU_HTTP_Response:
  71         return "HTTP Response";
  72     }
  73     return "Unknown request/response";
  74 }
  75
  76 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
  77                      Yaz_Proxy *parent) :
  78     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
  79 {
  80     m_PDU_Observable = the_PDU_Observable;
  81     m_client = 0;
  82     m_parent = parent;
  83     m_clientPool = 0;
  84     m_seqno = 1;
  85     m_keepalive_limit_bw = 500000;
  86     m_keepalive_limit_pdu = 1000;
  87     m_proxyTarget = 0;
  88     m_default_target = 0;
  89     m_proxy_authentication = 0;
  90     m_max_clients = 150;
  91     m_log_mask = 0;
  92     m_seed = time(0);
  93     m_client_idletime = 600;
  94     m_target_idletime = 600;
  95     m_optimize = xstrdup ("1");
  96     strcpy(m_session_str, "0 ");
  97     m_session_no=0;
  98     m_bytes_sent = m_bytes_recv = 0;
  99     m_bw_hold_PDU = 0;
 100     m_bw_max = 0;
 101     m_pdu_max = 0;
 102     m_max_record_retrieve = 0;
 103     m_reconfig_flag = 0;
 104     m_config_fname = 0;
 105     m_request_no = 0;
 106     m_invalid_session = 0;
 107     m_config = 0;
 108     m_marcxml_flag = 0;
 109     m_stylesheet_schema = 0;
 110     m_s2z_stylesheet = 0;
 111     m_s2z_database = 0;
 112     m_schema = 0;
 113     m_initRequest_apdu = 0;
 114     m_initRequest_mem = 0;
 115     m_initRequest_options = 0;
 116     m_initRequest_version = 0;
 117     m_apdu_invalid_session = 0;
 118     m_mem_invalid_session = 0;
 119     m_s2z_odr_init = 0;
 120     m_s2z_odr_search = 0;
 121     m_s2z_init_apdu = 0;
 122     m_s2z_search_apdu = 0;
 123     m_s2z_present_apdu = 0;
 124     m_http_keepalive = 0;
 125     m_http_version = 0;
 126     m_soap_ns = 0;
 127     m_s2z_packing = Z_SRW_recordPacking_string;
 128     m_time_tv.tv_sec = 0;
 129     m_time_tv.tv_usec = 0;
 130 }
 131
 132 Yaz_Proxy::~Yaz_Proxy()
 133 {
 134     yaz_log(LOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 135             m_bytes_sent, m_bytes_recv);
 136     nmem_destroy(m_initRequest_mem);
 137     nmem_destroy(m_mem_invalid_session);
 138     xfree (m_proxyTarget);
 139     xfree (m_default_target);
 140     xfree (m_proxy_authentication);
 141     xfree (m_optimize);
 142     xfree (m_stylesheet_schema);
 143     xfree (m_schema);
 144     if (m_s2z_odr_init)
 145         odr_destroy(m_s2z_odr_init);
 146     if (m_s2z_odr_search)
 147         odr_destroy(m_s2z_odr_search);
 148     delete m_config;
 149 }
 150
 151 int Yaz_Proxy::set_config(const char *config)
 152 {
 153     delete m_config;
 154     m_config = new Yaz_ProxyConfig();
 155     xfree(m_config_fname);
 156     m_config_fname = xstrdup(config);
 157     int r = m_config->read_xml(config);
 158     if (!r)
 159         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 160     return r;
 161 }
 162
 163 void Yaz_Proxy::set_default_target(const char *target)
 164 {
 165     xfree (m_default_target);
 166     m_default_target = 0;
 167     if (target)
 168         m_default_target = (char *) xstrdup (target);
 169 }
 170
 171 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 172 {
 173     xfree (m_proxy_authentication);
 174     m_proxy_authentication = 0;
 175     if (auth)
 176         m_proxy_authentication = (char *) xstrdup (auth);
 177 }
 178
 179 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 180 {
 181     if (m_parent)
 182         return m_parent->check_reconfigure();
 183
 184     Yaz_ProxyConfig *cfg = m_config;
 185     if (m_reconfig_flag)
 186     {
 187         yaz_log(LOG_LOG, "reconfigure");
 188         yaz_log_reopen();
 189         if (m_config_fname && cfg)
 190         {
 191             yaz_log(LOG_LOG, "reconfigure config %s", m_config_fname);
 192             int r = cfg->read_xml(m_config_fname);
 193             if (r)
 194                 yaz_log(LOG_WARN, "reconfigure failed");
 195             else
 196             {
 197                 m_log_mask = 0;
 198                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 199             }
 200         }
 201         else
 202             yaz_log(LOG_LOG, "reconfigure");
 203         m_reconfig_flag = 0;
 204     }
 205     return cfg;
 206 }
 207
 208 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 209                                             *the_PDU_Observable, int fd)
 210 {
 211     check_reconfigure();
 212     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 213     new_proxy->m_config = 0;
 214     new_proxy->m_config_fname = 0;
 215     new_proxy->timeout(m_client_idletime);
 216     new_proxy->m_target_idletime = m_target_idletime;
 217     new_proxy->set_default_target(m_default_target);
 218     new_proxy->m_max_clients = m_max_clients;
 219     new_proxy->m_log_mask = m_log_mask;
 220     new_proxy->set_APDU_log(get_APDU_log());
 221     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 222         new_proxy->set_APDU_yazlog(1);
 223     else
 224         new_proxy->set_APDU_yazlog(0);
 225     new_proxy->set_proxy_authentication(m_proxy_authentication);
 226     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 227     m_session_no++;
 228     yaz_log (LOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 229              the_PDU_Observable->getpeername());
 230     return new_proxy;
 231 }
 232
 233 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 234 {
 235     int oid[OID_SIZE];
 236     Z_OtherInformationUnit *oi;
 237     struct oident ent;
 238     ent.proto = PROTO_Z3950;
 239     ent.oclass = CLASS_USERINFO;
 240     ent.value = (oid_value) VAL_COOKIE;
 241     assert (oid_ent_to_oid (&ent, oid));
 242
 243     if (oid_ent_to_oid (&ent, oid) &&
 244         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 245         oi->which == Z_OtherInfo_characterInfo)
 246         return oi->information.characterInfo;
 247     return 0;
 248 }
 249
 250 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 251 {
 252     int oid[OID_SIZE];
 253     Z_OtherInformationUnit *oi;
 254     struct oident ent;
 255     ent.proto = PROTO_Z3950;
 256     ent.oclass = CLASS_USERINFO;
 257     ent.value = (oid_value) VAL_PROXY;
 258     if (oid_ent_to_oid (&ent, oid) &&
 259         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 260         oi->which == Z_OtherInfo_characterInfo)
 261         return oi->information.characterInfo;
 262     return 0;
 263 }
 264
 265 const char *Yaz_Proxy::load_balance(const char **url)
 266 {
 267     int zurl_in_use[MAX_ZURL_PLEX];
 268     Yaz_ProxyClient *c;
 269     int i;
 270
 271     for (i = 0; i<MAX_ZURL_PLEX; i++)
 272         zurl_in_use[i] = 0;
 273     for (c = m_parent->m_clientPool; c; c = c->m_next)
 274     {
 275         for (i = 0; url[i]; i++)
 276             if (!strcmp(url[i], c->get_hostname()))
 277                 zurl_in_use[i]++;
 278     }
 279     int min = 100000;
 280     const char *ret = 0;
 281     for (i = 0; url[i]; i++)
 282     {
 283         yaz_log(LOG_DEBUG, "%szurl=%s use=%d",
 284                 m_session_str, url[i], zurl_in_use[i]);
 285         if (min > zurl_in_use[i])
 286         {
 287             ret = url[i];
 288             min = zurl_in_use[i];
 289         }
 290     }
 291     return ret;
 292 }
 293
 294 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 295                                        const char *proxy_host)
 296 {
 297     assert (m_parent);
 298     Yaz_Proxy *parent = m_parent;
 299     Yaz_ProxyClient *c = m_client;
 300
 301     if (!m_proxyTarget)
 302     {
 303         const char *url[MAX_ZURL_PLEX];
 304         Yaz_ProxyConfig *cfg = check_reconfigure();
 305         if (proxy_host)
 306         {
 307 #if 1
 308 /* only to be enabled for debugging... */
 309             if (!strcmp(proxy_host, "stop"))
 310                 exit(0);
 311 #endif
 312             xfree(m_default_target);
 313             m_default_target = xstrdup(proxy_host);
 314             proxy_host = m_default_target;
 315         }
 316         int client_idletime = -1;
 317         const char *cql2rpn_fname = 0;
 318         url[0] = m_default_target;
 319         url[1] = 0;
 320         if (cfg)
 321         {
 322             int pre_init = 0;
 323             cfg->get_target_info(proxy_host, url, &m_bw_max,
 324                                  &m_pdu_max, &m_max_record_retrieve,
 325                                  &m_target_idletime, &client_idletime,
 326                                  &parent->m_max_clients,
 327                                  &m_keepalive_limit_bw,
 328                                  &m_keepalive_limit_pdu,
 329                                  &pre_init,
 330                                  &cql2rpn_fname);
 331         }
 332         if (client_idletime != -1)
 333         {
 334             m_client_idletime = client_idletime;
 335             timeout(m_client_idletime);
 336         }
 337         if (cql2rpn_fname)
 338             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 339         if (!url[0])
 340         {
 341             yaz_log(LOG_LOG, "%sNo default target", m_session_str);
 342             return 0;
 343         }
 344         // we don't handle multiplexing for cookie session, so we just
 345         // pick the first one in this case (anonymous users will be able
 346         // to use any backend)
 347         if (cookie && *cookie)
 348             m_proxyTarget = (char*) xstrdup(url[0]);
 349         else
 350             m_proxyTarget = (char*) xstrdup(load_balance(url));
 351     }
 352     if (cookie && *cookie)
 353     {   // search in sessions with a cookie
 354         for (c = parent->m_clientPool; c; c = c->m_next)
 355         {
 356             assert (c->m_prev);
 357             assert (*c->m_prev == c);
 358             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 359                 !strcmp(m_proxyTarget, c->get_hostname()))
 360             {
 361                 // Found it in cache
 362                 // The following handles "cancel"
 363                 // If connection is busy (waiting for PDU) and
 364                 // we have an initRequest we can safely do re-open
 365                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 366                 {
 367                     yaz_log (LOG_LOG, "%s REOPEN target=%s", m_session_str,
 368                              c->get_hostname());
 369                     c->close();
 370                     c->m_init_flag = 0;
 371
 372                     c->m_last_ok = 0;
 373                     c->m_cache.clear();
 374                     c->m_last_resultCount = 0;
 375                     c->m_sr_transform = 0;
 376                     c->m_waiting = 0;
 377                     c->m_resultSetStartPoint = 0;
 378                     c->m_target_idletime = m_target_idletime;
 379                     if (c->client(m_proxyTarget))
 380                     {
 381                         delete c;
 382                         return 0;
 383                     }
 384                     c->timeout(30);
 385                 }
 386                 c->m_seqno = parent->m_seqno;
 387                 if (c->m_server && c->m_server != this)
 388                     c->m_server->m_client = 0;
 389                 c->m_server = this;
 390                 (parent->m_seqno)++;
 391                 yaz_log (LOG_DEBUG, "get_client 1 %p %p", this, c);
 392                 return c;
 393             }
 394         }
 395     }
 396     else if (!c)
 397     {
 398         // don't have a client session yet. Search in session w/o cookie
 399         for (c = parent->m_clientPool; c; c = c->m_next)
 400         {
 401             assert (c->m_prev);
 402             assert (*c->m_prev == c);
 403             if (c->m_server == 0 && c->m_cookie == 0 &&
 404                 c->m_waiting == 0 &&
 405                 !strcmp(m_proxyTarget, c->get_hostname()))
 406             {
 407                 // found it in cache
 408                 yaz_log (LOG_LOG, "%sREUSE %s",
 409                          m_session_str, c->get_hostname());
 410
 411                 c->m_seqno = parent->m_seqno;
 412                 assert(c->m_server == 0);
 413                 c->m_server = this;
 414
 415                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 416                     c->set_APDU_yazlog(1);
 417                 else
 418                     c->set_APDU_yazlog(0);
 419
 420                 (parent->m_seqno)++;
 421
 422                 parent->pre_init();
 423
 424                 return c;
 425             }
 426         }
 427     }
 428     if (!m_client)
 429     {
 430         if (apdu->which != Z_APDU_initRequest)
 431         {
 432             yaz_log (LOG_LOG, "%sno init request as first PDU", m_session_str);
 433             return 0;
 434         }
 435         Z_InitRequest *initRequest = apdu->u.initRequest;
 436
 437         if (!initRequest->idAuthentication)
 438         {
 439             if (m_proxy_authentication)
 440             {
 441                 initRequest->idAuthentication =
 442                     (Z_IdAuthentication *)
 443                     odr_malloc (odr_encode(),
 444                                 sizeof(*initRequest->idAuthentication));
 445                 initRequest->idAuthentication->which =
 446                     Z_IdAuthentication_open;
 447                 initRequest->idAuthentication->u.open =
 448                     odr_strdup (odr_encode(), m_proxy_authentication);
 449             }
 450         }
 451         // go through list of clients - and find the lowest/oldest one.
 452         Yaz_ProxyClient *c_min = 0;
 453         int min_seq = -1;
 454         int no_of_clients = 0;
 455         if (parent->m_clientPool)
 456             yaz_log (LOG_DEBUG, "Existing sessions");
 457         for (c = parent->m_clientPool; c; c = c->m_next)
 458         {
 459             yaz_log (LOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 460                                c->m_waiting, c->get_hostname(),
 461                                c->m_cookie ? c->m_cookie : "");
 462             no_of_clients++;
 463             if (min_seq < 0 || c->m_seqno < min_seq)
 464             {
 465                 min_seq = c->m_seqno;
 466                 c_min = c;
 467             }
 468         }
 469         if (no_of_clients >= parent->m_max_clients)
 470         {
 471             c = c_min;
 472             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 473             {
 474                 yaz_log (LOG_LOG, "%sMAXCLIENTS Destroy %d",
 475                          m_session_str, c->m_seqno);
 476                 if (c->m_server && c->m_server != this)
 477                     delete c->m_server;
 478                 c->m_server = 0;
 479             }
 480             else
 481             {
 482                 yaz_log (LOG_LOG, "%sMAXCLIENTS Reuse %d %d %s",
 483                          m_session_str,
 484                          c->m_seqno, parent->m_seqno, c->get_hostname());
 485                 xfree (c->m_cookie);
 486                 c->m_cookie = 0;
 487                 if (cookie)
 488                     c->m_cookie = xstrdup(cookie);
 489                 c->m_seqno = parent->m_seqno;
 490                 if (c->m_server && c->m_server != this)
 491                 {
 492                     c->m_server->m_client = 0;
 493                     delete c->m_server;
 494                 }
 495                 (parent->m_seqno)++;
 496                 c->m_target_idletime = m_target_idletime;
 497                 c->timeout(m_target_idletime);
 498
 499                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 500                     c->set_APDU_yazlog(1);
 501                 else
 502                     c->set_APDU_yazlog(0);
 503
 504                 return c;
 505             }
 506         }
 507         else
 508         {
 509             yaz_log (LOG_LOG, "%sNEW %d %s",
 510                      m_session_str, parent->m_seqno, m_proxyTarget);
 511             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 512             c->m_next = parent->m_clientPool;
 513             if (c->m_next)
 514                 c->m_next->m_prev = &c->m_next;
 515             parent->m_clientPool = c;
 516             c->m_prev = &parent->m_clientPool;
 517         }
 518
 519         xfree (c->m_cookie);
 520         c->m_cookie = 0;
 521         if (cookie)
 522             c->m_cookie = xstrdup(cookie);
 523
 524         c->m_seqno = parent->m_seqno;
 525         c->m_init_flag = 0;
 526         c->m_last_resultCount = 0;
 527         c->m_last_ok = 0;
 528         c->m_cache.clear();
 529         c->m_sr_transform = 0;
 530         c->m_waiting = 0;
 531         c->m_resultSetStartPoint = 0;
 532         (parent->m_seqno)++;
 533         if (c->client(m_proxyTarget))
 534         {
 535             delete c;
 536             return 0;
 537         }
 538         c->m_target_idletime = m_target_idletime;
 539         c->timeout(30);
 540
 541         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 542             c->set_APDU_yazlog(1);
 543         else
 544             c->set_APDU_yazlog(0);
 545     }
 546     yaz_log (LOG_DEBUG, "get_client 3 %p %p", this, c);
 547     return c;
 548 }
 549
 550 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 551 {
 552     int i;
 553     for (i = 0; i<num; i++)
 554     {
 555         oident *ent;
 556         Z_DefaultDiagFormat *r;
 557         Z_DiagRec *p = pp[i];
 558         if (p->which != Z_DiagRec_defaultFormat)
 559         {
 560             yaz_log(LOG_LOG, "%sError no diagnostics", m_session_str);
 561             return;
 562         }
 563         else
 564             r = p->u.defaultFormat;
 565         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 566             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 567             yaz_log(LOG_LOG, "%sError unknown diagnostic set", m_session_str);
 568         switch (r->which)
 569         {
 570         case Z_DefaultDiagFormat_v2Addinfo:
 571             yaz_log(LOG_LOG, "%sError %d %s:%s",
 572                     m_session_str,
 573                     *r->condition, diagbib1_str(*r->condition),
 574                     r->u.v2Addinfo);
 575             break;
 576         case Z_DefaultDiagFormat_v3Addinfo:
 577             yaz_log(LOG_LOG, "%sError %d %s:%s",
 578                     m_session_str,
 579                     *r->condition, diagbib1_str(*r->condition),
 580                     r->u.v3Addinfo);
 581             break;
 582         }
 583     }
 584 }
 585
 586 void Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p)
 587 {
 588     if (!m_stylesheet_schema)
 589         return;
 590     xsltStylesheetPtr xsp;
 591
 592     xsp = xsltParseStylesheetFile((const xmlChar *) m_stylesheet_schema);
 593
 594     int i;
 595     for (i = 0; i < p->num_records; i++)
 596     {
 597         Z_NamePlusRecord *npr = p->records[i];
 598         if (npr->which == Z_NamePlusRecord_databaseRecord)
 599         {
 600             Z_External *r = npr->u.databaseRecord;
 601             if (r->which == Z_External_octet)
 602             {
 603                 xmlDocPtr res, doc = xmlParseMemory(
 604                     (char*) r->u.octet_aligned->buf,
 605                     r->u.octet_aligned->len);
 606
 607                 res = xsltApplyStylesheet(xsp, doc, 0);
 608
 609                 xmlChar *out_buf;
 610                 int out_len;
 611                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 612
 613                 p->records[i]->u.databaseRecord =
 614                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 615                                  (char*) out_buf, out_len);
 616                 xmlFree(out_buf);
 617                 xmlFreeDoc(doc);
 618                 xmlFreeDoc(res);
 619             }
 620         }
 621     }
 622     xsltFreeStylesheet(xsp);
 623 }
 624
 625 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p)
 626 {
 627     int i;
 628
 629     yaz_marc_t mt = yaz_marc_create();
 630     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 631     for (i = 0; i < p->num_records; i++)
 632     {
 633         Z_NamePlusRecord *npr = p->records[i];
 634         if (npr->which == Z_NamePlusRecord_databaseRecord)
 635         {
 636             Z_External *r = npr->u.databaseRecord;
 637             if (r->which == Z_External_octet)
 638             {
 639                 int rlen;
 640                 char *result;
 641                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 642                                         r->u.octet_aligned->len,
 643                                         &result, &rlen))
 644                 {
 645                     yaz_iconv_t cd = yaz_iconv_open("UTF-8", "MARC-8");
 646                     WRBUF wrbuf = wrbuf_alloc();
 647
 648                     char outbuf[120];
 649                     size_t inbytesleft = rlen;
 650                     const char *inp = result;
 651                     while (cd && inbytesleft)
 652                     {
 653                         size_t outbytesleft = sizeof(outbuf);
 654                         char *outp = outbuf;
 655                         size_t r;
 656
 657                         r = yaz_iconv (cd, (char**) &inp,
 658                                        &inbytesleft,
 659                                        &outp, &outbytesleft);
 660                         if (r == (size_t) (-1))
 661                         {
 662                             int e = yaz_iconv_error(cd);
 663                             if (e != YAZ_ICONV_E2BIG)
 664                             {
 665                                 yaz_log(LOG_WARN, "conversion failure");
 666                                 break;
 667                             }
 668                         }
 669                         wrbuf_write(wrbuf, outbuf, outp - outbuf);
 670                     }
 671                     if (cd)
 672                         yaz_iconv_close(cd);
 673
 674                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 675                                                          VAL_TEXT_XML,
 676                                                          wrbuf_buf(wrbuf),
 677                                                          wrbuf_len(wrbuf));
 678                     wrbuf_free(wrbuf, 1);
 679                 }
 680             }
 681         }
 682     }
 683     yaz_marc_destroy(mt);
 684 }
 685
 686 void Yaz_Proxy::logtime()
 687 {
 688     if (m_time_tv.tv_sec)
 689     {
 690         struct timeval tv;
 691         gettimeofday(&tv, 0);
 692         long diff = (tv.tv_sec - m_time_tv.tv_sec)*1000000 +
 693             (tv.tv_usec - m_time_tv.tv_usec);
 694         if (diff >= 0)
 695             yaz_log(LOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 696                     diff/1000000, (diff/1000)%1000);
 697     }
 698     m_time_tv.tv_sec = 0;
 699     m_time_tv.tv_usec = 0;
 700 }
 701
 702 int Yaz_Proxy::send_http_response(int code)
 703 {
 704     ODR o = odr_encode();
 705     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 706     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 707     if (m_http_version)
 708         hres->version = odr_strdup(o, m_http_version);
 709     m_http_keepalive = 0;
 710     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 711     {
 712         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 713                  gdu_name(gdu));
 714     }
 715     int len;
 716     int r = send_GDU(gdu, &len);
 717     logtime();
 718     return r;
 719 }
 720
 721 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 722 {
 723     ODR o = odr_encode();
 724     const char *ctype = "text/xml";
 725     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 726     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 727     if (m_http_version)
 728         hres->version = odr_strdup(o, m_http_version);
 729     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 730     if (m_http_keepalive)
 731         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 732
 733     static Z_SOAP_Handler soap_handlers[2] = {
 734 #if HAVE_XSLT
 735         {"http://www.loc.gov/zing/srw/", 0,
 736          (Z_SOAP_fun) yaz_srw_codec},
 737 #endif
 738         {0, 0, 0}
 739     };
 740
 741     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 742     soap_package->which = Z_SOAP_generic;
 743     soap_package->u.generic =
 744         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 745     soap_package->u.generic->no = 0;
 746     soap_package->u.generic->ns = soap_handlers[0].ns;
 747     soap_package->u.generic->p = (void *) srw_pdu;
 748     soap_package->ns = m_soap_ns;
 749     z_soap_codec_enc_xsl(o, &soap_package,
 750                          &hres->content_buf, &hres->content_len,
 751                          soap_handlers, 0, m_s2z_stylesheet);
 752     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 753     {
 754         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 755                  gdu_name(gdu));
 756     }
 757     int len;
 758     int r = send_GDU(gdu, &len);
 759     logtime();
 760     return r;
 761 }
 762
 763 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
 764 {
 765     ODR o = odr_encode();
 766     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 767     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 768
 769     srw_res->num_diagnostics = 1;
 770     srw_res->diagnostics = (Z_SRW_diagnostic *)
 771         odr_malloc(o, sizeof(*srw_res->diagnostics));
 772     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
 773     return send_srw_response(srw_pdu);
 774 }
 775
 776 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 777                              Z_DefaultDiagFormat *ddf)
 778 {
 779     int bib1_code = *ddf->condition;
 780     if (bib1_code == 109)
 781         return 404;
 782     srw_res->num_diagnostics = 1;
 783     srw_res->diagnostics = (Z_SRW_diagnostic *)
 784         odr_malloc(o, sizeof(*srw_res->diagnostics));
 785     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
 786                           yaz_diag_bib1_to_srw(*ddf->condition),
 787                           ddf->u.v2Addinfo);
 788     return 0;
 789 }
 790
 791 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
 792 {
 793     ODR o = odr_encode();
 794     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 795     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 796
 797     srw_res->numberOfRecords = odr_intdup (o, hits);
 798     if (records && records->which == Z_Records_DBOSD)
 799     {
 800         srw_res->num_records =
 801             records->u.databaseOrSurDiagnostics->num_records;
 802         int i;
 803         srw_res->records = (Z_SRW_record *)
 804             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
 805         for (i = 0; i < srw_res->num_records; i++)
 806         {
 807             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
 808             if (npr->which != Z_NamePlusRecord_databaseRecord)
 809             {
 810                 srw_res->records[i].recordSchema = "diagnostic";
 811                 srw_res->records[i].recordPacking = m_s2z_packing;
 812                 srw_res->records[i].recordData_buf = "67";
 813                 srw_res->records[i].recordData_len = 2;
 814                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 815                 continue;
 816             }
 817             Z_External *r = npr->u.databaseRecord;
 818             oident *ent = oid_getentbyoid(r->direct_reference);
 819             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
 820             {
 821                 srw_res->records[i].recordSchema = m_schema;
 822                 srw_res->records[i].recordPacking = m_s2z_packing;
 823                 srw_res->records[i].recordData_buf = (char*)
 824                     r->u.octet_aligned->buf;
 825                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
 826                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 827             }
 828             else
 829             {
 830                 srw_res->records[i].recordSchema = "diagnostic";
 831                 srw_res->records[i].recordPacking = m_s2z_packing;
 832                 srw_res->records[i].recordData_buf = "67";
 833                 srw_res->records[i].recordData_len = 2;
 834                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 835             }
 836         }
 837     }
 838     if (records && records->which == Z_Records_NSD)
 839     {
 840         int http_code;
 841         http_code = z_to_srw_diag(odr_encode(), srw_res,
 842                                    records->u.nonSurrogateDiagnostic);
 843         if (http_code)
 844             return send_http_response(http_code);
 845     }
 846     return send_srw_response(srw_pdu);
 847
 848 }
 849
 850 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
 851                                         int num_diagnostics)
 852 {
 853     Yaz_ProxyConfig *cfg = check_reconfigure();
 854     if (cfg)
 855     {
 856         int len;
 857         char *b = cfg->get_explain(odr_encode(), 0 /* target */,
 858                                    m_s2z_database, &len);
 859         if (b)
 860         {
 861             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
 862             Z_SRW_explainResponse *er = res->u.explain_response;
 863
 864             er->record.recordData_buf = b;
 865             er->record.recordData_len = len;
 866             er->record.recordPacking = m_s2z_packing;
 867
 868             er->diagnostics = diagnostics;
 869             er->num_diagnostics = num_diagnostics;
 870             return send_srw_response(res);
 871         }
 872     }
 873     return send_http_response(404);
 874 }
 875
 876 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu, int *len)
 877 {
 878     if (m_http_version)
 879     {
 880         if (apdu->which == Z_APDU_initResponse)
 881         {
 882             Z_InitResponse *res = apdu->u.initResponse;
 883             if (*res->result == 0)
 884             {
 885                 send_to_srw_client_error(3, 0);
 886             }
 887             else if (!m_s2z_search_apdu)
 888             {
 889                 send_srw_explain_response(0, 0);
 890             }
 891             else
 892             {
 893                 handle_incoming_Z_PDU(m_s2z_search_apdu);
 894             }
 895         }
 896         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
 897         {
 898             m_s2z_search_apdu = 0;
 899             Z_SearchResponse *res = apdu->u.searchResponse;
 900             m_s2z_hit_count = *res->resultCount;
 901             if (res->records && res->records->which == Z_Records_NSD)
 902             {
 903                 send_to_srw_client_ok(0, res->records, 1);
 904             }
 905             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
 906             {
 907                 // adjust
 908                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
 909
 910                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
 911                 {
 912                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
 913                         > m_s2z_hit_count)
 914                         *pr->numberOfRecordsRequested =
 915                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
 916                 }
 917                 handle_incoming_Z_PDU(m_s2z_present_apdu);
 918             }
 919             else
 920             {
 921                 m_s2z_present_apdu = 0;
 922                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
 923             }
 924         }
 925         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
 926         {
 927             int start =
 928                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
 929
 930             m_s2z_present_apdu = 0;
 931             Z_PresentResponse *res = apdu->u.presentResponse;
 932             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
 933         }
 934     }
 935     else
 936     {
 937         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 938             yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 939                      apdu_name(apdu));
 940         int r = send_Z_PDU(apdu, len);
 941         logtime();
 942         return r;
 943     }
 944     return 0;
 945 }
 946
 947 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 948 {
 949     int len = 0;
 950     int kill_session = 0;
 951     if (apdu->which == Z_APDU_searchResponse)
 952     {
 953         Z_SearchResponse *sr = apdu->u.searchResponse;
 954         Z_Records *p = sr->records;
 955         if (p && p->which == Z_Records_NSD)
 956         {
 957             Z_DiagRec dr, *dr_p = &dr;
 958             dr.which = Z_DiagRec_defaultFormat;
 959             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 960
 961             *sr->searchStatus = 0;
 962             display_diagrecs(&dr_p, 1);
 963         }
 964         else
 965         {
 966             if (p && p->which == Z_Records_DBOSD)
 967             {
 968                 if (m_marcxml_flag)
 969                     convert_to_marcxml(p->u.databaseOrSurDiagnostics);
 970                 convert_xsl(p->u.databaseOrSurDiagnostics);
 971             }
 972             if (sr->resultCount)
 973             {
 974                 yaz_log(LOG_LOG, "%s%d hits", m_session_str,
 975                         *sr->resultCount);
 976                 if (*sr->resultCount < 0)
 977                 {
 978                     m_invalid_session = 1;
 979                     kill_session = 1;
 980
 981                     *sr->searchStatus = 0;
 982                     sr->records =
 983                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
 984                     *sr->resultCount = 0;
 985                 }
 986             }
 987         }
 988     }
 989     else if (apdu->which == Z_APDU_presentResponse)
 990     {
 991         Z_PresentResponse *sr = apdu->u.presentResponse;
 992         Z_Records *p = sr->records;
 993         if (p && p->which == Z_Records_NSD)
 994         {
 995             Z_DiagRec dr, *dr_p = &dr;
 996             dr.which = Z_DiagRec_defaultFormat;
 997             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 998             if (*sr->presentStatus == Z_PresentStatus_success)
 999                 *sr->presentStatus = Z_PresentStatus_failure;
1000             display_diagrecs(&dr_p, 1);
1001         }
1002         if (p && p->which == Z_Records_DBOSD)
1003         {
1004             if (m_marcxml_flag)
1005                 convert_to_marcxml(p->u.databaseOrSurDiagnostics);
1006             convert_xsl(p->u.databaseOrSurDiagnostics);
1007         }
1008     }
1009     else if (apdu->which == Z_APDU_initResponse)
1010     {
1011         if (m_initRequest_options)
1012         {
1013             Z_Options *nopt =
1014                 (Odr_bitmask *)odr_malloc(odr_encode(),
1015                                           sizeof(Odr_bitmask));
1016             ODR_MASK_ZERO(nopt);
1017
1018             int i;
1019             for (i = 0; i<24; i++)
1020                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1021                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1022                     ODR_MASK_SET(nopt, i);
1023             apdu->u.initResponse->options = nopt;
1024         }
1025         if (m_initRequest_version)
1026         {
1027             Z_ProtocolVersion *nopt =
1028                 (Odr_bitmask *)odr_malloc(odr_encode(),
1029                                           sizeof(Odr_bitmask));
1030             ODR_MASK_ZERO(nopt);
1031
1032             int i;
1033             for (i = 0; i<8; i++)
1034                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1035                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1036                     ODR_MASK_SET(nopt, i);
1037             apdu->u.initResponse->protocolVersion = nopt;
1038         }
1039     }
1040     int r = send_PDU_convert(apdu, &len);
1041     if (r)
1042         return r;
1043     m_bytes_sent += len;
1044     m_bw_stat.add_bytes(len);
1045     if (kill_session)
1046     {
1047         delete m_client;
1048         m_client = 0;
1049         m_parent->pre_init();
1050     }
1051     if (m_http_version)
1052     {
1053         if (!m_http_keepalive)
1054         {
1055 #if 1
1056             timeout(1);
1057 #else
1058             shutdown();
1059             return -1;
1060 #endif
1061         }
1062     }
1063     return r;
1064 }
1065
1066 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1067 {
1068     int len = 0;
1069     const char *apdu_name_tmp = apdu_name(apdu);
1070     int r = send_Z_PDU(apdu, &len);
1071     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1072         yaz_log (LOG_LOG, "%sSending %s to %s %d bytes",
1073                  get_session_str(),
1074                  apdu_name_tmp, get_hostname(), len);
1075     m_bytes_sent += len;
1076     return r;
1077 }
1078
1079 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1080 {
1081     if (apdu->which == Z_APDU_presentRequest)
1082     {
1083         Z_PresentRequest *pr = apdu->u.presentRequest;
1084         int toget = *pr->numberOfRecordsRequested;
1085         int start = *pr->resultSetStartPoint;
1086
1087         yaz_log(LOG_LOG, "%sPresent %s %d+%d", m_session_str,
1088                 pr->resultSetId, start, toget);
1089
1090         if (*m_parent->m_optimize == '0')
1091             return apdu;
1092
1093         if (!m_client->m_last_resultSetId)
1094         {
1095             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1096             new_apdu->u.presentResponse->records =
1097                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1098                                                pr->resultSetId);
1099             send_to_client(new_apdu);
1100             return 0;
1101         }
1102         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1103         {
1104             if (start+toget-1 > m_client->m_last_resultCount)
1105             {
1106                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1107                 new_apdu->u.presentResponse->records =
1108                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1109                 send_to_client(new_apdu);
1110                 return 0;
1111             }
1112             Z_NamePlusRecordList *npr;
1113             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1114                                           pr->preferredRecordSyntax,
1115                                           pr->recordComposition))
1116             {
1117                 yaz_log (LOG_LOG, "%sReturned cached records for present request",
1118                          m_session_str);
1119                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1120                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1121
1122                 new_apdu->u.presentResponse->numberOfRecordsReturned
1123                     = odr_intdup(odr_encode(), toget);
1124
1125                 new_apdu->u.presentResponse->records = (Z_Records*)
1126                     odr_malloc(odr_encode(), sizeof(Z_Records));
1127                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1128                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1129                 new_apdu->u.presentResponse->nextResultSetPosition =
1130                     odr_intdup(odr_encode(), start+toget);
1131
1132                 send_to_client(new_apdu);
1133                 return 0;
1134             }
1135         }
1136     }
1137
1138     if (apdu->which != Z_APDU_searchRequest)
1139         return apdu;
1140     Z_SearchRequest *sr = apdu->u.searchRequest;
1141     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1142     Yaz_Z_Databases this_databases;
1143
1144     this_databases.set(sr->num_databaseNames, (const char **)
1145                        sr->databaseNames);
1146
1147     this_query->set_Z_Query(sr->query);
1148
1149     char query_str[120];
1150     this_query->print(query_str, sizeof(query_str)-1);
1151     yaz_log(LOG_LOG, "%sSearch %s", m_session_str, query_str);
1152
1153     if (*m_parent->m_optimize != '0' &&
1154         m_client->m_last_ok && m_client->m_last_query &&
1155         m_client->m_last_query->match(this_query) &&
1156         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1157         m_client->m_last_databases.match(this_databases))
1158     {
1159         delete this_query;
1160         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1161             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1162         {
1163             Z_NamePlusRecordList *npr;
1164             int toget = *sr->mediumSetPresentNumber;
1165             Z_RecordComposition *comp = 0;
1166
1167             if (toget > m_client->m_last_resultCount)
1168                 toget = m_client->m_last_resultCount;
1169
1170             if (sr->mediumSetElementSetNames)
1171             {
1172                 comp = (Z_RecordComposition *)
1173                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1174                 comp->which = Z_RecordComp_simple;
1175                 comp->u.simple = sr->mediumSetElementSetNames;
1176             }
1177
1178             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1179                                           sr->preferredRecordSyntax, comp))
1180             {
1181                 yaz_log (LOG_LOG, "%sReturned cached records for medium set",
1182                          m_session_str);
1183                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1184                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1185                 new_apdu->u.searchResponse->resultCount =
1186                     &m_client->m_last_resultCount;
1187
1188                 new_apdu->u.searchResponse->numberOfRecordsReturned
1189                     = odr_intdup(odr_encode(), toget);
1190
1191                 new_apdu->u.searchResponse->presentStatus =
1192                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1193                 new_apdu->u.searchResponse->records = (Z_Records*)
1194                     odr_malloc(odr_encode(), sizeof(Z_Records));
1195                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1196                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1197                 new_apdu->u.searchResponse->nextResultSetPosition =
1198                     odr_intdup(odr_encode(), toget+1);
1199                 send_to_client(new_apdu);
1200                 return 0;
1201             }
1202             else
1203             {
1204                 // medium Set
1205                 // send present request (medium size)
1206                 yaz_log (LOG_LOG, "%sOptimizing search for medium set",
1207                          m_session_str);
1208
1209                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1210                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1211                 pr->referenceId = sr->referenceId;
1212                 pr->resultSetId = sr->resultSetName;
1213                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1214                 *pr->numberOfRecordsRequested = toget;
1215                 pr->recordComposition = comp;
1216                 m_client->m_sr_transform = 1;
1217                 return new_apdu;
1218             }
1219         }
1220         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1221             m_client->m_last_resultCount <= 0)
1222         {
1223             // large set. Return pseudo-search response immediately
1224             yaz_log (LOG_LOG, "%sOptimizing search for large set",
1225                      m_session_str);
1226             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1227             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1228             new_apdu->u.searchResponse->resultCount =
1229                 &m_client->m_last_resultCount;
1230             send_to_client(new_apdu);
1231             return 0;
1232         }
1233         else
1234         {
1235             Z_NamePlusRecordList *npr;
1236             int toget = m_client->m_last_resultCount;
1237             Z_RecordComposition *comp = 0;
1238             // small set
1239             // send a present request (small set)
1240
1241             if (sr->smallSetElementSetNames)
1242             {
1243                 comp = (Z_RecordComposition *)
1244                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1245                 comp->which = Z_RecordComp_simple;
1246                 comp->u.simple = sr->smallSetElementSetNames;
1247             }
1248
1249             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1250                                           sr->preferredRecordSyntax, comp))
1251             {
1252                 yaz_log (LOG_LOG, "%sReturned cached records for small set",
1253                          m_session_str);
1254                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1255                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1256                 new_apdu->u.searchResponse->resultCount =
1257                     &m_client->m_last_resultCount;
1258
1259                 new_apdu->u.searchResponse->numberOfRecordsReturned
1260                     = odr_intdup(odr_encode(), toget);
1261
1262                 new_apdu->u.searchResponse->presentStatus =
1263                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1264                 new_apdu->u.searchResponse->records = (Z_Records*)
1265                     odr_malloc(odr_encode(), sizeof(Z_Records));
1266                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1267                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1268                 new_apdu->u.searchResponse->nextResultSetPosition =
1269                     odr_intdup(odr_encode(), toget+1);
1270                 send_to_client(new_apdu);
1271                 return 0;
1272             }
1273             else
1274             {
1275                 yaz_log (LOG_LOG, "%sOptimizing search for small set",
1276                          m_session_str);
1277                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1278                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1279                 pr->referenceId = sr->referenceId;
1280                 pr->resultSetId = sr->resultSetName;
1281                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1282                 *pr->numberOfRecordsRequested = toget;
1283                 pr->recordComposition = comp;
1284                 m_client->m_sr_transform = 1;
1285                 return new_apdu;
1286             }
1287         }
1288     }
1289     else  // query doesn't match
1290     {
1291         delete m_client->m_last_query;
1292         m_client->m_last_query = this_query;
1293         m_client->m_last_ok = 0;
1294         m_client->m_cache.clear();
1295         m_client->m_resultSetStartPoint = 0;
1296
1297         xfree (m_client->m_last_resultSetId);
1298         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1299
1300         m_client->m_last_databases.set(sr->num_databaseNames,
1301                                        (const char **) sr->databaseNames);
1302     }
1303     return apdu;
1304 }
1305
1306
1307 void Yaz_Proxy::inc_request_no()
1308 {
1309     char *cp = strchr(m_session_str, ' ');
1310     m_request_no++;
1311     if (cp)
1312         sprintf(cp+1, "%d ", m_request_no);
1313 }
1314
1315 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1316 {
1317     inc_request_no();
1318
1319     m_bytes_recv += len;
1320
1321     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
1322         yaz_log (LOG_DEBUG, "%sReceiving %s from client %d bytes",
1323                  m_session_str, gdu_name(apdu), len);
1324
1325     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1326         shutdown();
1327
1328     m_bw_stat.add_bytes(len);
1329     m_pdu_stat.add_bytes(1);
1330
1331     gettimeofday(&m_time_tv, 0);
1332
1333     int bw_total = m_bw_stat.get_total();
1334     int pdu_total = m_pdu_stat.get_total();
1335
1336     int reduce = 0;
1337     if (m_bw_max)
1338     {
1339         if (bw_total > m_bw_max)
1340         {
1341             reduce = (bw_total/m_bw_max);
1342         }
1343     }
1344     if (m_pdu_max)
1345     {
1346         if (pdu_total > m_pdu_max)
1347         {
1348             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1349             reduce = (reduce > nreduce) ? reduce : nreduce;
1350         }
1351     }
1352     if (reduce)
1353     {
1354         yaz_log(LOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1355                 m_session_str, reduce, bw_total, pdu_total,
1356                 m_bw_max, m_pdu_max);
1357
1358         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1359         timeout(reduce);       // call us reduce seconds later
1360     }
1361     else if (apdu->which == Z_GDU_Z3950)
1362         handle_incoming_Z_PDU(apdu->u.z3950);
1363     else if (apdu->which == Z_GDU_HTTP_Request)
1364         handle_incoming_HTTP(apdu->u.HTTP_Request);
1365 }
1366
1367 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1368 {
1369     if (m_max_record_retrieve)
1370     {
1371         if (apdu->which == Z_APDU_presentRequest)
1372         {
1373             Z_PresentRequest *pr = apdu->u.presentRequest;
1374             if (pr->numberOfRecordsRequested &&
1375                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1376                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1377         }
1378     }
1379 }
1380
1381 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1382                                                      int error,
1383                                                      const char *addinfo)
1384 {
1385     Z_Records *rec = (Z_Records *)
1386         odr_malloc (odr, sizeof(*rec));
1387     int *err = (int *)
1388         odr_malloc (odr, sizeof(*err));
1389     Z_DiagRec *drec = (Z_DiagRec *)
1390         odr_malloc (odr, sizeof(*drec));
1391     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1392         odr_malloc (odr, sizeof(*dr));
1393     *err = error;
1394     rec->which = Z_Records_NSD;
1395     rec->u.nonSurrogateDiagnostic = dr;
1396     dr->diagnosticSetId =
1397         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1398     dr->condition = err;
1399     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1400     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1401     return rec;
1402 }
1403
1404 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1405 {
1406     if (apdu->which == Z_APDU_searchRequest &&
1407         apdu->u.searchRequest->query &&
1408         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1409         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1410     {
1411         Z_RPNQuery *rpnquery = 0;
1412         Z_SearchRequest *sr = apdu->u.searchRequest;
1413         char *addinfo = 0;
1414
1415         yaz_log(LOG_LOG, "%sCQL: %s", m_session_str,
1416                 sr->query->u.type_104->u.cql);
1417
1418         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1419                                           &rpnquery, odr_encode(),
1420                                           &addinfo);
1421         if (r == -3)
1422             yaz_log(LOG_LOG, "%sNo CQL to RPN table", m_session_str);
1423         else if (r)
1424         {
1425             yaz_log(LOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1426             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1427
1428             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1429             new_apdu->u.searchResponse->records =
1430                 create_nonSurrogateDiagnostics(odr_encode(),
1431                                                yaz_diag_srw_to_bib1(r),
1432                                                addinfo);
1433             *new_apdu->u.searchResponse->searchStatus = 0;
1434
1435             send_to_client(new_apdu);
1436
1437             return 0;
1438         }
1439         else
1440         {
1441             sr->query->which = Z_Query_type_1;
1442             sr->query->u.type_1 = rpnquery;
1443         }
1444         return apdu;
1445     }
1446     return apdu;
1447 }
1448
1449 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1450 {
1451     if (apdu->which == Z_APDU_searchRequest)
1452     {
1453         Z_SearchRequest *sr = apdu->u.searchRequest;
1454         int err = 0;
1455         char *addinfo = 0;
1456
1457         Yaz_ProxyConfig *cfg = check_reconfigure();
1458         if (cfg)
1459             err = cfg->check_query(odr_encode(), m_default_target,
1460                                    sr->query, &addinfo);
1461         if (err)
1462         {
1463             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1464
1465             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1466             new_apdu->u.searchResponse->records =
1467                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1468             *new_apdu->u.searchResponse->searchStatus = 0;
1469
1470             send_to_client(new_apdu);
1471
1472             return 0;
1473         }
1474     }
1475     return apdu;
1476 }
1477
1478 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1479 {
1480     m_marcxml_flag = 0;
1481     if (apdu->which == Z_APDU_searchRequest)
1482     {
1483         Z_SearchRequest *sr = apdu->u.searchRequest;
1484         int err = 0;
1485         char *addinfo = 0;
1486         Yaz_ProxyConfig *cfg = check_reconfigure();
1487
1488         Z_RecordComposition rc_temp, *rc = 0;
1489         if (sr->smallSetElementSetNames)
1490         {
1491             rc_temp.which = Z_RecordComp_simple;
1492             rc_temp.u.simple = sr->smallSetElementSetNames;
1493             rc = &rc_temp;
1494         }
1495
1496         if (cfg)
1497             err = cfg->check_syntax(odr_encode(),
1498                                     m_default_target,
1499                                     sr->preferredRecordSyntax, rc,
1500                                     &addinfo, &m_stylesheet_schema, &m_schema);
1501         if (err == -1)
1502         {
1503             sr->preferredRecordSyntax =
1504                 yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN, VAL_USMARC);
1505             m_marcxml_flag = 1;
1506         }
1507         else if (err)
1508         {
1509             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1510
1511             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1512             new_apdu->u.searchResponse->records =
1513                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1514             *new_apdu->u.searchResponse->searchStatus = 0;
1515
1516             send_to_client(new_apdu);
1517
1518             return 0;
1519         }
1520     }
1521     else if (apdu->which == Z_APDU_presentRequest)
1522     {
1523         Z_PresentRequest *pr = apdu->u.presentRequest;
1524         int err = 0;
1525         char *addinfo = 0;
1526         Yaz_ProxyConfig *cfg = check_reconfigure();
1527
1528         if (cfg)
1529             err = cfg->check_syntax(odr_encode(), m_default_target,
1530                                     pr->preferredRecordSyntax,
1531                                     pr->recordComposition,
1532                                     &addinfo, &m_stylesheet_schema, &m_schema);
1533         if (err == -1)
1534         {
1535             pr->preferredRecordSyntax =
1536                 yaz_oidval_to_z3950oid(odr_decode(), CLASS_RECSYN, VAL_USMARC);
1537             m_marcxml_flag = 1;
1538         }
1539         else if (err)
1540         {
1541             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1542
1543             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1544             new_apdu->u.presentResponse->records =
1545                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1546             *new_apdu->u.presentResponse->presentStatus =
1547                 Z_PresentStatus_failure;
1548
1549             send_to_client(new_apdu);
1550
1551             return 0;
1552         }
1553     }
1554     return apdu;
1555 }
1556
1557 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
1558 {
1559     if (!schema)
1560         return 0;
1561     Z_ElementSetNames *esn = (Z_ElementSetNames *)
1562         odr_malloc(o, sizeof(Z_ElementSetNames));
1563     esn->which = Z_ElementSetNames_generic;
1564     esn->u.generic = odr_strdup(o, schema);
1565     return esn;
1566 }
1567
1568 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1569 {
1570     if (m_s2z_odr_init)
1571     {
1572         odr_destroy(m_s2z_odr_init);
1573         m_s2z_odr_init = 0;
1574     }
1575     if (m_s2z_odr_search)
1576     {
1577         odr_destroy(m_s2z_odr_search);
1578         m_s2z_odr_search = 0;
1579     }
1580
1581     m_http_keepalive = 0;
1582     m_http_version = 0;
1583     if (!strcmp(hreq->version, "1.0"))
1584     {
1585         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1586         if (v && !strcmp(v, "Keep-Alive"))
1587             m_http_keepalive = 1;
1588         else
1589             m_http_keepalive = 0;
1590         m_http_version = "1.0";
1591     }
1592     else
1593     {
1594         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1595         if (v && !strcmp(v, "close"))
1596             m_http_keepalive = 0;
1597         else
1598             m_http_keepalive = 1;
1599         m_http_version = "1.1";
1600     }
1601
1602     Z_SRW_PDU *srw_pdu = 0;
1603     Z_SOAP *soap_package = 0;
1604     char *charset = 0;
1605     Z_SRW_diagnostic *diagnostic = 0;
1606     int num_diagnostic = 0;
1607     if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1608                        &charset) == 0
1609         || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1610                           &charset, &diagnostic, &num_diagnostic) == 0)
1611     {
1612         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
1613         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
1614         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
1615         m_s2z_init_apdu = 0;
1616         m_s2z_search_apdu = 0;
1617         m_s2z_present_apdu = 0;
1618
1619         m_s2z_stylesheet = 0;
1620
1621         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
1622         {
1623             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
1624
1625             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1626             // recordXPath unsupported.
1627             if (srw_req->recordXPath)
1628             {
1629                 yaz_add_srw_diagnostic(odr_decode(),
1630                                        &diagnostic, &num_diagnostic,
1631                                        72, 0);
1632             }
1633             // must have a query
1634             if (!srw_req->query.cql)
1635             {
1636                 yaz_add_srw_diagnostic(odr_decode(),
1637                                        &diagnostic, &num_diagnostic,
1638                                        7, "query");
1639             }
1640             // sort unsupported
1641             if (srw_req->sort_type != Z_SRW_sort_type_none)
1642             {
1643                 yaz_add_srw_diagnostic(odr_decode(),
1644                                        &diagnostic, &num_diagnostic,
1645                                        80, 0);
1646             }
1647             // save stylesheet
1648             if (srw_req->stylesheet)
1649                 m_s2z_stylesheet =
1650                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1651
1652             // set packing for response records ..
1653             if (srw_req->recordPacking &&
1654                 !strcmp(srw_req->recordPacking, "xml"))
1655                 m_s2z_packing = Z_SRW_recordPacking_XML;
1656             else
1657                 m_s2z_packing = Z_SRW_recordPacking_string;
1658
1659             if (num_diagnostic)
1660             {
1661                 Z_SRW_PDU *srw_pdu =
1662                     yaz_srw_get(odr_encode(),
1663                                 Z_SRW_searchRetrieve_response);
1664                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1665
1666                 srw_res->diagnostics = diagnostic;
1667                 srw_res->num_diagnostics = num_diagnostic;
1668                 send_srw_response(srw_pdu);
1669                 return;
1670             }
1671
1672             // prepare search PDU
1673             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
1674                                           Z_APDU_searchRequest);
1675             Z_SearchRequest *z_searchRequest =
1676                 m_s2z_search_apdu->u.searchRequest;
1677
1678             z_searchRequest->num_databaseNames = 1;
1679             z_searchRequest->databaseNames = (char**)
1680                 odr_malloc(m_s2z_odr_search, sizeof(char *));
1681             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
1682                                                            srw_req->database);
1683
1684             // query transformation
1685             Z_Query *query = (Z_Query *)
1686                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
1687             z_searchRequest->query = query;
1688
1689             if (srw_req->query_type == Z_SRW_query_type_cql)
1690             {
1691                 Z_External *ext = (Z_External *)
1692                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
1693                 ext->direct_reference =
1694                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
1695                 ext->indirect_reference = 0;
1696                 ext->descriptor = 0;
1697                 ext->which = Z_External_CQL;
1698                 ext->u.cql = srw_req->query.cql;
1699
1700                 query->which = Z_Query_type_104;
1701                 query->u.type_104 =  ext;
1702             }
1703             else if (srw_req->query_type == Z_SRW_query_type_pqf)
1704             {
1705                 Z_RPNQuery *RPNquery;
1706                 YAZ_PQF_Parser pqf_parser;
1707
1708                 pqf_parser = yaz_pqf_create ();
1709
1710                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
1711                                           srw_req->query.pqf);
1712                 if (!RPNquery)
1713                 {
1714                     const char *pqf_msg;
1715                     size_t off;
1716                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
1717                     yaz_log(LOG_LOG, "%*s^\n", off+4, "");
1718                     yaz_log(LOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
1719
1720                     send_to_srw_client_error(10, 0);
1721                     return;
1722                 }
1723                 query->which = Z_Query_type_1;
1724                 query->u.type_1 =  RPNquery;
1725
1726                 yaz_pqf_destroy (pqf_parser);
1727             }
1728             else
1729             {
1730                 send_to_srw_client_error(7, "query");
1731                 return;
1732             }
1733
1734             // present
1735             m_s2z_present_apdu = 0;
1736             int max = 0;
1737             if (srw_req->maximumRecords)
1738                 max = *srw_req->maximumRecords;
1739             int start = 1;
1740             if (srw_req->startRecord)
1741                 start = *srw_req->startRecord;
1742             if (max > 0)
1743             {
1744                 // Some backend, such as Voyager doesn't honor piggyback
1745                 // So we use present always (0 &&).
1746                 if (0 && start <= 1)  // Z39.50 piggyback
1747                 {
1748                     *z_searchRequest->smallSetUpperBound = max;
1749                     *z_searchRequest->mediumSetPresentNumber = max;
1750                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
1751
1752                     z_searchRequest->preferredRecordSyntax =
1753                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1754                                                VAL_TEXT_XML);
1755                     if (srw_req->recordSchema)
1756                     {
1757                         z_searchRequest->smallSetElementSetNames =
1758                             z_searchRequest->mediumSetElementSetNames =
1759                             mk_esn_from_schema(m_s2z_odr_search,
1760                                                srw_req->recordSchema);
1761                     }
1762                 }
1763                 else   // Z39.50 present
1764                 {
1765                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
1766                                                    Z_APDU_presentRequest);
1767                     Z_PresentRequest *z_presentRequest =
1768                         m_s2z_present_apdu->u.presentRequest;
1769                     *z_presentRequest->resultSetStartPoint = start;
1770                     *z_presentRequest->numberOfRecordsRequested = max;
1771                     z_presentRequest->preferredRecordSyntax =
1772                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1773                                                VAL_TEXT_XML);
1774                     z_presentRequest->recordComposition =
1775                         (Z_RecordComposition *)
1776                         odr_malloc(m_s2z_odr_search,
1777                                    sizeof(Z_RecordComposition));
1778                     if (srw_req->recordSchema)
1779                     {
1780                         z_presentRequest->recordComposition->which =
1781                             Z_RecordComp_simple;
1782                         z_presentRequest->recordComposition->u.simple =
1783                             mk_esn_from_schema(m_s2z_odr_search,
1784                                                srw_req->recordSchema);
1785                     }
1786                 }
1787             }
1788             if (!m_client)
1789             {
1790                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1791                                             Z_APDU_initRequest);
1792
1793                 // prevent m_initRequest_apdu memory from being grabbed
1794                 // in Yaz_Proxy::handle_incoming_Z_PDU
1795                 m_initRequest_apdu = m_s2z_init_apdu;
1796                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1797                 return;
1798             }
1799             else
1800             {
1801                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1802                 return;
1803             }
1804         }
1805         else if (srw_pdu->which == Z_SRW_explain_request)
1806         {
1807             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
1808
1809             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1810
1811             // save stylesheet
1812             if (srw_req->stylesheet)
1813                 m_s2z_stylesheet =
1814                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1815
1816             if (srw_req->recordPacking &&
1817                 !strcmp(srw_req->recordPacking, "xml"))
1818                 m_s2z_packing = Z_SRW_recordPacking_XML;
1819             else
1820                 m_s2z_packing = Z_SRW_recordPacking_string;
1821
1822             if (num_diagnostic)
1823             {
1824                 send_srw_explain_response(diagnostic, num_diagnostic);
1825                 return;
1826             }
1827
1828             if (!m_client)
1829             {
1830                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1831                                             Z_APDU_initRequest);
1832
1833                 // prevent m_initRequest_apdu memory from being grabbed
1834                 // in Yaz_Proxy::handle_incoming_Z_PDU
1835                 m_initRequest_apdu = m_s2z_init_apdu;
1836                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1837             }
1838             else
1839                 send_srw_explain_response(0, 0);
1840             return;
1841         }
1842         else if (srw_pdu->which == Z_SRW_scan_request)
1843         {
1844             m_s2z_database = odr_strdup(m_s2z_odr_init,
1845                                         srw_pdu->u.scan_request->database);
1846
1847             yaz_add_srw_diagnostic(odr_decode(),
1848                                    &diagnostic, &num_diagnostic,
1849                                    4, "scan");
1850             Z_SRW_PDU *srw_pdu =
1851                 yaz_srw_get(odr_encode(),
1852                             Z_SRW_scan_response);
1853             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1854
1855             srw_res->diagnostics = diagnostic;
1856             srw_res->num_diagnostics = num_diagnostic;
1857             send_srw_response(srw_pdu);
1858             return;
1859         }
1860         else
1861         {
1862             m_s2z_database = 0;
1863
1864             send_to_srw_client_error(4, 0);
1865         }
1866     }
1867     int len = 0;
1868     Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
1869     send_GDU(p, &len);
1870     timeout(1);
1871 }
1872
1873 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
1874 {
1875     if (!m_client && m_invalid_session)
1876     {
1877         m_apdu_invalid_session = apdu;
1878         m_mem_invalid_session = odr_extract_mem(odr_decode());
1879         apdu = m_initRequest_apdu;
1880     }
1881
1882     // Determine our client.
1883     Z_OtherInformation **oi;
1884     get_otherInfoAPDU(apdu, &oi);
1885     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
1886     if (!m_client)
1887     {
1888         delete this;
1889         return;
1890     }
1891     m_client->m_server = this;
1892
1893     if (apdu->which == Z_APDU_initRequest)
1894     {
1895         if (apdu->u.initRequest->implementationId)
1896             yaz_log(LOG_LOG, "%simplementationId: %s",
1897                     m_session_str, apdu->u.initRequest->implementationId);
1898         if (apdu->u.initRequest->implementationName)
1899             yaz_log(LOG_LOG, "%simplementationName: %s",
1900                     m_session_str, apdu->u.initRequest->implementationName);
1901         if (apdu->u.initRequest->implementationVersion)
1902             yaz_log(LOG_LOG, "%simplementationVersion: %s",
1903                     m_session_str, apdu->u.initRequest->implementationVersion);
1904         if (m_initRequest_apdu == 0)
1905         {
1906             if (m_initRequest_mem)
1907                 nmem_destroy(m_initRequest_mem);
1908             m_initRequest_apdu = apdu;
1909             m_initRequest_mem = odr_extract_mem(odr_decode());
1910
1911             // save init options for the response..
1912             m_initRequest_options = apdu->u.initRequest->options;
1913
1914             apdu->u.initRequest->options =
1915                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
1916                                            sizeof(Odr_bitmask));
1917             ODR_MASK_ZERO(apdu->u.initRequest->options);
1918             int i;
1919             for (i = 0; i<= 24; i++)
1920                 ODR_MASK_SET(apdu->u.initRequest->options, i);
1921             ODR_MASK_CLEAR(apdu->u.initRequest->options,
1922                            Z_Options_negotiationModel);
1923
1924             // make new version
1925             m_initRequest_version = apdu->u.initRequest->protocolVersion;
1926             apdu->u.initRequest->protocolVersion =
1927                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
1928                                            sizeof(Odr_bitmask));
1929             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
1930
1931             for (i = 0; i<= 8; i++)
1932                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
1933         }
1934         if (m_client->m_init_flag)
1935         {
1936             if (handle_init_response_for_invalid_session(apdu))
1937                 return;
1938             Z_APDU *apdu2 = m_client->m_initResponse;
1939             apdu2->u.initResponse->otherInfo = 0;
1940             if (m_client->m_cookie && *m_client->m_cookie)
1941                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
1942                                            m_client->m_cookie);
1943             apdu2->u.initResponse->referenceId =
1944                 apdu->u.initRequest->referenceId;
1945             apdu2->u.initResponse->options = m_client->m_initResponse_options;
1946             apdu2->u.initResponse->protocolVersion =
1947                 m_client->m_initResponse_version;
1948
1949             send_to_client(apdu2);
1950             return;
1951         }
1952         m_client->m_init_flag = 1;
1953     }
1954     handle_max_record_retrieve(apdu);
1955
1956     if (apdu)
1957         apdu = handle_syntax_validation(apdu);
1958
1959     if (apdu)
1960         apdu = handle_query_transformation(apdu);
1961
1962     if (apdu)
1963         apdu = handle_query_validation(apdu);
1964
1965     if (apdu)
1966         apdu = result_set_optimize(apdu);
1967     if (!apdu)
1968     {
1969         m_client->timeout(m_target_idletime);  // mark it active even
1970         // though we didn't use it
1971         return;
1972     }
1973
1974     // delete other info part from PDU before sending to target
1975     get_otherInfoAPDU(apdu, &oi);
1976     if (oi)
1977         *oi = 0;
1978
1979     if (apdu->which == Z_APDU_presentRequest &&
1980         m_client->m_resultSetStartPoint == 0)
1981     {
1982         Z_PresentRequest *pr = apdu->u.presentRequest;
1983         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
1984         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
1985     } else {
1986         m_client->m_resultSetStartPoint = 0;
1987     }
1988     if (m_client->send_to_target(apdu) < 0)
1989     {
1990         delete m_client;
1991         m_client = 0;
1992         delete this;
1993     }
1994     else
1995         m_client->m_waiting = 1;
1996 }
1997
1998 void Yaz_Proxy::connectNotify()
1999 {
2000 }
2001
2002 void Yaz_Proxy::shutdown()
2003 {
2004     m_invalid_session = 0;
2005     // only keep if keep_alive flag is set...
2006     if (m_client &&
2007         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2008         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2009         m_client->m_waiting == 0)
2010     {
2011         yaz_log(LOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2012                  m_session_str,
2013                  m_client->get_hostname());
2014         yaz_log(LOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2015                 m_session_str, m_client->m_pdu_recv,
2016                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2017                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2018         assert (m_client->m_waiting != 2);
2019         // Tell client (if any) that no server connection is there..
2020         m_client->m_server = 0;
2021         m_invalid_session = 0;
2022     }
2023     else if (m_client)
2024     {
2025         yaz_log (LOG_LOG, "%sShutdown (client to proxy) close %s",
2026                  m_session_str,
2027                  m_client->get_hostname());
2028         assert (m_client->m_waiting != 2);
2029         delete m_client;
2030     }
2031     else if (!m_parent)
2032     {
2033         yaz_log (LOG_LOG, "%sshutdown (client to proxy) bad state",
2034                  m_session_str);
2035         assert (m_parent);
2036     }
2037     else
2038     {
2039         yaz_log (LOG_LOG, "%sShutdown (client to proxy)",
2040                  m_session_str);
2041     }
2042     if (m_parent)
2043         m_parent->pre_init();
2044     delete this;
2045 }
2046
2047 const char *Yaz_ProxyClient::get_session_str()
2048 {
2049     if (!m_server)
2050         return "0 ";
2051     return m_server->get_session_str();
2052 }
2053
2054 void Yaz_ProxyClient::shutdown()
2055 {
2056     yaz_log (LOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2057              get_hostname());
2058     delete m_server;
2059     delete this;
2060 }
2061
2062 void Yaz_Proxy::failNotify()
2063 {
2064     inc_request_no();
2065     yaz_log (LOG_LOG, "%sConnection closed by client",
2066              get_session_str());
2067     shutdown();
2068 }
2069
2070 void Yaz_ProxyClient::failNotify()
2071 {
2072     if (m_server)
2073         m_server->inc_request_no();
2074     yaz_log (LOG_LOG, "%sConnection closed by target %s",
2075              get_session_str(), get_hostname());
2076     shutdown();
2077 }
2078
2079 void Yaz_ProxyClient::connectNotify()
2080 {
2081     const char *s = get_session_str();
2082     const char *h = get_hostname();
2083     yaz_log (LOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2084              m_target_idletime);
2085     timeout(m_target_idletime);
2086     if (!m_server)
2087         pre_init_client();
2088 }
2089
2090 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2091                                                   *the_PDU_Observable, int fd)
2092 {
2093     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2094 }
2095
2096 Yaz_ProxyClient::~Yaz_ProxyClient()
2097 {
2098     if (m_prev)
2099         *m_prev = m_next;
2100     if (m_next)
2101         m_next->m_prev = m_prev;
2102     m_waiting = 2;     // for debugging purposes only.
2103     odr_destroy(m_init_odr);
2104     delete m_last_query;
2105     xfree (m_last_resultSetId);
2106     xfree (m_cookie);
2107 }
2108
2109 void Yaz_ProxyClient::pre_init_client()
2110 {
2111     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2112     Z_InitRequest *req = apdu->u.initRequest;
2113
2114     int i;
2115     for (i = 0; i<= 24; i++)
2116         ODR_MASK_SET(req->options, i);
2117     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2118                    Z_Options_negotiationModel);
2119     for (i = 0; i<= 10; i++)
2120         ODR_MASK_SET(req->protocolVersion, i);
2121
2122     if (send_to_target(apdu) < 0)
2123     {
2124         delete this;
2125     }
2126     else
2127     {
2128         m_waiting = 1;
2129         m_init_flag = 1;
2130     }
2131 }
2132
2133 void Yaz_Proxy::pre_init()
2134 {
2135     int i;
2136     const char *name = 0;
2137     const char *zurl_in_use[MAX_ZURL_PLEX];
2138     int limit_bw, limit_pdu, limit_req;
2139     int target_idletime, client_idletime;
2140     int max_clients;
2141     int keepalive_limit_bw, keepalive_limit_pdu;
2142     int pre_init;
2143     const char *cql2rpn = 0;
2144
2145     Yaz_ProxyConfig *cfg = check_reconfigure();
2146
2147     zurl_in_use[0] = 0;
2148
2149     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2150         set_APDU_yazlog(1);
2151     else
2152         set_APDU_yazlog(0);
2153
2154     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2155                                           &limit_bw, &limit_pdu, &limit_req,
2156                                           &target_idletime, &client_idletime,
2157                                           &max_clients,
2158                                           &keepalive_limit_bw,
2159                                           &keepalive_limit_pdu,
2160                                           &pre_init,
2161                                           &cql2rpn) ; i++)
2162     {
2163         if (pre_init)
2164         {
2165             int j;
2166             for (j = 0; zurl_in_use[j]; j++)
2167             {
2168                 Yaz_ProxyClient *c;
2169                 int spare = 0;
2170                 int in_use = 0;
2171                 int other = 0;
2172                 for (c = m_clientPool; c; c = c->m_next)
2173                 {
2174                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
2175                     {
2176                         if (c->m_cookie == 0)
2177                         {
2178                             if (c->m_server == 0)
2179                                 spare++;
2180                             else
2181                                 in_use++;
2182                         }
2183                         else
2184                             other++;
2185                     }
2186                 }
2187                 yaz_log(LOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2188                         "preinit=%d",m_session_str,
2189                         name, zurl_in_use[j], in_use, other, spare, pre_init);
2190                 if (spare < pre_init)
2191                 {
2192                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2193                     c->m_next = m_clientPool;
2194                     if (c->m_next)
2195                         c->m_next->m_prev = &c->m_next;
2196                     m_clientPool = c;
2197                     c->m_prev = &m_clientPool;
2198
2199                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
2200                         c->set_APDU_yazlog(1);
2201                     else
2202                         c->set_APDU_yazlog(0);
2203
2204                     if (c->client(zurl_in_use[j]))
2205                     {
2206                         timeout(60);
2207                         delete c;
2208                         return;
2209                     }
2210                     c->timeout(30);
2211                     c->m_waiting = 1;
2212                     c->m_target_idletime = target_idletime;
2213                     c->m_seqno = m_seqno++;
2214                 }
2215             }
2216         }
2217     }
2218 }
2219
2220 void Yaz_Proxy::timeoutNotify()
2221 {
2222     if (m_parent)
2223     {
2224         if (m_bw_hold_PDU)
2225         {
2226             timeout(m_client_idletime);
2227             Z_GDU *apdu = m_bw_hold_PDU;
2228             m_bw_hold_PDU = 0;
2229
2230             if (apdu->which == Z_GDU_Z3950)
2231                 handle_incoming_Z_PDU(apdu->u.z3950);
2232             else if (apdu->which == Z_GDU_HTTP_Request)
2233                 handle_incoming_HTTP(apdu->u.HTTP_Request);
2234         }
2235         else
2236         {
2237             inc_request_no();
2238
2239             yaz_log (LOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2240             shutdown();
2241         }
2242     }
2243     else
2244     {
2245         timeout(600);
2246         pre_init();
2247     }
2248 }
2249
2250 void Yaz_Proxy::markInvalid()
2251 {
2252     m_client = 0;
2253     m_invalid_session = 1;
2254 }
2255
2256 void Yaz_ProxyClient::timeoutNotify()
2257 {
2258     if (m_server)
2259         m_server->inc_request_no();
2260
2261     yaz_log (LOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2262              get_hostname());
2263     m_waiting = 1;
2264     m_root->pre_init();
2265     if (m_server && m_init_flag)
2266     {
2267         // target timed out in a session that was properly initialized
2268         // server object stay alive but we mark it as invalid so it
2269         // gets initialized again
2270         m_server->markInvalid();
2271         m_server = 0;
2272     }
2273     shutdown();
2274 }
2275
2276 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2277                                  Yaz_Proxy *parent) :
2278     Yaz_Z_Assoc (the_PDU_Observable)
2279 {
2280     m_cookie = 0;
2281     m_next = 0;
2282     m_prev = 0;
2283     m_init_flag = 0;
2284     m_last_query = 0;
2285     m_last_resultSetId = 0;
2286     m_last_resultCount = 0;
2287     m_last_ok = 0;
2288     m_sr_transform = 0;
2289     m_waiting = 0;
2290     m_init_odr = odr_createmem (ODR_DECODE);
2291     m_initResponse = 0;
2292     m_initResponse_options = 0;
2293     m_initResponse_version = 0;
2294     m_resultSetStartPoint = 0;
2295     m_bytes_sent = m_bytes_recv = 0;
2296     m_pdu_recv = 0;
2297     m_server = 0;
2298     m_seqno = 0;
2299     m_target_idletime = 600;
2300     m_root = parent;
2301 }
2302
2303 const char *Yaz_Proxy::option(const char *name, const char *value)
2304 {
2305     if (!strcmp (name, "optimize")) {
2306         if (value) {
2307             xfree (m_optimize);
2308             m_optimize = xstrdup (value);
2309         }
2310         return m_optimize;
2311     }
2312     return 0;
2313 }
2314
2315 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2316 {
2317
2318 }
2319
2320 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2321 {
2322     if (apdu->which == Z_GDU_Z3950)
2323         recv_Z_PDU(apdu->u.z3950, len);
2324     else if (apdu->which == Z_GDU_HTTP_Response)
2325         recv_HTTP_response(apdu->u.HTTP_Response, len);
2326     else
2327         shutdown();
2328 }
2329
2330 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2331 {
2332     if (!m_invalid_session)
2333         return 0;
2334     m_invalid_session = 0;
2335     handle_incoming_Z_PDU(m_apdu_invalid_session);
2336     assert (m_mem_invalid_session);
2337     nmem_destroy(m_mem_invalid_session);
2338     m_mem_invalid_session = 0;
2339     return 1;
2340 }
2341
2342 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2343 {
2344     m_bytes_recv += len;
2345     m_pdu_recv++;
2346     m_waiting = 0;
2347     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2348         yaz_log (LOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2349                  apdu_name(apdu), get_hostname(), len);
2350     if (apdu->which == Z_APDU_initResponse)
2351     {
2352         if (!m_server)  // if this is a pre init session , check for more
2353             m_root->pre_init();
2354         NMEM nmem = odr_extract_mem (odr_decode());
2355         odr_reset (m_init_odr);
2356         nmem_transfer (m_init_odr->mem, nmem);
2357         m_initResponse = apdu;
2358         m_initResponse_options = apdu->u.initResponse->options;
2359         m_initResponse_version = apdu->u.initResponse->protocolVersion;
2360
2361         Z_InitResponse *ir = apdu->u.initResponse;
2362         char *im0 = ir->implementationName;
2363
2364         char *im1 = (char*)
2365             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2366         *im1 = '\0';
2367         if (im0)
2368         {
2369             strcat(im1, im0);
2370             strcat(im1, " ");
2371         }
2372         strcat(im1, "(YAZ Proxy)");
2373         ir->implementationName = im1;
2374
2375         nmem_destroy (nmem);
2376
2377         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2378             return;
2379     }
2380     if (apdu->which == Z_APDU_searchResponse)
2381     {
2382         Z_SearchResponse *sr = apdu->u.searchResponse;
2383         m_last_resultCount = *sr->resultCount;
2384         int status = *sr->searchStatus;
2385         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2386         {
2387             m_last_ok = 1;
2388
2389             if (sr->records && sr->records->which == Z_Records_DBOSD)
2390             {
2391                 m_cache.add(odr_decode(),
2392                             sr->records->u.databaseOrSurDiagnostics, 1,
2393                             *sr->resultCount);
2394             }
2395         }
2396     }
2397     if (apdu->which == Z_APDU_presentResponse)
2398     {
2399         Z_PresentResponse *pr = apdu->u.presentResponse;
2400         if (m_sr_transform)
2401         {
2402             m_sr_transform = 0;
2403             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2404             Z_SearchResponse *sr = new_apdu->u.searchResponse;
2405             sr->referenceId = pr->referenceId;
2406             *sr->resultCount = m_last_resultCount;
2407             sr->records = pr->records;
2408             sr->nextResultSetPosition = pr->nextResultSetPosition;
2409             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2410             apdu = new_apdu;
2411         }
2412         if (pr->records &&
2413             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2414         {
2415             m_cache.add(odr_decode(),
2416                         pr->records->u.databaseOrSurDiagnostics,
2417                         m_resultSetStartPoint, -1);
2418             m_resultSetStartPoint = 0;
2419         }
2420     }
2421     if (m_cookie)
2422         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2423     if (m_server)
2424     {
2425         m_server->send_to_client(apdu);
2426     }
2427     if (apdu->which == Z_APDU_close)
2428     {
2429         shutdown();
2430     }
2431 }
2432
2433 int Yaz_Proxy::server(const char *addr)
2434 {
2435     int r = Yaz_Z_Assoc::server(addr);
2436     if (!r)
2437     {
2438         yaz_log(LOG_LOG, "%sStarted proxy " VERSION " on %s", m_session_str, addr);
2439         timeout(1);
2440     }
2441     return r;
2442 }
2443