src/yaz-proxy.cpp

   1 /*
   2  * Copyright (c) 1998-2004, Index Data.
   3  * See the file LICENSE for details.
   4  *
   5  * $Id: yaz-proxy.cpp,v 1.91 2004-01-15 23:44:58 adam Exp $
   6  */
   7
   8 #include <assert.h>
   9 #include <time.h>
  10
  11 #include <yaz/srw.h>
  12 #include <yaz/marcdisp.h>
  13 #include <yaz/yaz-iconv.h>
  14 #include <yaz/log.h>
  15 #include <yaz/diagbib1.h>
  16 #include <yaz++/proxy.h>
  17 #include <yaz/pquery.h>
  18
  19 #if HAVE_XSLT
  20 #include <libxslt/xsltutils.h>
  21 #include <libxslt/transform.h>
  22 #endif
  23
  24 static const char *apdu_name(Z_APDU *apdu)
  25 {
  26     switch (apdu->which)
  27     {
  28     case Z_APDU_initRequest:
  29         return "initRequest";
  30     case Z_APDU_initResponse:
  31         return "initResponse";
  32     case Z_APDU_searchRequest:
  33         return "searchRequest";
  34     case Z_APDU_searchResponse:
  35         return "searchResponse";
  36     case Z_APDU_presentRequest:
  37         return "presentRequest";
  38     case Z_APDU_presentResponse:
  39         return "presentResponse";
  40     case Z_APDU_deleteResultSetRequest:
  41         return "deleteResultSetRequest";
  42     case Z_APDU_deleteResultSetResponse:
  43         return "deleteResultSetResponse";
  44     case Z_APDU_scanRequest:
  45         return "scanRequest";
  46     case Z_APDU_scanResponse:
  47         return "scanResponse";
  48     case Z_APDU_sortRequest:
  49         return "sortRequest";
  50     case Z_APDU_sortResponse:
  51         return "sortResponse";
  52     case Z_APDU_extendedServicesRequest:
  53         return "extendedServicesRequest";
  54     case Z_APDU_extendedServicesResponse:
  55         return "extendedServicesResponse";
  56     case Z_APDU_close:
  57         return "close";
  58     }
  59     return "other";
  60 }
  61
  62 static const char *gdu_name(Z_GDU *gdu)
  63 {
  64     switch(gdu->which)
  65     {
  66     case Z_GDU_Z3950:
  67         return apdu_name(gdu->u.z3950);
  68     case Z_GDU_HTTP_Request:
  69         return "HTTP Request";
  70     case Z_GDU_HTTP_Response:
  71         return "HTTP Response";
  72     }
  73     return "Unknown request/response";
  74 }
  75
  76 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
  77                      Yaz_Proxy *parent) :
  78     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
  79 {
  80     m_PDU_Observable = the_PDU_Observable;
  81     m_client = 0;
  82     m_parent = parent;
  83     m_clientPool = 0;
  84     m_seqno = 1;
  85     m_keepalive_limit_bw = 500000;
  86     m_keepalive_limit_pdu = 1000;
  87     m_proxyTarget = 0;
  88     m_default_target = 0;
  89     m_proxy_authentication = 0;
  90     m_max_clients = 150;
  91     m_log_mask = 0;
  92     m_seed = time(0);
  93     m_client_idletime = 600;
  94     m_target_idletime = 600;
  95     m_optimize = xstrdup ("1");
  96     strcpy(m_session_str, "0 ");
  97     m_session_no=0;
  98     m_bytes_sent = m_bytes_recv = 0;
  99     m_bw_hold_PDU = 0;
 100     m_bw_max = 0;
 101     m_pdu_max = 0;
 102     m_max_record_retrieve = 0;
 103     m_reconfig_flag = 0;
 104     m_config_fname = 0;
 105     m_request_no = 0;
 106     m_invalid_session = 0;
 107     m_config = 0;
 108     m_marcxml_flag = 0;
 109     m_stylesheet_schema = 0;
 110     m_s2z_stylesheet = 0;
 111     m_s2z_database = 0;
 112     m_schema = 0;
 113     m_initRequest_apdu = 0;
 114     m_initRequest_mem = 0;
 115     m_initRequest_options = 0;
 116     m_initRequest_version = 0;
 117     m_apdu_invalid_session = 0;
 118     m_mem_invalid_session = 0;
 119     m_s2z_odr_init = 0;
 120     m_s2z_odr_search = 0;
 121     m_s2z_init_apdu = 0;
 122     m_s2z_search_apdu = 0;
 123     m_s2z_present_apdu = 0;
 124     m_http_keepalive = 0;
 125     m_http_version = 0;
 126     m_soap_ns = 0;
 127     m_s2z_packing = Z_SRW_recordPacking_string;
 128     m_time_tv.tv_sec = 0;
 129     m_time_tv.tv_usec = 0;
 130 }
 131
 132 Yaz_Proxy::~Yaz_Proxy()
 133 {
 134     yaz_log(LOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 135             m_bytes_sent, m_bytes_recv);
 136     nmem_destroy(m_initRequest_mem);
 137     nmem_destroy(m_mem_invalid_session);
 138     xfree (m_proxyTarget);
 139     xfree (m_default_target);
 140     xfree (m_proxy_authentication);
 141     xfree (m_optimize);
 142     xfree (m_stylesheet_schema);
 143     xfree (m_schema);
 144     if (m_s2z_odr_init)
 145         odr_destroy(m_s2z_odr_init);
 146     if (m_s2z_odr_search)
 147         odr_destroy(m_s2z_odr_search);
 148     delete m_config;
 149 }
 150
 151 int Yaz_Proxy::set_config(const char *config)
 152 {
 153     delete m_config;
 154     m_config = new Yaz_ProxyConfig();
 155     xfree(m_config_fname);
 156     m_config_fname = xstrdup(config);
 157     int r = m_config->read_xml(config);
 158     if (!r)
 159         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 160     return r;
 161 }
 162
 163 void Yaz_Proxy::set_default_target(const char *target)
 164 {
 165     xfree (m_default_target);
 166     m_default_target = 0;
 167     if (target)
 168         m_default_target = (char *) xstrdup (target);
 169 }
 170
 171 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 172 {
 173     xfree (m_proxy_authentication);
 174     m_proxy_authentication = 0;
 175     if (auth)
 176         m_proxy_authentication = (char *) xstrdup (auth);
 177 }
 178
 179 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 180 {
 181     if (m_parent)
 182         return m_parent->check_reconfigure();
 183
 184     Yaz_ProxyConfig *cfg = m_config;
 185     if (m_reconfig_flag)
 186     {
 187         yaz_log(LOG_LOG, "reconfigure");
 188         yaz_log_reopen();
 189         if (m_config_fname && cfg)
 190         {
 191             yaz_log(LOG_LOG, "reconfigure config %s", m_config_fname);
 192             int r = cfg->read_xml(m_config_fname);
 193             if (r)
 194                 yaz_log(LOG_WARN, "reconfigure failed");
 195             else
 196             {
 197                 m_log_mask = 0;
 198                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 199             }
 200         }
 201         else
 202             yaz_log(LOG_LOG, "reconfigure");
 203         m_reconfig_flag = 0;
 204     }
 205     return cfg;
 206 }
 207
 208 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 209                                             *the_PDU_Observable, int fd)
 210 {
 211     check_reconfigure();
 212     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 213     new_proxy->m_config = 0;
 214     new_proxy->m_config_fname = 0;
 215     new_proxy->timeout(m_client_idletime);
 216     new_proxy->m_target_idletime = m_target_idletime;
 217     new_proxy->set_default_target(m_default_target);
 218     new_proxy->m_max_clients = m_max_clients;
 219     new_proxy->m_log_mask = m_log_mask;
 220     new_proxy->set_APDU_log(get_APDU_log());
 221     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 222         new_proxy->set_APDU_yazlog(1);
 223     else
 224         new_proxy->set_APDU_yazlog(0);
 225     new_proxy->set_proxy_authentication(m_proxy_authentication);
 226     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 227     m_session_no++;
 228     yaz_log (LOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 229              the_PDU_Observable->getpeername());
 230     return new_proxy;
 231 }
 232
 233 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 234 {
 235     int oid[OID_SIZE];
 236     Z_OtherInformationUnit *oi;
 237     struct oident ent;
 238     ent.proto = PROTO_Z3950;
 239     ent.oclass = CLASS_USERINFO;
 240     ent.value = (oid_value) VAL_COOKIE;
 241     assert (oid_ent_to_oid (&ent, oid));
 242
 243     if (oid_ent_to_oid (&ent, oid) &&
 244         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 245         oi->which == Z_OtherInfo_characterInfo)
 246         return oi->information.characterInfo;
 247     return 0;
 248 }
 249
 250 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 251 {
 252     int oid[OID_SIZE];
 253     Z_OtherInformationUnit *oi;
 254     struct oident ent;
 255     ent.proto = PROTO_Z3950;
 256     ent.oclass = CLASS_USERINFO;
 257     ent.value = (oid_value) VAL_PROXY;
 258     if (oid_ent_to_oid (&ent, oid) &&
 259         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 260         oi->which == Z_OtherInfo_characterInfo)
 261         return oi->information.characterInfo;
 262     return 0;
 263 }
 264
 265 const char *Yaz_Proxy::load_balance(const char **url)
 266 {
 267     int zurl_in_use[MAX_ZURL_PLEX];
 268     Yaz_ProxyClient *c;
 269     int i;
 270
 271     for (i = 0; i<MAX_ZURL_PLEX; i++)
 272         zurl_in_use[i] = 0;
 273     for (c = m_parent->m_clientPool; c; c = c->m_next)
 274     {
 275         for (i = 0; url[i]; i++)
 276             if (!strcmp(url[i], c->get_hostname()))
 277                 zurl_in_use[i]++;
 278     }
 279     int min = 100000;
 280     const char *ret = 0;
 281     for (i = 0; url[i]; i++)
 282     {
 283         yaz_log(LOG_DEBUG, "%szurl=%s use=%d",
 284                 m_session_str, url[i], zurl_in_use[i]);
 285         if (min > zurl_in_use[i])
 286         {
 287             ret = url[i];
 288             min = zurl_in_use[i];
 289         }
 290     }
 291     return ret;
 292 }
 293
 294 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 295                                        const char *proxy_host)
 296 {
 297     assert (m_parent);
 298     Yaz_Proxy *parent = m_parent;
 299     Yaz_ProxyClient *c = m_client;
 300
 301     if (!m_proxyTarget)
 302     {
 303         const char *url[MAX_ZURL_PLEX];
 304         Yaz_ProxyConfig *cfg = check_reconfigure();
 305         if (proxy_host)
 306         {
 307 #if 1
 308 /* only to be enabled for debugging... */
 309             if (!strcmp(proxy_host, "stop"))
 310                 exit(0);
 311 #endif
 312             xfree(m_default_target);
 313             m_default_target = xstrdup(proxy_host);
 314             proxy_host = m_default_target;
 315         }
 316         int client_idletime = -1;
 317         const char *cql2rpn_fname = 0;
 318         url[0] = m_default_target;
 319         url[1] = 0;
 320         if (cfg)
 321         {
 322             int pre_init = 0;
 323             cfg->get_target_info(proxy_host, url, &m_bw_max,
 324                                  &m_pdu_max, &m_max_record_retrieve,
 325                                  &m_target_idletime, &client_idletime,
 326                                  &parent->m_max_clients,
 327                                  &m_keepalive_limit_bw,
 328                                  &m_keepalive_limit_pdu,
 329                                  &pre_init,
 330                                  &cql2rpn_fname);
 331         }
 332         if (client_idletime != -1)
 333         {
 334             m_client_idletime = client_idletime;
 335             timeout(m_client_idletime);
 336         }
 337         if (cql2rpn_fname)
 338             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 339         if (!url[0])
 340         {
 341             yaz_log(LOG_LOG, "%sNo default target", m_session_str);
 342             return 0;
 343         }
 344         // we don't handle multiplexing for cookie session, so we just
 345         // pick the first one in this case (anonymous users will be able
 346         // to use any backend)
 347         if (cookie && *cookie)
 348             m_proxyTarget = (char*) xstrdup(url[0]);
 349         else
 350             m_proxyTarget = (char*) xstrdup(load_balance(url));
 351     }
 352     if (cookie && *cookie)
 353     {   // search in sessions with a cookie
 354         for (c = parent->m_clientPool; c; c = c->m_next)
 355         {
 356             assert (c->m_prev);
 357             assert (*c->m_prev == c);
 358             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 359                 !strcmp(m_proxyTarget, c->get_hostname()))
 360             {
 361                 // Found it in cache
 362                 // The following handles "cancel"
 363                 // If connection is busy (waiting for PDU) and
 364                 // we have an initRequest we can safely do re-open
 365                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 366                 {
 367                     yaz_log (LOG_LOG, "%s REOPEN target=%s", m_session_str,
 368                              c->get_hostname());
 369                     c->close();
 370                     c->m_init_flag = 0;
 371
 372                     c->m_last_ok = 0;
 373                     c->m_cache.clear();
 374                     c->m_last_resultCount = 0;
 375                     c->m_sr_transform = 0;
 376                     c->m_waiting = 0;
 377                     c->m_resultSetStartPoint = 0;
 378                     c->m_target_idletime = m_target_idletime;
 379                     if (c->client(m_proxyTarget))
 380                     {
 381                         delete c;
 382                         return 0;
 383                     }
 384                     c->timeout(30);
 385                 }
 386                 c->m_seqno = parent->m_seqno;
 387                 if (c->m_server && c->m_server != this)
 388                     c->m_server->m_client = 0;
 389                 c->m_server = this;
 390                 (parent->m_seqno)++;
 391                 yaz_log (LOG_DEBUG, "get_client 1 %p %p", this, c);
 392                 return c;
 393             }
 394         }
 395     }
 396     else if (!c)
 397     {
 398         // don't have a client session yet. Search in session w/o cookie
 399         for (c = parent->m_clientPool; c; c = c->m_next)
 400         {
 401             assert (c->m_prev);
 402             assert (*c->m_prev == c);
 403             if (c->m_server == 0 && c->m_cookie == 0 &&
 404                 c->m_waiting == 0 &&
 405                 !strcmp(m_proxyTarget, c->get_hostname()))
 406             {
 407                 // found it in cache
 408                 yaz_log (LOG_LOG, "%sREUSE %s",
 409                          m_session_str, c->get_hostname());
 410
 411                 c->m_seqno = parent->m_seqno;
 412                 assert(c->m_server == 0);
 413                 c->m_server = this;
 414
 415                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 416                     c->set_APDU_yazlog(1);
 417                 else
 418                     c->set_APDU_yazlog(0);
 419
 420                 (parent->m_seqno)++;
 421
 422                 parent->pre_init();
 423
 424                 return c;
 425             }
 426         }
 427     }
 428     if (!m_client)
 429     {
 430         if (apdu->which != Z_APDU_initRequest)
 431         {
 432             yaz_log (LOG_LOG, "%sno init request as first PDU", m_session_str);
 433             return 0;
 434         }
 435         Z_InitRequest *initRequest = apdu->u.initRequest;
 436
 437         if (!initRequest->idAuthentication)
 438         {
 439             if (m_proxy_authentication)
 440             {
 441                 initRequest->idAuthentication =
 442                     (Z_IdAuthentication *)
 443                     odr_malloc (odr_encode(),
 444                                 sizeof(*initRequest->idAuthentication));
 445                 initRequest->idAuthentication->which =
 446                     Z_IdAuthentication_open;
 447                 initRequest->idAuthentication->u.open =
 448                     odr_strdup (odr_encode(), m_proxy_authentication);
 449             }
 450         }
 451         // go through list of clients - and find the lowest/oldest one.
 452         Yaz_ProxyClient *c_min = 0;
 453         int min_seq = -1;
 454         int no_of_clients = 0;
 455         if (parent->m_clientPool)
 456             yaz_log (LOG_DEBUG, "Existing sessions");
 457         for (c = parent->m_clientPool; c; c = c->m_next)
 458         {
 459             yaz_log (LOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 460                                c->m_waiting, c->get_hostname(),
 461                                c->m_cookie ? c->m_cookie : "");
 462             no_of_clients++;
 463             if (min_seq < 0 || c->m_seqno < min_seq)
 464             {
 465                 min_seq = c->m_seqno;
 466                 c_min = c;
 467             }
 468         }
 469         if (no_of_clients >= parent->m_max_clients)
 470         {
 471             c = c_min;
 472             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 473             {
 474                 yaz_log (LOG_LOG, "%sMAXCLIENTS Destroy %d",
 475                          m_session_str, c->m_seqno);
 476                 if (c->m_server && c->m_server != this)
 477                     delete c->m_server;
 478                 c->m_server = 0;
 479             }
 480             else
 481             {
 482                 yaz_log (LOG_LOG, "%sMAXCLIENTS Reuse %d %d %s",
 483                          m_session_str,
 484                          c->m_seqno, parent->m_seqno, c->get_hostname());
 485                 xfree (c->m_cookie);
 486                 c->m_cookie = 0;
 487                 if (cookie)
 488                     c->m_cookie = xstrdup(cookie);
 489                 c->m_seqno = parent->m_seqno;
 490                 if (c->m_server && c->m_server != this)
 491                 {
 492                     c->m_server->m_client = 0;
 493                     delete c->m_server;
 494                 }
 495                 (parent->m_seqno)++;
 496                 c->m_target_idletime = m_target_idletime;
 497                 c->timeout(m_target_idletime);
 498
 499                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 500                     c->set_APDU_yazlog(1);
 501                 else
 502                     c->set_APDU_yazlog(0);
 503
 504                 return c;
 505             }
 506         }
 507         else
 508         {
 509             yaz_log (LOG_LOG, "%sNEW %d %s",
 510                      m_session_str, parent->m_seqno, m_proxyTarget);
 511             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 512             c->m_next = parent->m_clientPool;
 513             if (c->m_next)
 514                 c->m_next->m_prev = &c->m_next;
 515             parent->m_clientPool = c;
 516             c->m_prev = &parent->m_clientPool;
 517         }
 518
 519         xfree (c->m_cookie);
 520         c->m_cookie = 0;
 521         if (cookie)
 522             c->m_cookie = xstrdup(cookie);
 523
 524         c->m_seqno = parent->m_seqno;
 525         c->m_init_flag = 0;
 526         c->m_last_resultCount = 0;
 527         c->m_last_ok = 0;
 528         c->m_cache.clear();
 529         c->m_sr_transform = 0;
 530         c->m_waiting = 0;
 531         c->m_resultSetStartPoint = 0;
 532         (parent->m_seqno)++;
 533         if (c->client(m_proxyTarget))
 534         {
 535             delete c;
 536             return 0;
 537         }
 538         c->m_target_idletime = m_target_idletime;
 539         c->timeout(30);
 540
 541         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 542             c->set_APDU_yazlog(1);
 543         else
 544             c->set_APDU_yazlog(0);
 545     }
 546     yaz_log (LOG_DEBUG, "get_client 3 %p %p", this, c);
 547     return c;
 548 }
 549
 550 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 551 {
 552     int i;
 553     for (i = 0; i<num; i++)
 554     {
 555         oident *ent;
 556         Z_DefaultDiagFormat *r;
 557         Z_DiagRec *p = pp[i];
 558         if (p->which != Z_DiagRec_defaultFormat)
 559         {
 560             yaz_log(LOG_LOG, "%sError no diagnostics", m_session_str);
 561             return;
 562         }
 563         else
 564             r = p->u.defaultFormat;
 565         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 566             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 567             yaz_log(LOG_LOG, "%sError unknown diagnostic set", m_session_str);
 568         switch (r->which)
 569         {
 570         case Z_DefaultDiagFormat_v2Addinfo:
 571             yaz_log(LOG_LOG, "%sError %d %s:%s",
 572                     m_session_str,
 573                     *r->condition, diagbib1_str(*r->condition),
 574                     r->u.v2Addinfo);
 575             break;
 576         case Z_DefaultDiagFormat_v3Addinfo:
 577             yaz_log(LOG_LOG, "%sError %d %s:%s",
 578                     m_session_str,
 579                     *r->condition, diagbib1_str(*r->condition),
 580                     r->u.v3Addinfo);
 581             break;
 582         }
 583     }
 584 }
 585
 586 void Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p)
 587 {
 588     if (!m_stylesheet_schema)
 589         return;
 590     xsltStylesheetPtr xsp;
 591
 592     xsp = xsltParseStylesheetFile((const xmlChar *) m_stylesheet_schema);
 593
 594     int i;
 595     for (i = 0; i < p->num_records; i++)
 596     {
 597         Z_NamePlusRecord *npr = p->records[i];
 598         if (npr->which == Z_NamePlusRecord_databaseRecord)
 599         {
 600             Z_External *r = npr->u.databaseRecord;
 601             if (r->which == Z_External_octet)
 602             {
 603                 xmlDocPtr res, doc = xmlParseMemory(
 604                     (char*) r->u.octet_aligned->buf,
 605                     r->u.octet_aligned->len);
 606
 607                 res = xsltApplyStylesheet(xsp, doc, 0);
 608
 609                 xmlChar *out_buf;
 610                 int out_len;
 611                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 612
 613                 p->records[i]->u.databaseRecord =
 614                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 615                                  (char*) out_buf, out_len);
 616                 xmlFree(out_buf);
 617                 xmlFreeDoc(doc);
 618                 xmlFreeDoc(res);
 619             }
 620         }
 621     }
 622     xsltFreeStylesheet(xsp);
 623 }
 624
 625 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p)
 626 {
 627     int i;
 628
 629     yaz_marc_t mt = yaz_marc_create();
 630     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 631     for (i = 0; i < p->num_records; i++)
 632     {
 633         Z_NamePlusRecord *npr = p->records[i];
 634         if (npr->which == Z_NamePlusRecord_databaseRecord)
 635         {
 636             Z_External *r = npr->u.databaseRecord;
 637             if (r->which == Z_External_octet)
 638             {
 639                 int rlen;
 640                 char *result;
 641                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 642                                         r->u.octet_aligned->len,
 643                                         &result, &rlen))
 644                 {
 645                     yaz_iconv_t cd = yaz_iconv_open("UTF-8", "MARC-8");
 646                     WRBUF wrbuf = wrbuf_alloc();
 647
 648                     char outbuf[120];
 649                     size_t inbytesleft = rlen;
 650                     const char *inp = result;
 651                     while (cd && inbytesleft)
 652                     {
 653                         size_t outbytesleft = sizeof(outbuf);
 654                         char *outp = outbuf;
 655                         size_t r;
 656
 657                         r = yaz_iconv (cd, (char**) &inp,
 658                                        &inbytesleft,
 659                                        &outp, &outbytesleft);
 660                         if (r == (size_t) (-1))
 661                         {
 662                             int e = yaz_iconv_error(cd);
 663                             if (e != YAZ_ICONV_E2BIG)
 664                             {
 665                                 yaz_log(LOG_WARN, "conversion failure");
 666                                 break;
 667                             }
 668                         }
 669                         wrbuf_write(wrbuf, outbuf, outp - outbuf);
 670                     }
 671                     if (cd)
 672                         yaz_iconv_close(cd);
 673
 674                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 675                                                          VAL_TEXT_XML,
 676                                                          wrbuf_buf(wrbuf),
 677                                                          wrbuf_len(wrbuf));
 678                     wrbuf_free(wrbuf, 1);
 679                 }
 680             }
 681         }
 682     }
 683     yaz_marc_destroy(mt);
 684 }
 685
 686 void Yaz_Proxy::logtime()
 687 {
 688     if (m_time_tv.tv_sec)
 689     {
 690         struct timeval tv;
 691         gettimeofday(&tv, 0);
 692         long diff = (tv.tv_sec - m_time_tv.tv_sec)*1000000 +
 693             (tv.tv_usec - m_time_tv.tv_usec);
 694         if (diff >= 0)
 695             yaz_log(LOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 696                     diff/1000000, (diff/1000)%1000);
 697     }
 698     m_time_tv.tv_sec = 0;
 699     m_time_tv.tv_usec = 0;
 700 }
 701
 702 int Yaz_Proxy::send_http_response(int code)
 703 {
 704     ODR o = odr_encode();
 705     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 706     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 707     if (m_http_version)
 708         hres->version = odr_strdup(o, m_http_version);
 709     m_http_keepalive = 0;
 710     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 711     {
 712         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 713                  gdu_name(gdu));
 714     }
 715     int len;
 716     int r = send_GDU(gdu, &len);
 717     logtime();
 718     return r;
 719 }
 720
 721 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 722 {
 723     ODR o = odr_encode();
 724     const char *ctype = "text/xml";
 725     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 726     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 727     if (m_http_version)
 728         hres->version = odr_strdup(o, m_http_version);
 729     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 730     if (m_http_keepalive)
 731         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 732
 733     static Z_SOAP_Handler soap_handlers[2] = {
 734 #if HAVE_XSLT
 735         {"http://www.loc.gov/zing/srw/", 0,
 736          (Z_SOAP_fun) yaz_srw_codec},
 737 #endif
 738         {0, 0, 0}
 739     };
 740
 741     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 742     soap_package->which = Z_SOAP_generic;
 743     soap_package->u.generic =
 744         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 745     soap_package->u.generic->no = 0;
 746     soap_package->u.generic->ns = soap_handlers[0].ns;
 747     soap_package->u.generic->p = (void *) srw_pdu;
 748     soap_package->ns = m_soap_ns;
 749     z_soap_codec_enc_xsl(o, &soap_package,
 750                          &hres->content_buf, &hres->content_len,
 751                          soap_handlers, 0, m_s2z_stylesheet);
 752     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 753     {
 754         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 755                  gdu_name(gdu));
 756     }
 757     int len;
 758     int r = send_GDU(gdu, &len);
 759     logtime();
 760     return r;
 761 }
 762
 763 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
 764 {
 765     ODR o = odr_encode();
 766     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 767     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 768
 769     srw_res->num_diagnostics = 1;
 770     srw_res->diagnostics = (Z_SRW_diagnostic *)
 771         odr_malloc(o, sizeof(*srw_res->diagnostics));
 772     srw_res->diagnostics[0].code =  odr_intdup(o, srw_error);
 773     srw_res->diagnostics[0].details = add ? odr_strdup(o, add) : 0;
 774     return send_srw_response(srw_pdu);
 775 }
 776
 777 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 778                              Z_DefaultDiagFormat *ddf)
 779 {
 780     int bib1_code = *ddf->condition;
 781     if (bib1_code == 109)
 782         return 404;
 783     srw_res->num_diagnostics = 1;
 784     srw_res->diagnostics = (Z_SRW_diagnostic *)
 785         odr_malloc(o, sizeof(*srw_res->diagnostics));
 786     srw_res->diagnostics[0].code =
 787         odr_intdup(o, yaz_diag_bib1_to_srw(*ddf->condition));
 788     srw_res->diagnostics[0].details = ddf->u.v2Addinfo;
 789     return 0;
 790 }
 791
 792 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
 793 {
 794     ODR o = odr_encode();
 795     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 796     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 797
 798     srw_res->numberOfRecords = odr_intdup (o, hits);
 799     if (records && records->which == Z_Records_DBOSD)
 800     {
 801         srw_res->num_records =
 802             records->u.databaseOrSurDiagnostics->num_records;
 803         int i;
 804         srw_res->records = (Z_SRW_record *)
 805             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
 806         for (i = 0; i < srw_res->num_records; i++)
 807         {
 808             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
 809             if (npr->which != Z_NamePlusRecord_databaseRecord)
 810             {
 811                 srw_res->records[i].recordSchema = "diagnostic";
 812                 srw_res->records[i].recordPacking = m_s2z_packing;
 813                 srw_res->records[i].recordData_buf = "67";
 814                 srw_res->records[i].recordData_len = 2;
 815                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 816                 continue;
 817             }
 818             Z_External *r = npr->u.databaseRecord;
 819             oident *ent = oid_getentbyoid(r->direct_reference);
 820             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
 821             {
 822                 srw_res->records[i].recordSchema = m_schema;
 823                 srw_res->records[i].recordPacking = m_s2z_packing;
 824                 srw_res->records[i].recordData_buf = (char*)
 825                     r->u.octet_aligned->buf;
 826                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
 827                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 828             }
 829             else
 830             {
 831                 srw_res->records[i].recordSchema = "diagnostic";
 832                 srw_res->records[i].recordPacking = m_s2z_packing;
 833                 srw_res->records[i].recordData_buf = "67";
 834                 srw_res->records[i].recordData_len = 2;
 835                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 836             }
 837         }
 838     }
 839     if (records && records->which == Z_Records_NSD)
 840     {
 841         int http_code;
 842         http_code = z_to_srw_diag(odr_encode(), srw_res,
 843                                    records->u.nonSurrogateDiagnostic);
 844         if (http_code)
 845             return send_http_response(http_code);
 846     }
 847     return send_srw_response(srw_pdu);
 848
 849 }
 850
 851 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
 852                                         int num_diagnostics)
 853 {
 854     Yaz_ProxyConfig *cfg = check_reconfigure();
 855     if (cfg)
 856     {
 857         int len;
 858         char *b = cfg->get_explain(odr_encode(), 0 /* target */,
 859                                    m_s2z_database, &len);
 860         if (b)
 861         {
 862             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
 863             Z_SRW_explainResponse *er = res->u.explain_response;
 864
 865             er->record.recordData_buf = b;
 866             er->record.recordData_len = len;
 867             er->record.recordPacking = m_s2z_packing;
 868
 869             er->diagnostics = diagnostics;
 870             er->num_diagnostics = num_diagnostics;
 871             return send_srw_response(res);
 872         }
 873     }
 874     return send_http_response(404);
 875 }
 876
 877 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu, int *len)
 878 {
 879     if (m_http_version)
 880     {
 881         if (apdu->which == Z_APDU_initResponse)
 882         {
 883             Z_InitResponse *res = apdu->u.initResponse;
 884             if (*res->result == 0)
 885             {
 886                 send_to_srw_client_error(3, 0);
 887             }
 888             else if (!m_s2z_search_apdu)
 889             {
 890                 send_srw_explain_response(0, 0);
 891             }
 892             else
 893             {
 894                 handle_incoming_Z_PDU(m_s2z_search_apdu);
 895             }
 896         }
 897         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
 898         {
 899             m_s2z_search_apdu = 0;
 900             Z_SearchResponse *res = apdu->u.searchResponse;
 901             m_s2z_hit_count = *res->resultCount;
 902             if (res->records && res->records->which == Z_Records_NSD)
 903             {
 904                 send_to_srw_client_ok(0, res->records, 1);
 905             }
 906             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
 907             {
 908                 // adjust
 909                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
 910
 911                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
 912                 {
 913                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
 914                         > m_s2z_hit_count)
 915                         *pr->numberOfRecordsRequested =
 916                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
 917                 }
 918                 handle_incoming_Z_PDU(m_s2z_present_apdu);
 919             }
 920             else
 921             {
 922                 m_s2z_present_apdu = 0;
 923                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
 924             }
 925         }
 926         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
 927         {
 928             int start =
 929                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
 930
 931             m_s2z_present_apdu = 0;
 932             Z_PresentResponse *res = apdu->u.presentResponse;
 933             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
 934         }
 935     }
 936     else
 937     {
 938         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 939             yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 940                      apdu_name(apdu));
 941         int r = send_Z_PDU(apdu, len);
 942         logtime();
 943         return r;
 944     }
 945     return 0;
 946 }
 947
 948 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 949 {
 950     int len = 0;
 951     int kill_session = 0;
 952     if (apdu->which == Z_APDU_searchResponse)
 953     {
 954         Z_SearchResponse *sr = apdu->u.searchResponse;
 955         Z_Records *p = sr->records;
 956         if (p && p->which == Z_Records_NSD)
 957         {
 958             Z_DiagRec dr, *dr_p = &dr;
 959             dr.which = Z_DiagRec_defaultFormat;
 960             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 961
 962             *sr->searchStatus = 0;
 963             display_diagrecs(&dr_p, 1);
 964         }
 965         else
 966         {
 967             if (p && p->which == Z_Records_DBOSD)
 968             {
 969                 if (m_marcxml_flag)
 970                     convert_to_marcxml(p->u.databaseOrSurDiagnostics);
 971                 convert_xsl(p->u.databaseOrSurDiagnostics);
 972             }
 973             if (sr->resultCount)
 974             {
 975                 yaz_log(LOG_LOG, "%s%d hits", m_session_str,
 976                         *sr->resultCount);
 977                 if (*sr->resultCount < 0)
 978                 {
 979                     m_invalid_session = 1;
 980                     kill_session = 1;
 981
 982                     *sr->searchStatus = 0;
 983                     sr->records =
 984                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
 985                     *sr->resultCount = 0;
 986                 }
 987             }
 988         }
 989     }
 990     else if (apdu->which == Z_APDU_presentResponse)
 991     {
 992         Z_PresentResponse *sr = apdu->u.presentResponse;
 993         Z_Records *p = sr->records;
 994         if (p && p->which == Z_Records_NSD)
 995         {
 996             Z_DiagRec dr, *dr_p = &dr;
 997             dr.which = Z_DiagRec_defaultFormat;
 998             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 999             if (*sr->presentStatus == Z_PresentStatus_success)
1000                 *sr->presentStatus = Z_PresentStatus_failure;
1001             display_diagrecs(&dr_p, 1);
1002         }
1003         if (p && p->which == Z_Records_DBOSD)
1004         {
1005             if (m_marcxml_flag)
1006                 convert_to_marcxml(p->u.databaseOrSurDiagnostics);
1007             convert_xsl(p->u.databaseOrSurDiagnostics);
1008         }
1009     }
1010     else if (apdu->which == Z_APDU_initResponse)
1011     {
1012         if (m_initRequest_options)
1013         {
1014             Z_Options *nopt =
1015                 (Odr_bitmask *)odr_malloc(odr_encode(),
1016                                           sizeof(Odr_bitmask));
1017             ODR_MASK_ZERO(nopt);
1018
1019             int i;
1020             for (i = 0; i<24; i++)
1021                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1022                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1023                     ODR_MASK_SET(nopt, i);
1024             apdu->u.initResponse->options = nopt;
1025         }
1026         if (m_initRequest_version)
1027         {
1028             Z_ProtocolVersion *nopt =
1029                 (Odr_bitmask *)odr_malloc(odr_encode(),
1030                                           sizeof(Odr_bitmask));
1031             ODR_MASK_ZERO(nopt);
1032
1033             int i;
1034             for (i = 0; i<8; i++)
1035                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1036                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1037                     ODR_MASK_SET(nopt, i);
1038             apdu->u.initResponse->protocolVersion = nopt;
1039         }
1040     }
1041     int r = send_PDU_convert(apdu, &len);
1042     if (r)
1043         return r;
1044     m_bytes_sent += len;
1045     m_bw_stat.add_bytes(len);
1046     if (kill_session)
1047     {
1048         delete m_client;
1049         m_client = 0;
1050         m_parent->pre_init();
1051     }
1052     if (m_http_version)
1053     {
1054         if (!m_http_keepalive)
1055         {
1056 #if 1
1057             timeout(1);
1058 #else
1059             shutdown();
1060             return -1;
1061 #endif
1062         }
1063     }
1064     return r;
1065 }
1066
1067 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1068 {
1069     int len = 0;
1070     const char *apdu_name_tmp = apdu_name(apdu);
1071     int r = send_Z_PDU(apdu, &len);
1072     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1073         yaz_log (LOG_LOG, "%sSending %s to %s %d bytes",
1074                  get_session_str(),
1075                  apdu_name_tmp, get_hostname(), len);
1076     m_bytes_sent += len;
1077     return r;
1078 }
1079
1080 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1081 {
1082     if (apdu->which == Z_APDU_presentRequest)
1083     {
1084         Z_PresentRequest *pr = apdu->u.presentRequest;
1085         int toget = *pr->numberOfRecordsRequested;
1086         int start = *pr->resultSetStartPoint;
1087
1088         yaz_log(LOG_LOG, "%sPresent %s %d+%d", m_session_str,
1089                 pr->resultSetId, start, toget);
1090
1091         if (*m_parent->m_optimize == '0')
1092             return apdu;
1093
1094         if (!m_client->m_last_resultSetId)
1095         {
1096             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1097             new_apdu->u.presentResponse->records =
1098                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1099                                                pr->resultSetId);
1100             send_to_client(new_apdu);
1101             return 0;
1102         }
1103         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1104         {
1105             if (start+toget-1 > m_client->m_last_resultCount)
1106             {
1107                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1108                 new_apdu->u.presentResponse->records =
1109                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1110                 send_to_client(new_apdu);
1111                 return 0;
1112             }
1113             Z_NamePlusRecordList *npr;
1114             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1115                                           pr->preferredRecordSyntax,
1116                                           pr->recordComposition))
1117             {
1118                 yaz_log (LOG_LOG, "%sReturned cached records for present request",
1119                          m_session_str);
1120                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1121                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1122
1123                 new_apdu->u.presentResponse->numberOfRecordsReturned
1124                     = odr_intdup(odr_encode(), toget);
1125
1126                 new_apdu->u.presentResponse->records = (Z_Records*)
1127                     odr_malloc(odr_encode(), sizeof(Z_Records));
1128                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1129                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1130                 new_apdu->u.presentResponse->nextResultSetPosition =
1131                     odr_intdup(odr_encode(), start+toget);
1132
1133                 send_to_client(new_apdu);
1134                 return 0;
1135             }
1136         }
1137     }
1138
1139     if (apdu->which != Z_APDU_searchRequest)
1140         return apdu;
1141     Z_SearchRequest *sr = apdu->u.searchRequest;
1142     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1143     Yaz_Z_Databases this_databases;
1144
1145     this_databases.set(sr->num_databaseNames, (const char **)
1146                        sr->databaseNames);
1147
1148     this_query->set_Z_Query(sr->query);
1149
1150     char query_str[120];
1151     this_query->print(query_str, sizeof(query_str)-1);
1152     yaz_log(LOG_LOG, "%sSearch %s", m_session_str, query_str);
1153
1154     if (*m_parent->m_optimize != '0' &&
1155         m_client->m_last_ok && m_client->m_last_query &&
1156         m_client->m_last_query->match(this_query) &&
1157         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1158         m_client->m_last_databases.match(this_databases))
1159     {
1160         delete this_query;
1161         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1162             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1163         {
1164             Z_NamePlusRecordList *npr;
1165             int toget = *sr->mediumSetPresentNumber;
1166             Z_RecordComposition *comp = 0;
1167
1168             if (toget > m_client->m_last_resultCount)
1169                 toget = m_client->m_last_resultCount;
1170
1171             if (sr->mediumSetElementSetNames)
1172             {
1173                 comp = (Z_RecordComposition *)
1174                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1175                 comp->which = Z_RecordComp_simple;
1176                 comp->u.simple = sr->mediumSetElementSetNames;
1177             }
1178
1179             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1180                                           sr->preferredRecordSyntax, comp))
1181             {
1182                 yaz_log (LOG_LOG, "%sReturned cached records for medium set",
1183                          m_session_str);
1184                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1185                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1186                 new_apdu->u.searchResponse->resultCount =
1187                     &m_client->m_last_resultCount;
1188
1189                 new_apdu->u.searchResponse->numberOfRecordsReturned
1190                     = odr_intdup(odr_encode(), toget);
1191
1192                 new_apdu->u.searchResponse->presentStatus =
1193                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1194                 new_apdu->u.searchResponse->records = (Z_Records*)
1195                     odr_malloc(odr_encode(), sizeof(Z_Records));
1196                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1197                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1198                 new_apdu->u.searchResponse->nextResultSetPosition =
1199                     odr_intdup(odr_encode(), toget+1);
1200                 send_to_client(new_apdu);
1201                 return 0;
1202             }
1203             else
1204             {
1205                 // medium Set
1206                 // send present request (medium size)
1207                 yaz_log (LOG_LOG, "%sOptimizing search for medium set",
1208                          m_session_str);
1209
1210                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1211                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1212                 pr->referenceId = sr->referenceId;
1213                 pr->resultSetId = sr->resultSetName;
1214                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1215                 *pr->numberOfRecordsRequested = toget;
1216                 pr->recordComposition = comp;
1217                 m_client->m_sr_transform = 1;
1218                 return new_apdu;
1219             }
1220         }
1221         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1222             m_client->m_last_resultCount <= 0)
1223         {
1224             // large set. Return pseudo-search response immediately
1225             yaz_log (LOG_LOG, "%sOptimizing search for large set",
1226                      m_session_str);
1227             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1228             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1229             new_apdu->u.searchResponse->resultCount =
1230                 &m_client->m_last_resultCount;
1231             send_to_client(new_apdu);
1232             return 0;
1233         }
1234         else
1235         {
1236             Z_NamePlusRecordList *npr;
1237             int toget = m_client->m_last_resultCount;
1238             Z_RecordComposition *comp = 0;
1239             // small set
1240             // send a present request (small set)
1241
1242             if (sr->smallSetElementSetNames)
1243             {
1244                 comp = (Z_RecordComposition *)
1245                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1246                 comp->which = Z_RecordComp_simple;
1247                 comp->u.simple = sr->smallSetElementSetNames;
1248             }
1249
1250             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1251                                           sr->preferredRecordSyntax, comp))
1252             {
1253                 yaz_log (LOG_LOG, "%sReturned cached records for small set",
1254                          m_session_str);
1255                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1256                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1257                 new_apdu->u.searchResponse->resultCount =
1258                     &m_client->m_last_resultCount;
1259
1260                 new_apdu->u.searchResponse->numberOfRecordsReturned
1261                     = odr_intdup(odr_encode(), toget);
1262
1263                 new_apdu->u.searchResponse->presentStatus =
1264                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1265                 new_apdu->u.searchResponse->records = (Z_Records*)
1266                     odr_malloc(odr_encode(), sizeof(Z_Records));
1267                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1268                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1269                 new_apdu->u.searchResponse->nextResultSetPosition =
1270                     odr_intdup(odr_encode(), toget+1);
1271                 send_to_client(new_apdu);
1272                 return 0;
1273             }
1274             else
1275             {
1276                 yaz_log (LOG_LOG, "%sOptimizing search for small set",
1277                          m_session_str);
1278                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1279                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1280                 pr->referenceId = sr->referenceId;
1281                 pr->resultSetId = sr->resultSetName;
1282                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1283                 *pr->numberOfRecordsRequested = toget;
1284                 pr->recordComposition = comp;
1285                 m_client->m_sr_transform = 1;
1286                 return new_apdu;
1287             }
1288         }
1289     }
1290     else  // query doesn't match
1291     {
1292         delete m_client->m_last_query;
1293         m_client->m_last_query = this_query;
1294         m_client->m_last_ok = 0;
1295         m_client->m_cache.clear();
1296         m_client->m_resultSetStartPoint = 0;
1297
1298         xfree (m_client->m_last_resultSetId);
1299         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1300
1301         m_client->m_last_databases.set(sr->num_databaseNames,
1302                                        (const char **) sr->databaseNames);
1303     }
1304     return apdu;
1305 }
1306
1307
1308 void Yaz_Proxy::inc_request_no()
1309 {
1310     char *cp = strchr(m_session_str, ' ');
1311     m_request_no++;
1312     if (cp)
1313         sprintf(cp+1, "%d ", m_request_no);
1314 }
1315
1316 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1317 {
1318     inc_request_no();
1319
1320     m_bytes_recv += len;
1321
1322     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
1323         yaz_log (LOG_DEBUG, "%sReceiving %s from client %d bytes",
1324                  m_session_str, gdu_name(apdu), len);
1325
1326     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1327         shutdown();
1328
1329     m_bw_stat.add_bytes(len);
1330     m_pdu_stat.add_bytes(1);
1331
1332     gettimeofday(&m_time_tv, 0);
1333
1334     int bw_total = m_bw_stat.get_total();
1335     int pdu_total = m_pdu_stat.get_total();
1336
1337     int reduce = 0;
1338     if (m_bw_max)
1339     {
1340         if (bw_total > m_bw_max)
1341         {
1342             reduce = (bw_total/m_bw_max);
1343         }
1344     }
1345     if (m_pdu_max)
1346     {
1347         if (pdu_total > m_pdu_max)
1348         {
1349             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1350             reduce = (reduce > nreduce) ? reduce : nreduce;
1351         }
1352     }
1353     if (reduce)
1354     {
1355         yaz_log(LOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1356                 m_session_str, reduce, bw_total, pdu_total,
1357                 m_bw_max, m_pdu_max);
1358
1359         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1360         timeout(reduce);       // call us reduce seconds later
1361     }
1362     else if (apdu->which == Z_GDU_Z3950)
1363         handle_incoming_Z_PDU(apdu->u.z3950);
1364     else if (apdu->which == Z_GDU_HTTP_Request)
1365         handle_incoming_HTTP(apdu->u.HTTP_Request);
1366 }
1367
1368 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1369 {
1370     if (m_max_record_retrieve)
1371     {
1372         if (apdu->which == Z_APDU_presentRequest)
1373         {
1374             Z_PresentRequest *pr = apdu->u.presentRequest;
1375             if (pr->numberOfRecordsRequested &&
1376                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1377                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1378         }
1379     }
1380 }
1381
1382 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1383                                                      int error,
1384                                                      const char *addinfo)
1385 {
1386     Z_Records *rec = (Z_Records *)
1387         odr_malloc (odr, sizeof(*rec));
1388     int *err = (int *)
1389         odr_malloc (odr, sizeof(*err));
1390     Z_DiagRec *drec = (Z_DiagRec *)
1391         odr_malloc (odr, sizeof(*drec));
1392     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1393         odr_malloc (odr, sizeof(*dr));
1394     *err = error;
1395     rec->which = Z_Records_NSD;
1396     rec->u.nonSurrogateDiagnostic = dr;
1397     dr->diagnosticSetId =
1398         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1399     dr->condition = err;
1400     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1401     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1402     return rec;
1403 }
1404
1405 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1406 {
1407     if (apdu->which == Z_APDU_searchRequest &&
1408         apdu->u.searchRequest->query &&
1409         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1410         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1411     {
1412         Z_RPNQuery *rpnquery = 0;
1413         Z_SearchRequest *sr = apdu->u.searchRequest;
1414         char *addinfo = 0;
1415
1416         yaz_log(LOG_LOG, "%sCQL: %s", m_session_str,
1417                 sr->query->u.type_104->u.cql);
1418
1419         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1420                                           &rpnquery, odr_encode(),
1421                                           &addinfo);
1422         if (r == -3)
1423             yaz_log(LOG_LOG, "%sNo CQL to RPN table", m_session_str);
1424         else if (r)
1425         {
1426             yaz_log(LOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1427             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1428
1429             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1430             new_apdu->u.searchResponse->records =
1431                 create_nonSurrogateDiagnostics(odr_encode(),
1432                                                yaz_diag_srw_to_bib1(r),
1433                                                addinfo);
1434             *new_apdu->u.searchResponse->searchStatus = 0;
1435
1436             send_to_client(new_apdu);
1437
1438             return 0;
1439         }
1440         else
1441         {
1442             sr->query->which = Z_Query_type_1;
1443             sr->query->u.type_1 = rpnquery;
1444         }
1445         return apdu;
1446     }
1447     return apdu;
1448 }
1449
1450 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1451 {
1452     if (apdu->which == Z_APDU_searchRequest)
1453     {
1454         Z_SearchRequest *sr = apdu->u.searchRequest;
1455         int err = 0;
1456         char *addinfo = 0;
1457
1458         Yaz_ProxyConfig *cfg = check_reconfigure();
1459         if (cfg)
1460             err = cfg->check_query(odr_encode(), m_default_target,
1461                                    sr->query, &addinfo);
1462         if (err)
1463         {
1464             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1465
1466             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1467             new_apdu->u.searchResponse->records =
1468                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1469             *new_apdu->u.searchResponse->searchStatus = 0;
1470
1471             send_to_client(new_apdu);
1472
1473             return 0;
1474         }
1475     }
1476     return apdu;
1477 }
1478
1479 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1480 {
1481     m_marcxml_flag = 0;
1482     if (apdu->which == Z_APDU_searchRequest)
1483     {
1484         Z_SearchRequest *sr = apdu->u.searchRequest;
1485         int err = 0;
1486         char *addinfo = 0;
1487         Yaz_ProxyConfig *cfg = check_reconfigure();
1488
1489         Z_RecordComposition rc_temp, *rc = 0;
1490         if (sr->smallSetElementSetNames)
1491         {
1492             rc_temp.which = Z_RecordComp_simple;
1493             rc_temp.u.simple = sr->smallSetElementSetNames;
1494             rc = &rc_temp;
1495         }
1496
1497         if (cfg)
1498             err = cfg->check_syntax(odr_encode(),
1499                                     m_default_target,
1500                                     sr->preferredRecordSyntax, rc,
1501                                     &addinfo, &m_stylesheet_schema, &m_schema);
1502         if (err == -1)
1503         {
1504             sr->preferredRecordSyntax =
1505                 yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN, VAL_USMARC);
1506             m_marcxml_flag = 1;
1507         }
1508         else if (err)
1509         {
1510             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1511
1512             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1513             new_apdu->u.searchResponse->records =
1514                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1515             *new_apdu->u.searchResponse->searchStatus = 0;
1516
1517             send_to_client(new_apdu);
1518
1519             return 0;
1520         }
1521     }
1522     else if (apdu->which == Z_APDU_presentRequest)
1523     {
1524         Z_PresentRequest *pr = apdu->u.presentRequest;
1525         int err = 0;
1526         char *addinfo = 0;
1527         Yaz_ProxyConfig *cfg = check_reconfigure();
1528
1529         if (cfg)
1530             err = cfg->check_syntax(odr_encode(), m_default_target,
1531                                     pr->preferredRecordSyntax,
1532                                     pr->recordComposition,
1533                                     &addinfo, &m_stylesheet_schema, &m_schema);
1534         if (err == -1)
1535         {
1536             pr->preferredRecordSyntax =
1537                 yaz_oidval_to_z3950oid(odr_decode(), CLASS_RECSYN, VAL_USMARC);
1538             m_marcxml_flag = 1;
1539         }
1540         else if (err)
1541         {
1542             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1543
1544             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1545             new_apdu->u.presentResponse->records =
1546                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1547             *new_apdu->u.presentResponse->presentStatus =
1548                 Z_PresentStatus_failure;
1549
1550             send_to_client(new_apdu);
1551
1552             return 0;
1553         }
1554     }
1555     return apdu;
1556 }
1557
1558 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
1559 {
1560     if (!schema)
1561         return 0;
1562     Z_ElementSetNames *esn = (Z_ElementSetNames *)
1563         odr_malloc(o, sizeof(Z_ElementSetNames));
1564     esn->which = Z_ElementSetNames_generic;
1565     esn->u.generic = odr_strdup(o, schema);
1566     return esn;
1567 }
1568
1569 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1570 {
1571     if (m_s2z_odr_init)
1572     {
1573         odr_destroy(m_s2z_odr_init);
1574         m_s2z_odr_init = 0;
1575     }
1576     if (m_s2z_odr_search)
1577     {
1578         odr_destroy(m_s2z_odr_search);
1579         m_s2z_odr_search = 0;
1580     }
1581
1582     m_http_keepalive = 0;
1583     m_http_version = 0;
1584     if (!strcmp(hreq->version, "1.0"))
1585     {
1586         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1587         if (v && !strcmp(v, "Keep-Alive"))
1588             m_http_keepalive = 1;
1589         else
1590             m_http_keepalive = 0;
1591         m_http_version = "1.0";
1592     }
1593     else
1594     {
1595         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1596         if (v && !strcmp(v, "close"))
1597             m_http_keepalive = 0;
1598         else
1599             m_http_keepalive = 1;
1600         m_http_version = "1.1";
1601     }
1602
1603     Z_SRW_PDU *srw_pdu = 0;
1604     Z_SOAP *soap_package = 0;
1605     char *charset = 0;
1606     Z_SRW_diagnostic *diagnostic = 0;
1607     int num_diagnostic = 0;
1608     if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1609                        &charset) == 0
1610         || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1611                           &charset, &diagnostic, &num_diagnostic) == 0)
1612     {
1613         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
1614         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
1615         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
1616         m_s2z_init_apdu = 0;
1617         m_s2z_search_apdu = 0;
1618         m_s2z_present_apdu = 0;
1619
1620         m_s2z_stylesheet = 0;
1621
1622         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
1623         {
1624             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
1625
1626             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1627             // recordXPath unsupported.
1628             if (srw_req->recordXPath)
1629             {
1630                 yaz_add_srw_diagnostic(odr_decode(),
1631                                        &diagnostic, &num_diagnostic,
1632                                        72, 0);
1633             }
1634             // must have a query
1635             if (!srw_req->query.cql)
1636             {
1637                 yaz_add_srw_diagnostic(odr_decode(),
1638                                        &diagnostic, &num_diagnostic,
1639                                        7, "query");
1640             }
1641             // sort unsupported
1642             if (srw_req->sort_type != Z_SRW_sort_type_none)
1643             {
1644                 yaz_add_srw_diagnostic(odr_decode(),
1645                                        &diagnostic, &num_diagnostic,
1646                                        80, 0);
1647             }
1648             // save stylesheet
1649             if (srw_req->stylesheet)
1650                 m_s2z_stylesheet =
1651                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1652
1653             // set packing for response records ..
1654             if (srw_req->recordPacking &&
1655                 !strcmp(srw_req->recordPacking, "xml"))
1656                 m_s2z_packing = Z_SRW_recordPacking_XML;
1657             else
1658                 m_s2z_packing = Z_SRW_recordPacking_string;
1659
1660             if (num_diagnostic)
1661             {
1662                 Z_SRW_PDU *srw_pdu =
1663                     yaz_srw_get(odr_encode(),
1664                                 Z_SRW_searchRetrieve_response);
1665                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1666
1667                 srw_res->diagnostics = diagnostic;
1668                 srw_res->num_diagnostics = num_diagnostic;
1669                 send_srw_response(srw_pdu);
1670                 return;
1671             }
1672
1673             // prepare search PDU
1674             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
1675                                           Z_APDU_searchRequest);
1676             Z_SearchRequest *z_searchRequest =
1677                 m_s2z_search_apdu->u.searchRequest;
1678
1679             z_searchRequest->num_databaseNames = 1;
1680             z_searchRequest->databaseNames = (char**)
1681                 odr_malloc(m_s2z_odr_search, sizeof(char *));
1682             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
1683                                                            srw_req->database);
1684
1685             // query transformation
1686             Z_Query *query = (Z_Query *)
1687                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
1688             z_searchRequest->query = query;
1689
1690             if (srw_req->query_type == Z_SRW_query_type_cql)
1691             {
1692                 Z_External *ext = (Z_External *)
1693                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
1694                 ext->direct_reference =
1695                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
1696                 ext->indirect_reference = 0;
1697                 ext->descriptor = 0;
1698                 ext->which = Z_External_CQL;
1699                 ext->u.cql = srw_req->query.cql;
1700
1701                 query->which = Z_Query_type_104;
1702                 query->u.type_104 =  ext;
1703             }
1704             else if (srw_req->query_type == Z_SRW_query_type_pqf)
1705             {
1706                 Z_RPNQuery *RPNquery;
1707                 YAZ_PQF_Parser pqf_parser;
1708
1709                 pqf_parser = yaz_pqf_create ();
1710
1711                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
1712                                           srw_req->query.pqf);
1713                 if (!RPNquery)
1714                 {
1715                     const char *pqf_msg;
1716                     size_t off;
1717                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
1718                     yaz_log(LOG_LOG, "%*s^\n", off+4, "");
1719                     yaz_log(LOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
1720
1721                     send_to_srw_client_error(10, 0);
1722                     return;
1723                 }
1724                 query->which = Z_Query_type_1;
1725                 query->u.type_1 =  RPNquery;
1726
1727                 yaz_pqf_destroy (pqf_parser);
1728             }
1729             else
1730             {
1731                 send_to_srw_client_error(7, "query");
1732                 return;
1733             }
1734
1735             // present
1736             m_s2z_present_apdu = 0;
1737             int max = 0;
1738             if (srw_req->maximumRecords)
1739                 max = *srw_req->maximumRecords;
1740             int start = 1;
1741             if (srw_req->startRecord)
1742                 start = *srw_req->startRecord;
1743             if (max > 0)
1744             {
1745                 // Some backend, such as Voyager doesn't honor piggyback
1746                 // So we use present always (0 &&).
1747                 if (0 && start <= 1)  // Z39.50 piggyback
1748                 {
1749                     *z_searchRequest->smallSetUpperBound = max;
1750                     *z_searchRequest->mediumSetPresentNumber = max;
1751                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
1752
1753                     z_searchRequest->preferredRecordSyntax =
1754                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1755                                                VAL_TEXT_XML);
1756                     if (srw_req->recordSchema)
1757                     {
1758                         z_searchRequest->smallSetElementSetNames =
1759                             z_searchRequest->mediumSetElementSetNames =
1760                             mk_esn_from_schema(m_s2z_odr_search,
1761                                                srw_req->recordSchema);
1762                     }
1763                 }
1764                 else   // Z39.50 present
1765                 {
1766                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
1767                                                    Z_APDU_presentRequest);
1768                     Z_PresentRequest *z_presentRequest =
1769                         m_s2z_present_apdu->u.presentRequest;
1770                     *z_presentRequest->resultSetStartPoint = start;
1771                     *z_presentRequest->numberOfRecordsRequested = max;
1772                     z_presentRequest->preferredRecordSyntax =
1773                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1774                                                VAL_TEXT_XML);
1775                     z_presentRequest->recordComposition =
1776                         (Z_RecordComposition *)
1777                         odr_malloc(m_s2z_odr_search,
1778                                    sizeof(Z_RecordComposition));
1779                     if (srw_req->recordSchema)
1780                     {
1781                         z_presentRequest->recordComposition->which =
1782                             Z_RecordComp_simple;
1783                         z_presentRequest->recordComposition->u.simple =
1784                             mk_esn_from_schema(m_s2z_odr_search,
1785                                                srw_req->recordSchema);
1786                     }
1787                 }
1788             }
1789             if (!m_client)
1790             {
1791                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1792                                             Z_APDU_initRequest);
1793
1794                 // prevent m_initRequest_apdu memory from being grabbed
1795                 // in Yaz_Proxy::handle_incoming_Z_PDU
1796                 m_initRequest_apdu = m_s2z_init_apdu;
1797                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1798                 return;
1799             }
1800             else
1801             {
1802                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1803                 return;
1804             }
1805         }
1806         else if (srw_pdu->which == Z_SRW_explain_request)
1807         {
1808             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
1809
1810             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1811
1812             // save stylesheet
1813             if (srw_req->stylesheet)
1814                 m_s2z_stylesheet =
1815                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1816
1817             if (srw_req->recordPacking &&
1818                 !strcmp(srw_req->recordPacking, "xml"))
1819                 m_s2z_packing = Z_SRW_recordPacking_XML;
1820             else
1821                 m_s2z_packing = Z_SRW_recordPacking_string;
1822
1823             if (num_diagnostic)
1824             {
1825                 send_srw_explain_response(diagnostic, num_diagnostic);
1826                 return;
1827             }
1828
1829             if (!m_client)
1830             {
1831                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1832                                             Z_APDU_initRequest);
1833
1834                 // prevent m_initRequest_apdu memory from being grabbed
1835                 // in Yaz_Proxy::handle_incoming_Z_PDU
1836                 m_initRequest_apdu = m_s2z_init_apdu;
1837                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1838             }
1839             else
1840                 send_srw_explain_response(0, 0);
1841             return;
1842         }
1843         else if (srw_pdu->which == Z_SRW_scan_request)
1844         {
1845             m_s2z_database = odr_strdup(m_s2z_odr_init,
1846                                         srw_pdu->u.scan_request->database);
1847
1848             yaz_add_srw_diagnostic(odr_decode(),
1849                                    &diagnostic, &num_diagnostic,
1850                                    4, "scan");
1851             Z_SRW_PDU *srw_pdu =
1852                 yaz_srw_get(odr_encode(),
1853                             Z_SRW_scan_response);
1854             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1855
1856             srw_res->diagnostics = diagnostic;
1857             srw_res->num_diagnostics = num_diagnostic;
1858             send_srw_response(srw_pdu);
1859             return;
1860         }
1861         else
1862         {
1863             m_s2z_database = 0;
1864
1865             send_to_srw_client_error(4, 0);
1866         }
1867     }
1868     int len = 0;
1869     Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
1870     send_GDU(p, &len);
1871     timeout(1);
1872 }
1873
1874 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
1875 {
1876     if (!m_client && m_invalid_session)
1877     {
1878         m_apdu_invalid_session = apdu;
1879         m_mem_invalid_session = odr_extract_mem(odr_decode());
1880         apdu = m_initRequest_apdu;
1881     }
1882
1883     // Determine our client.
1884     Z_OtherInformation **oi;
1885     get_otherInfoAPDU(apdu, &oi);
1886     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
1887     if (!m_client)
1888     {
1889         delete this;
1890         return;
1891     }
1892     m_client->m_server = this;
1893
1894     if (apdu->which == Z_APDU_initRequest)
1895     {
1896         if (apdu->u.initRequest->implementationId)
1897             yaz_log(LOG_LOG, "%simplementationId: %s",
1898                     m_session_str, apdu->u.initRequest->implementationId);
1899         if (apdu->u.initRequest->implementationName)
1900             yaz_log(LOG_LOG, "%simplementationName: %s",
1901                     m_session_str, apdu->u.initRequest->implementationName);
1902         if (apdu->u.initRequest->implementationVersion)
1903             yaz_log(LOG_LOG, "%simplementationVersion: %s",
1904                     m_session_str, apdu->u.initRequest->implementationVersion);
1905         if (m_initRequest_apdu == 0)
1906         {
1907             if (m_initRequest_mem)
1908                 nmem_destroy(m_initRequest_mem);
1909             m_initRequest_apdu = apdu;
1910             m_initRequest_mem = odr_extract_mem(odr_decode());
1911
1912             // save init options for the response..
1913             m_initRequest_options = apdu->u.initRequest->options;
1914
1915             apdu->u.initRequest->options =
1916                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
1917                                            sizeof(Odr_bitmask));
1918             ODR_MASK_ZERO(apdu->u.initRequest->options);
1919             int i;
1920             for (i = 0; i<= 24; i++)
1921                 ODR_MASK_SET(apdu->u.initRequest->options, i);
1922             ODR_MASK_CLEAR(apdu->u.initRequest->options,
1923                            Z_Options_negotiationModel);
1924
1925             // make new version
1926             m_initRequest_version = apdu->u.initRequest->protocolVersion;
1927             apdu->u.initRequest->protocolVersion =
1928                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
1929                                            sizeof(Odr_bitmask));
1930             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
1931
1932             for (i = 0; i<= 8; i++)
1933                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
1934         }
1935         if (m_client->m_init_flag)
1936         {
1937             if (handle_init_response_for_invalid_session(apdu))
1938                 return;
1939             Z_APDU *apdu2 = m_client->m_initResponse;
1940             apdu2->u.initResponse->otherInfo = 0;
1941             if (m_client->m_cookie && *m_client->m_cookie)
1942                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
1943                                            m_client->m_cookie);
1944             apdu2->u.initResponse->referenceId =
1945                 apdu->u.initRequest->referenceId;
1946             apdu2->u.initResponse->options = m_client->m_initResponse_options;
1947             apdu2->u.initResponse->protocolVersion =
1948                 m_client->m_initResponse_version;
1949
1950             send_to_client(apdu2);
1951             return;
1952         }
1953         m_client->m_init_flag = 1;
1954     }
1955     handle_max_record_retrieve(apdu);
1956
1957     if (apdu)
1958         apdu = handle_syntax_validation(apdu);
1959
1960     if (apdu)
1961         apdu = handle_query_transformation(apdu);
1962
1963     if (apdu)
1964         apdu = handle_query_validation(apdu);
1965
1966     if (apdu)
1967         apdu = result_set_optimize(apdu);
1968     if (!apdu)
1969     {
1970         m_client->timeout(m_target_idletime);  // mark it active even
1971         // though we didn't use it
1972         return;
1973     }
1974
1975     // delete other info part from PDU before sending to target
1976     get_otherInfoAPDU(apdu, &oi);
1977     if (oi)
1978         *oi = 0;
1979
1980     if (apdu->which == Z_APDU_presentRequest &&
1981         m_client->m_resultSetStartPoint == 0)
1982     {
1983         Z_PresentRequest *pr = apdu->u.presentRequest;
1984         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
1985         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
1986     } else {
1987         m_client->m_resultSetStartPoint = 0;
1988     }
1989     if (m_client->send_to_target(apdu) < 0)
1990     {
1991         delete m_client;
1992         m_client = 0;
1993         delete this;
1994     }
1995     else
1996         m_client->m_waiting = 1;
1997 }
1998
1999 void Yaz_Proxy::connectNotify()
2000 {
2001 }
2002
2003 void Yaz_Proxy::shutdown()
2004 {
2005     m_invalid_session = 0;
2006     // only keep if keep_alive flag is set...
2007     if (m_client &&
2008         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2009         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2010         m_client->m_waiting == 0)
2011     {
2012         yaz_log(LOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2013                  m_session_str,
2014                  m_client->get_hostname());
2015         yaz_log(LOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2016                 m_session_str, m_client->m_pdu_recv,
2017                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2018                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2019         assert (m_client->m_waiting != 2);
2020         // Tell client (if any) that no server connection is there..
2021         m_client->m_server = 0;
2022         m_invalid_session = 0;
2023     }
2024     else if (m_client)
2025     {
2026         yaz_log (LOG_LOG, "%sShutdown (client to proxy) close %s",
2027                  m_session_str,
2028                  m_client->get_hostname());
2029         assert (m_client->m_waiting != 2);
2030         delete m_client;
2031     }
2032     else if (!m_parent)
2033     {
2034         yaz_log (LOG_LOG, "%sshutdown (client to proxy) bad state",
2035                  m_session_str);
2036         assert (m_parent);
2037     }
2038     else
2039     {
2040         yaz_log (LOG_LOG, "%sShutdown (client to proxy)",
2041                  m_session_str);
2042     }
2043     if (m_parent)
2044         m_parent->pre_init();
2045     delete this;
2046 }
2047
2048 const char *Yaz_ProxyClient::get_session_str()
2049 {
2050     if (!m_server)
2051         return "0 ";
2052     return m_server->get_session_str();
2053 }
2054
2055 void Yaz_ProxyClient::shutdown()
2056 {
2057     yaz_log (LOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2058              get_hostname());
2059     delete m_server;
2060     delete this;
2061 }
2062
2063 void Yaz_Proxy::failNotify()
2064 {
2065     inc_request_no();
2066     yaz_log (LOG_LOG, "%sConnection closed by client",
2067              get_session_str());
2068     shutdown();
2069 }
2070
2071 void Yaz_ProxyClient::failNotify()
2072 {
2073     if (m_server)
2074         m_server->inc_request_no();
2075     yaz_log (LOG_LOG, "%sConnection closed by target %s",
2076              get_session_str(), get_hostname());
2077     shutdown();
2078 }
2079
2080 void Yaz_ProxyClient::connectNotify()
2081 {
2082     const char *s = get_session_str();
2083     const char *h = get_hostname();
2084     yaz_log (LOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2085              m_target_idletime);
2086     timeout(m_target_idletime);
2087     if (!m_server)
2088         pre_init_client();
2089 }
2090
2091 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2092                                                   *the_PDU_Observable, int fd)
2093 {
2094     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2095 }
2096
2097 Yaz_ProxyClient::~Yaz_ProxyClient()
2098 {
2099     if (m_prev)
2100         *m_prev = m_next;
2101     if (m_next)
2102         m_next->m_prev = m_prev;
2103     m_waiting = 2;     // for debugging purposes only.
2104     odr_destroy(m_init_odr);
2105     delete m_last_query;
2106     xfree (m_last_resultSetId);
2107     xfree (m_cookie);
2108 }
2109
2110 void Yaz_ProxyClient::pre_init_client()
2111 {
2112     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2113     Z_InitRequest *req = apdu->u.initRequest;
2114
2115     int i;
2116     for (i = 0; i<= 24; i++)
2117         ODR_MASK_SET(req->options, i);
2118     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2119                    Z_Options_negotiationModel);
2120     for (i = 0; i<= 10; i++)
2121         ODR_MASK_SET(req->protocolVersion, i);
2122
2123     if (send_to_target(apdu) < 0)
2124     {
2125         delete this;
2126     }
2127     else
2128     {
2129         m_waiting = 1;
2130         m_init_flag = 1;
2131     }
2132 }
2133
2134 void Yaz_Proxy::pre_init()
2135 {
2136     int i;
2137     const char *name = 0;
2138     const char *zurl_in_use[MAX_ZURL_PLEX];
2139     int limit_bw, limit_pdu, limit_req;
2140     int target_idletime, client_idletime;
2141     int max_clients;
2142     int keepalive_limit_bw, keepalive_limit_pdu;
2143     int pre_init;
2144     const char *cql2rpn = 0;
2145
2146     Yaz_ProxyConfig *cfg = check_reconfigure();
2147
2148     zurl_in_use[0] = 0;
2149
2150     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2151         set_APDU_yazlog(1);
2152     else
2153         set_APDU_yazlog(0);
2154
2155     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2156                                           &limit_bw, &limit_pdu, &limit_req,
2157                                           &target_idletime, &client_idletime,
2158                                           &max_clients,
2159                                           &keepalive_limit_bw,
2160                                           &keepalive_limit_pdu,
2161                                           &pre_init,
2162                                           &cql2rpn) ; i++)
2163     {
2164         if (pre_init)
2165         {
2166             int j;
2167             for (j = 0; zurl_in_use[j]; j++)
2168             {
2169                 Yaz_ProxyClient *c;
2170                 int spare = 0;
2171                 int in_use = 0;
2172                 int other = 0;
2173                 for (c = m_clientPool; c; c = c->m_next)
2174                 {
2175                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
2176                     {
2177                         if (c->m_cookie == 0)
2178                         {
2179                             if (c->m_server == 0)
2180                                 spare++;
2181                             else
2182                                 in_use++;
2183                         }
2184                         else
2185                             other++;
2186                     }
2187                 }
2188                 yaz_log(LOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2189                         "preinit=%d",m_session_str,
2190                         name, zurl_in_use[j], in_use, other, spare, pre_init);
2191                 if (spare < pre_init)
2192                 {
2193                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2194                     c->m_next = m_clientPool;
2195                     if (c->m_next)
2196                         c->m_next->m_prev = &c->m_next;
2197                     m_clientPool = c;
2198                     c->m_prev = &m_clientPool;
2199
2200                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
2201                         c->set_APDU_yazlog(1);
2202                     else
2203                         c->set_APDU_yazlog(0);
2204
2205                     if (c->client(zurl_in_use[j]))
2206                     {
2207                         timeout(60);
2208                         delete c;
2209                         return;
2210                     }
2211                     c->timeout(30);
2212                     c->m_waiting = 1;
2213                     c->m_target_idletime = target_idletime;
2214                     c->m_seqno = m_seqno++;
2215                 }
2216             }
2217         }
2218     }
2219 }
2220
2221 void Yaz_Proxy::timeoutNotify()
2222 {
2223     if (m_parent)
2224     {
2225         if (m_bw_hold_PDU)
2226         {
2227             timeout(m_client_idletime);
2228             Z_GDU *apdu = m_bw_hold_PDU;
2229             m_bw_hold_PDU = 0;
2230
2231             if (apdu->which == Z_GDU_Z3950)
2232                 handle_incoming_Z_PDU(apdu->u.z3950);
2233             else if (apdu->which == Z_GDU_HTTP_Request)
2234                 handle_incoming_HTTP(apdu->u.HTTP_Request);
2235         }
2236         else
2237         {
2238             inc_request_no();
2239
2240             yaz_log (LOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2241             shutdown();
2242         }
2243     }
2244     else
2245     {
2246         timeout(600);
2247         pre_init();
2248     }
2249 }
2250
2251 void Yaz_Proxy::markInvalid()
2252 {
2253     m_client = 0;
2254     m_invalid_session = 1;
2255 }
2256
2257 void Yaz_ProxyClient::timeoutNotify()
2258 {
2259     if (m_server)
2260         m_server->inc_request_no();
2261
2262     yaz_log (LOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2263              get_hostname());
2264     m_waiting = 1;
2265     m_root->pre_init();
2266     if (m_server && m_init_flag)
2267     {
2268         // target timed out in a session that was properly initialized
2269         // server object stay alive but we mark it as invalid so it
2270         // gets initialized again
2271         m_server->markInvalid();
2272         m_server = 0;
2273     }
2274     shutdown();
2275 }
2276
2277 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2278                                  Yaz_Proxy *parent) :
2279     Yaz_Z_Assoc (the_PDU_Observable)
2280 {
2281     m_cookie = 0;
2282     m_next = 0;
2283     m_prev = 0;
2284     m_init_flag = 0;
2285     m_last_query = 0;
2286     m_last_resultSetId = 0;
2287     m_last_resultCount = 0;
2288     m_last_ok = 0;
2289     m_sr_transform = 0;
2290     m_waiting = 0;
2291     m_init_odr = odr_createmem (ODR_DECODE);
2292     m_initResponse = 0;
2293     m_initResponse_options = 0;
2294     m_initResponse_version = 0;
2295     m_resultSetStartPoint = 0;
2296     m_bytes_sent = m_bytes_recv = 0;
2297     m_pdu_recv = 0;
2298     m_server = 0;
2299     m_seqno = 0;
2300     m_target_idletime = 600;
2301     m_root = parent;
2302 }
2303
2304 const char *Yaz_Proxy::option(const char *name, const char *value)
2305 {
2306     if (!strcmp (name, "optimize")) {
2307         if (value) {
2308             xfree (m_optimize);
2309             m_optimize = xstrdup (value);
2310         }
2311         return m_optimize;
2312     }
2313     return 0;
2314 }
2315
2316 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2317 {
2318
2319 }
2320
2321 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2322 {
2323     if (apdu->which == Z_GDU_Z3950)
2324         recv_Z_PDU(apdu->u.z3950, len);
2325     else if (apdu->which == Z_GDU_HTTP_Response)
2326         recv_HTTP_response(apdu->u.HTTP_Response, len);
2327     else
2328         shutdown();
2329 }
2330
2331 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2332 {
2333     if (!m_invalid_session)
2334         return 0;
2335     m_invalid_session = 0;
2336     handle_incoming_Z_PDU(m_apdu_invalid_session);
2337     assert (m_mem_invalid_session);
2338     nmem_destroy(m_mem_invalid_session);
2339     m_mem_invalid_session = 0;
2340     return 1;
2341 }
2342
2343 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2344 {
2345     m_bytes_recv += len;
2346     m_pdu_recv++;
2347     m_waiting = 0;
2348     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2349         yaz_log (LOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2350                  apdu_name(apdu), get_hostname(), len);
2351     if (apdu->which == Z_APDU_initResponse)
2352     {
2353         if (!m_server)  // if this is a pre init session , check for more
2354             m_root->pre_init();
2355         NMEM nmem = odr_extract_mem (odr_decode());
2356         odr_reset (m_init_odr);
2357         nmem_transfer (m_init_odr->mem, nmem);
2358         m_initResponse = apdu;
2359         m_initResponse_options = apdu->u.initResponse->options;
2360         m_initResponse_version = apdu->u.initResponse->protocolVersion;
2361
2362         Z_InitResponse *ir = apdu->u.initResponse;
2363         char *im0 = ir->implementationName;
2364
2365         char *im1 = (char*)
2366             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2367         *im1 = '\0';
2368         if (im0)
2369         {
2370             strcat(im1, im0);
2371             strcat(im1, " ");
2372         }
2373         strcat(im1, "(YAZ Proxy)");
2374         ir->implementationName = im1;
2375
2376         nmem_destroy (nmem);
2377
2378         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2379             return;
2380     }
2381     if (apdu->which == Z_APDU_searchResponse)
2382     {
2383         Z_SearchResponse *sr = apdu->u.searchResponse;
2384         m_last_resultCount = *sr->resultCount;
2385         int status = *sr->searchStatus;
2386         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2387         {
2388             m_last_ok = 1;
2389
2390             if (sr->records && sr->records->which == Z_Records_DBOSD)
2391             {
2392                 m_cache.add(odr_decode(),
2393                             sr->records->u.databaseOrSurDiagnostics, 1,
2394                             *sr->resultCount);
2395             }
2396         }
2397     }
2398     if (apdu->which == Z_APDU_presentResponse)
2399     {
2400         Z_PresentResponse *pr = apdu->u.presentResponse;
2401         if (m_sr_transform)
2402         {
2403             m_sr_transform = 0;
2404             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2405             Z_SearchResponse *sr = new_apdu->u.searchResponse;
2406             sr->referenceId = pr->referenceId;
2407             *sr->resultCount = m_last_resultCount;
2408             sr->records = pr->records;
2409             sr->nextResultSetPosition = pr->nextResultSetPosition;
2410             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2411             apdu = new_apdu;
2412         }
2413         if (pr->records &&
2414             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2415         {
2416             m_cache.add(odr_decode(),
2417                         pr->records->u.databaseOrSurDiagnostics,
2418                         m_resultSetStartPoint, -1);
2419             m_resultSetStartPoint = 0;
2420         }
2421     }
2422     if (m_cookie)
2423         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2424     if (m_server)
2425     {
2426         m_server->send_to_client(apdu);
2427     }
2428     if (apdu->which == Z_APDU_close)
2429     {
2430         shutdown();
2431     }
2432 }
2433
2434 int Yaz_Proxy::server(const char *addr)
2435 {
2436     int r = Yaz_Z_Assoc::server(addr);
2437     if (!r)
2438     {
2439         yaz_log(LOG_LOG, "%sStarted proxy " VERSION " on %s", m_session_str, addr);
2440         timeout(1);
2441     }
2442     return r;
2443 }
2444