src/yaz-proxy.cpp

   1 /*
   2  * Copyright (c) 1998-2004, Index Data.
   3  * See the file LICENSE for details.
   4  *
   5  * $Id: yaz-proxy.cpp,v 1.90 2004-01-15 15:47:52 adam Exp $
   6  */
   7
   8 #include <assert.h>
   9 #include <time.h>
  10
  11 #include <yaz/srw.h>
  12 #include <yaz/marcdisp.h>
  13 #include <yaz/yaz-iconv.h>
  14 #include <yaz/log.h>
  15 #include <yaz/diagbib1.h>
  16 #include <yaz++/proxy.h>
  17 #include <yaz/pquery.h>
  18
  19 #if HAVE_XSLT
  20 #include <libxslt/xsltutils.h>
  21 #include <libxslt/transform.h>
  22 #endif
  23
  24 static const char *apdu_name(Z_APDU *apdu)
  25 {
  26     switch (apdu->which)
  27     {
  28     case Z_APDU_initRequest:
  29         return "initRequest";
  30     case Z_APDU_initResponse:
  31         return "initResponse";
  32     case Z_APDU_searchRequest:
  33         return "searchRequest";
  34     case Z_APDU_searchResponse:
  35         return "searchResponse";
  36     case Z_APDU_presentRequest:
  37         return "presentRequest";
  38     case Z_APDU_presentResponse:
  39         return "presentResponse";
  40     case Z_APDU_deleteResultSetRequest:
  41         return "deleteResultSetRequest";
  42     case Z_APDU_deleteResultSetResponse:
  43         return "deleteResultSetResponse";
  44     case Z_APDU_scanRequest:
  45         return "scanRequest";
  46     case Z_APDU_scanResponse:
  47         return "scanResponse";
  48     case Z_APDU_sortRequest:
  49         return "sortRequest";
  50     case Z_APDU_sortResponse:
  51         return "sortResponse";
  52     case Z_APDU_extendedServicesRequest:
  53         return "extendedServicesRequest";
  54     case Z_APDU_extendedServicesResponse:
  55         return "extendedServicesResponse";
  56     case Z_APDU_close:
  57         return "close";
  58     }
  59     return "other";
  60 }
  61
  62 static const char *gdu_name(Z_GDU *gdu)
  63 {
  64     switch(gdu->which)
  65     {
  66     case Z_GDU_Z3950:
  67         return apdu_name(gdu->u.z3950);
  68     case Z_GDU_HTTP_Request:
  69         return "HTTP Request";
  70     case Z_GDU_HTTP_Response:
  71         return "HTTP Response";
  72     }
  73     return "Unknown request/response";
  74 }
  75
  76 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
  77                      Yaz_Proxy *parent) :
  78     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
  79 {
  80     m_PDU_Observable = the_PDU_Observable;
  81     m_client = 0;
  82     m_parent = parent;
  83     m_clientPool = 0;
  84     m_seqno = 1;
  85     m_keepalive_limit_bw = 500000;
  86     m_keepalive_limit_pdu = 1000;
  87     m_proxyTarget = 0;
  88     m_default_target = 0;
  89     m_proxy_authentication = 0;
  90     m_max_clients = 150;
  91     m_log_mask = 0;
  92     m_seed = time(0);
  93     m_client_idletime = 600;
  94     m_target_idletime = 600;
  95     m_optimize = xstrdup ("1");
  96     strcpy(m_session_str, "0 ");
  97     m_session_no=0;
  98     m_bytes_sent = m_bytes_recv = 0;
  99     m_bw_hold_PDU = 0;
 100     m_bw_max = 0;
 101     m_pdu_max = 0;
 102     m_max_record_retrieve = 0;
 103     m_reconfig_flag = 0;
 104     m_config_fname = 0;
 105     m_request_no = 0;
 106     m_invalid_session = 0;
 107     m_config = 0;
 108     m_marcxml_flag = 0;
 109     m_stylesheet_schema = 0;
 110     m_s2z_stylesheet = 0;
 111     m_schema = 0;
 112     m_initRequest_apdu = 0;
 113     m_initRequest_mem = 0;
 114     m_initRequest_options = 0;
 115     m_initRequest_version = 0;
 116     m_apdu_invalid_session = 0;
 117     m_mem_invalid_session = 0;
 118     m_s2z_odr_init = 0;
 119     m_s2z_odr_search = 0;
 120     m_s2z_init_apdu = 0;
 121     m_s2z_search_apdu = 0;
 122     m_s2z_present_apdu = 0;
 123     m_http_keepalive = 0;
 124     m_http_version = 0;
 125     m_soap_ns = 0;
 126     m_s2z_packing = Z_SRW_recordPacking_string;
 127     m_time_tv.tv_sec = 0;
 128     m_time_tv.tv_usec = 0;
 129 }
 130
 131 Yaz_Proxy::~Yaz_Proxy()
 132 {
 133     yaz_log(LOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 134             m_bytes_sent, m_bytes_recv);
 135     nmem_destroy(m_initRequest_mem);
 136     nmem_destroy(m_mem_invalid_session);
 137     xfree (m_proxyTarget);
 138     xfree (m_default_target);
 139     xfree (m_proxy_authentication);
 140     xfree (m_optimize);
 141     xfree (m_stylesheet_schema);
 142     xfree (m_schema);
 143     if (m_s2z_odr_init)
 144         odr_destroy(m_s2z_odr_init);
 145     if (m_s2z_odr_search)
 146         odr_destroy(m_s2z_odr_search);
 147     delete m_config;
 148 }
 149
 150 int Yaz_Proxy::set_config(const char *config)
 151 {
 152     delete m_config;
 153     m_config = new Yaz_ProxyConfig();
 154     xfree(m_config_fname);
 155     m_config_fname = xstrdup(config);
 156     int r = m_config->read_xml(config);
 157     if (!r)
 158         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 159     return r;
 160 }
 161
 162 void Yaz_Proxy::set_default_target(const char *target)
 163 {
 164     xfree (m_default_target);
 165     m_default_target = 0;
 166     if (target)
 167         m_default_target = (char *) xstrdup (target);
 168 }
 169
 170 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 171 {
 172     xfree (m_proxy_authentication);
 173     m_proxy_authentication = 0;
 174     if (auth)
 175         m_proxy_authentication = (char *) xstrdup (auth);
 176 }
 177
 178 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 179 {
 180     if (m_parent)
 181         return m_parent->check_reconfigure();
 182
 183     Yaz_ProxyConfig *cfg = m_config;
 184     if (m_reconfig_flag)
 185     {
 186         yaz_log(LOG_LOG, "reconfigure");
 187         yaz_log_reopen();
 188         if (m_config_fname && cfg)
 189         {
 190             yaz_log(LOG_LOG, "reconfigure config %s", m_config_fname);
 191             int r = cfg->read_xml(m_config_fname);
 192             if (r)
 193                 yaz_log(LOG_WARN, "reconfigure failed");
 194             else
 195             {
 196                 m_log_mask = 0;
 197                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 198             }
 199         }
 200         else
 201             yaz_log(LOG_LOG, "reconfigure");
 202         m_reconfig_flag = 0;
 203     }
 204     return cfg;
 205 }
 206
 207 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 208                                             *the_PDU_Observable, int fd)
 209 {
 210     check_reconfigure();
 211     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 212     new_proxy->m_config = 0;
 213     new_proxy->m_config_fname = 0;
 214     new_proxy->timeout(m_client_idletime);
 215     new_proxy->m_target_idletime = m_target_idletime;
 216     new_proxy->set_default_target(m_default_target);
 217     new_proxy->m_max_clients = m_max_clients;
 218     new_proxy->m_log_mask = m_log_mask;
 219     new_proxy->set_APDU_log(get_APDU_log());
 220     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 221         new_proxy->set_APDU_yazlog(1);
 222     else
 223         new_proxy->set_APDU_yazlog(0);
 224     new_proxy->set_proxy_authentication(m_proxy_authentication);
 225     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 226     m_session_no++;
 227     yaz_log (LOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 228              the_PDU_Observable->getpeername());
 229     return new_proxy;
 230 }
 231
 232 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 233 {
 234     int oid[OID_SIZE];
 235     Z_OtherInformationUnit *oi;
 236     struct oident ent;
 237     ent.proto = PROTO_Z3950;
 238     ent.oclass = CLASS_USERINFO;
 239     ent.value = (oid_value) VAL_COOKIE;
 240     assert (oid_ent_to_oid (&ent, oid));
 241
 242     if (oid_ent_to_oid (&ent, oid) &&
 243         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 244         oi->which == Z_OtherInfo_characterInfo)
 245         return oi->information.characterInfo;
 246     return 0;
 247 }
 248
 249 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 250 {
 251     int oid[OID_SIZE];
 252     Z_OtherInformationUnit *oi;
 253     struct oident ent;
 254     ent.proto = PROTO_Z3950;
 255     ent.oclass = CLASS_USERINFO;
 256     ent.value = (oid_value) VAL_PROXY;
 257     if (oid_ent_to_oid (&ent, oid) &&
 258         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 259         oi->which == Z_OtherInfo_characterInfo)
 260         return oi->information.characterInfo;
 261     return 0;
 262 }
 263
 264 const char *Yaz_Proxy::load_balance(const char **url)
 265 {
 266     int zurl_in_use[MAX_ZURL_PLEX];
 267     Yaz_ProxyClient *c;
 268     int i;
 269
 270     for (i = 0; i<MAX_ZURL_PLEX; i++)
 271         zurl_in_use[i] = 0;
 272     for (c = m_parent->m_clientPool; c; c = c->m_next)
 273     {
 274         for (i = 0; url[i]; i++)
 275             if (!strcmp(url[i], c->get_hostname()))
 276                 zurl_in_use[i]++;
 277     }
 278     int min = 100000;
 279     const char *ret = 0;
 280     for (i = 0; url[i]; i++)
 281     {
 282         yaz_log(LOG_DEBUG, "%szurl=%s use=%d",
 283                 m_session_str, url[i], zurl_in_use[i]);
 284         if (min > zurl_in_use[i])
 285         {
 286             ret = url[i];
 287             min = zurl_in_use[i];
 288         }
 289     }
 290     return ret;
 291 }
 292
 293 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 294                                        const char *proxy_host)
 295 {
 296     assert (m_parent);
 297     Yaz_Proxy *parent = m_parent;
 298     Yaz_ProxyClient *c = m_client;
 299
 300     if (!m_proxyTarget)
 301     {
 302         const char *url[MAX_ZURL_PLEX];
 303         Yaz_ProxyConfig *cfg = check_reconfigure();
 304         if (proxy_host)
 305         {
 306 #if 1
 307 /* only to be enabled for debugging... */
 308             if (!strcmp(proxy_host, "stop"))
 309                 exit(0);
 310 #endif
 311             xfree(m_default_target);
 312             m_default_target = xstrdup(proxy_host);
 313             proxy_host = m_default_target;
 314         }
 315         int client_idletime = -1;
 316         const char *cql2rpn_fname = 0;
 317         url[0] = m_default_target;
 318         url[1] = 0;
 319         if (cfg)
 320         {
 321             int pre_init = 0;
 322             cfg->get_target_info(proxy_host, url, &m_bw_max,
 323                                  &m_pdu_max, &m_max_record_retrieve,
 324                                  &m_target_idletime, &client_idletime,
 325                                  &parent->m_max_clients,
 326                                  &m_keepalive_limit_bw,
 327                                  &m_keepalive_limit_pdu,
 328                                  &pre_init,
 329                                  &cql2rpn_fname);
 330         }
 331         if (client_idletime != -1)
 332         {
 333             m_client_idletime = client_idletime;
 334             timeout(m_client_idletime);
 335         }
 336         if (cql2rpn_fname)
 337             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 338         if (!url[0])
 339         {
 340             yaz_log(LOG_LOG, "%sNo default target", m_session_str);
 341             return 0;
 342         }
 343         // we don't handle multiplexing for cookie session, so we just
 344         // pick the first one in this case (anonymous users will be able
 345         // to use any backend)
 346         if (cookie && *cookie)
 347             m_proxyTarget = (char*) xstrdup(url[0]);
 348         else
 349             m_proxyTarget = (char*) xstrdup(load_balance(url));
 350     }
 351     if (cookie && *cookie)
 352     {   // search in sessions with a cookie
 353         for (c = parent->m_clientPool; c; c = c->m_next)
 354         {
 355             assert (c->m_prev);
 356             assert (*c->m_prev == c);
 357             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 358                 !strcmp(m_proxyTarget, c->get_hostname()))
 359             {
 360                 // Found it in cache
 361                 // The following handles "cancel"
 362                 // If connection is busy (waiting for PDU) and
 363                 // we have an initRequest we can safely do re-open
 364                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 365                 {
 366                     yaz_log (LOG_LOG, "%s REOPEN target=%s", m_session_str,
 367                              c->get_hostname());
 368                     c->close();
 369                     c->m_init_flag = 0;
 370
 371                     c->m_last_ok = 0;
 372                     c->m_cache.clear();
 373                     c->m_last_resultCount = 0;
 374                     c->m_sr_transform = 0;
 375                     c->m_waiting = 0;
 376                     c->m_resultSetStartPoint = 0;
 377                     c->m_target_idletime = m_target_idletime;
 378                     if (c->client(m_proxyTarget))
 379                     {
 380                         delete c;
 381                         return 0;
 382                     }
 383                     c->timeout(30);
 384                 }
 385                 c->m_seqno = parent->m_seqno;
 386                 if (c->m_server && c->m_server != this)
 387                     c->m_server->m_client = 0;
 388                 c->m_server = this;
 389                 (parent->m_seqno)++;
 390                 yaz_log (LOG_DEBUG, "get_client 1 %p %p", this, c);
 391                 return c;
 392             }
 393         }
 394     }
 395     else if (!c)
 396     {
 397         // don't have a client session yet. Search in session w/o cookie
 398         for (c = parent->m_clientPool; c; c = c->m_next)
 399         {
 400             assert (c->m_prev);
 401             assert (*c->m_prev == c);
 402             if (c->m_server == 0 && c->m_cookie == 0 &&
 403                 c->m_waiting == 0 &&
 404                 !strcmp(m_proxyTarget, c->get_hostname()))
 405             {
 406                 // found it in cache
 407                 yaz_log (LOG_LOG, "%sREUSE %s",
 408                          m_session_str, c->get_hostname());
 409
 410                 c->m_seqno = parent->m_seqno;
 411                 assert(c->m_server == 0);
 412                 c->m_server = this;
 413
 414                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 415                     c->set_APDU_yazlog(1);
 416                 else
 417                     c->set_APDU_yazlog(0);
 418
 419                 (parent->m_seqno)++;
 420
 421                 parent->pre_init();
 422
 423                 return c;
 424             }
 425         }
 426     }
 427     if (!m_client)
 428     {
 429         if (apdu->which != Z_APDU_initRequest)
 430         {
 431             yaz_log (LOG_LOG, "%sno init request as first PDU", m_session_str);
 432             return 0;
 433         }
 434         Z_InitRequest *initRequest = apdu->u.initRequest;
 435
 436         if (!initRequest->idAuthentication)
 437         {
 438             if (m_proxy_authentication)
 439             {
 440                 initRequest->idAuthentication =
 441                     (Z_IdAuthentication *)
 442                     odr_malloc (odr_encode(),
 443                                 sizeof(*initRequest->idAuthentication));
 444                 initRequest->idAuthentication->which =
 445                     Z_IdAuthentication_open;
 446                 initRequest->idAuthentication->u.open =
 447                     odr_strdup (odr_encode(), m_proxy_authentication);
 448             }
 449         }
 450         // go through list of clients - and find the lowest/oldest one.
 451         Yaz_ProxyClient *c_min = 0;
 452         int min_seq = -1;
 453         int no_of_clients = 0;
 454         if (parent->m_clientPool)
 455             yaz_log (LOG_DEBUG, "Existing sessions");
 456         for (c = parent->m_clientPool; c; c = c->m_next)
 457         {
 458             yaz_log (LOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 459                                c->m_waiting, c->get_hostname(),
 460                                c->m_cookie ? c->m_cookie : "");
 461             no_of_clients++;
 462             if (min_seq < 0 || c->m_seqno < min_seq)
 463             {
 464                 min_seq = c->m_seqno;
 465                 c_min = c;
 466             }
 467         }
 468         if (no_of_clients >= parent->m_max_clients)
 469         {
 470             c = c_min;
 471             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 472             {
 473                 yaz_log (LOG_LOG, "%sMAXCLIENTS Destroy %d",
 474                          m_session_str, c->m_seqno);
 475                 if (c->m_server && c->m_server != this)
 476                     delete c->m_server;
 477                 c->m_server = 0;
 478             }
 479             else
 480             {
 481                 yaz_log (LOG_LOG, "%sMAXCLIENTS Reuse %d %d %s",
 482                          m_session_str,
 483                          c->m_seqno, parent->m_seqno, c->get_hostname());
 484                 xfree (c->m_cookie);
 485                 c->m_cookie = 0;
 486                 if (cookie)
 487                     c->m_cookie = xstrdup(cookie);
 488                 c->m_seqno = parent->m_seqno;
 489                 if (c->m_server && c->m_server != this)
 490                 {
 491                     c->m_server->m_client = 0;
 492                     delete c->m_server;
 493                 }
 494                 (parent->m_seqno)++;
 495                 c->m_target_idletime = m_target_idletime;
 496                 c->timeout(m_target_idletime);
 497
 498                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 499                     c->set_APDU_yazlog(1);
 500                 else
 501                     c->set_APDU_yazlog(0);
 502
 503                 return c;
 504             }
 505         }
 506         else
 507         {
 508             yaz_log (LOG_LOG, "%sNEW %d %s",
 509                      m_session_str, parent->m_seqno, m_proxyTarget);
 510             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 511             c->m_next = parent->m_clientPool;
 512             if (c->m_next)
 513                 c->m_next->m_prev = &c->m_next;
 514             parent->m_clientPool = c;
 515             c->m_prev = &parent->m_clientPool;
 516         }
 517
 518         xfree (c->m_cookie);
 519         c->m_cookie = 0;
 520         if (cookie)
 521             c->m_cookie = xstrdup(cookie);
 522
 523         c->m_seqno = parent->m_seqno;
 524         c->m_init_flag = 0;
 525         c->m_last_resultCount = 0;
 526         c->m_last_ok = 0;
 527         c->m_cache.clear();
 528         c->m_sr_transform = 0;
 529         c->m_waiting = 0;
 530         c->m_resultSetStartPoint = 0;
 531         (parent->m_seqno)++;
 532         if (c->client(m_proxyTarget))
 533         {
 534             delete c;
 535             return 0;
 536         }
 537         c->m_target_idletime = m_target_idletime;
 538         c->timeout(30);
 539
 540         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 541             c->set_APDU_yazlog(1);
 542         else
 543             c->set_APDU_yazlog(0);
 544     }
 545     yaz_log (LOG_DEBUG, "get_client 3 %p %p", this, c);
 546     return c;
 547 }
 548
 549 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 550 {
 551     int i;
 552     for (i = 0; i<num; i++)
 553     {
 554         oident *ent;
 555         Z_DefaultDiagFormat *r;
 556         Z_DiagRec *p = pp[i];
 557         if (p->which != Z_DiagRec_defaultFormat)
 558         {
 559             yaz_log(LOG_LOG, "%sError no diagnostics", m_session_str);
 560             return;
 561         }
 562         else
 563             r = p->u.defaultFormat;
 564         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 565             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 566             yaz_log(LOG_LOG, "%sError unknown diagnostic set", m_session_str);
 567         switch (r->which)
 568         {
 569         case Z_DefaultDiagFormat_v2Addinfo:
 570             yaz_log(LOG_LOG, "%sError %d %s:%s",
 571                     m_session_str,
 572                     *r->condition, diagbib1_str(*r->condition),
 573                     r->u.v2Addinfo);
 574             break;
 575         case Z_DefaultDiagFormat_v3Addinfo:
 576             yaz_log(LOG_LOG, "%sError %d %s:%s",
 577                     m_session_str,
 578                     *r->condition, diagbib1_str(*r->condition),
 579                     r->u.v3Addinfo);
 580             break;
 581         }
 582     }
 583 }
 584
 585 void Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p)
 586 {
 587     if (!m_stylesheet_schema)
 588         return;
 589     xsltStylesheetPtr xsp;
 590
 591     xsp = xsltParseStylesheetFile((const xmlChar *) m_stylesheet_schema);
 592
 593     int i;
 594     for (i = 0; i < p->num_records; i++)
 595     {
 596         Z_NamePlusRecord *npr = p->records[i];
 597         if (npr->which == Z_NamePlusRecord_databaseRecord)
 598         {
 599             Z_External *r = npr->u.databaseRecord;
 600             if (r->which == Z_External_octet)
 601             {
 602                 xmlDocPtr res, doc = xmlParseMemory(
 603                     (char*) r->u.octet_aligned->buf,
 604                     r->u.octet_aligned->len);
 605
 606                 res = xsltApplyStylesheet(xsp, doc, 0);
 607
 608                 xmlChar *out_buf;
 609                 int out_len;
 610                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 611
 612                 p->records[i]->u.databaseRecord =
 613                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 614                                  (char*) out_buf, out_len);
 615                 xmlFree(out_buf);
 616                 xmlFreeDoc(doc);
 617                 xmlFreeDoc(res);
 618             }
 619         }
 620     }
 621     xsltFreeStylesheet(xsp);
 622 }
 623
 624 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p)
 625 {
 626     int i;
 627
 628     yaz_marc_t mt = yaz_marc_create();
 629     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 630     for (i = 0; i < p->num_records; i++)
 631     {
 632         Z_NamePlusRecord *npr = p->records[i];
 633         if (npr->which == Z_NamePlusRecord_databaseRecord)
 634         {
 635             Z_External *r = npr->u.databaseRecord;
 636             if (r->which == Z_External_octet)
 637             {
 638                 int rlen;
 639                 char *result;
 640                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 641                                         r->u.octet_aligned->len,
 642                                         &result, &rlen))
 643                 {
 644                     yaz_iconv_t cd = yaz_iconv_open("UTF-8", "MARC-8");
 645                     WRBUF wrbuf = wrbuf_alloc();
 646
 647                     char outbuf[120];
 648                     size_t inbytesleft = rlen;
 649                     const char *inp = result;
 650                     while (cd && inbytesleft)
 651                     {
 652                         size_t outbytesleft = sizeof(outbuf);
 653                         char *outp = outbuf;
 654                         size_t r;
 655
 656                         r = yaz_iconv (cd, (char**) &inp,
 657                                        &inbytesleft,
 658                                        &outp, &outbytesleft);
 659                         if (r == (size_t) (-1))
 660                         {
 661                             int e = yaz_iconv_error(cd);
 662                             if (e != YAZ_ICONV_E2BIG)
 663                             {
 664                                 yaz_log(LOG_WARN, "conversion failure");
 665                                 break;
 666                             }
 667                         }
 668                         wrbuf_write(wrbuf, outbuf, outp - outbuf);
 669                     }
 670                     if (cd)
 671                         yaz_iconv_close(cd);
 672
 673                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 674                                                          VAL_TEXT_XML,
 675                                                          wrbuf_buf(wrbuf),
 676                                                          wrbuf_len(wrbuf));
 677                     wrbuf_free(wrbuf, 1);
 678                 }
 679             }
 680         }
 681     }
 682     yaz_marc_destroy(mt);
 683 }
 684
 685 void Yaz_Proxy::logtime()
 686 {
 687     if (m_time_tv.tv_sec)
 688     {
 689         struct timeval tv;
 690         gettimeofday(&tv, 0);
 691         long diff = (tv.tv_sec - m_time_tv.tv_sec)*1000000 +
 692             (tv.tv_usec - m_time_tv.tv_usec);
 693         if (diff >= 0)
 694             yaz_log(LOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 695                     diff/1000000, (diff/1000)%1000);
 696     }
 697     m_time_tv.tv_sec = 0;
 698     m_time_tv.tv_usec = 0;
 699 }
 700
 701 int Yaz_Proxy::send_http_response(int code)
 702 {
 703     ODR o = odr_encode();
 704     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 705     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 706     if (m_http_version)
 707         hres->version = odr_strdup(o, m_http_version);
 708     m_http_keepalive = 0;
 709     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 710     {
 711         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 712                  gdu_name(gdu));
 713     }
 714     int len;
 715     int r = send_GDU(gdu, &len);
 716     logtime();
 717     return r;
 718 }
 719
 720 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 721 {
 722     ODR o = odr_encode();
 723     const char *ctype = "text/xml";
 724     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 725     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 726     if (m_http_version)
 727         hres->version = odr_strdup(o, m_http_version);
 728     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 729     if (m_http_keepalive)
 730         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 731
 732     static Z_SOAP_Handler soap_handlers[2] = {
 733 #if HAVE_XSLT
 734         {"http://www.loc.gov/zing/srw/", 0,
 735          (Z_SOAP_fun) yaz_srw_codec},
 736 #endif
 737         {0, 0, 0}
 738     };
 739
 740     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 741     soap_package->which = Z_SOAP_generic;
 742     soap_package->u.generic =
 743         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 744     soap_package->u.generic->no = 0;
 745     soap_package->u.generic->ns = soap_handlers[0].ns;
 746     soap_package->u.generic->p = (void *) srw_pdu;
 747     soap_package->ns = m_soap_ns;
 748     z_soap_codec_enc_xsl(o, &soap_package,
 749                          &hres->content_buf, &hres->content_len,
 750                          soap_handlers, 0, m_s2z_stylesheet);
 751     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 752     {
 753         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 754                  gdu_name(gdu));
 755     }
 756     int len;
 757     int r = send_GDU(gdu, &len);
 758     logtime();
 759     return r;
 760 }
 761
 762 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
 763 {
 764     ODR o = odr_encode();
 765     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 766     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 767
 768     srw_res->num_diagnostics = 1;
 769     srw_res->diagnostics = (Z_SRW_diagnostic *)
 770         odr_malloc(o, sizeof(*srw_res->diagnostics));
 771     srw_res->diagnostics[0].code =  odr_intdup(o, srw_error);
 772     srw_res->diagnostics[0].details = add ? odr_strdup(o, add) : 0;
 773     return send_srw_response(srw_pdu);
 774 }
 775
 776 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 777                              Z_DefaultDiagFormat *ddf)
 778 {
 779     int bib1_code = *ddf->condition;
 780     if (bib1_code == 109)
 781         return 404;
 782     srw_res->num_diagnostics = 1;
 783     srw_res->diagnostics = (Z_SRW_diagnostic *)
 784         odr_malloc(o, sizeof(*srw_res->diagnostics));
 785     srw_res->diagnostics[0].code =
 786         odr_intdup(o, yaz_diag_bib1_to_srw(*ddf->condition));
 787     srw_res->diagnostics[0].details = ddf->u.v2Addinfo;
 788     return 0;
 789 }
 790
 791 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
 792 {
 793     ODR o = odr_encode();
 794     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 795     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 796
 797     srw_res->numberOfRecords = odr_intdup (o, hits);
 798     if (records && records->which == Z_Records_DBOSD)
 799     {
 800         srw_res->num_records =
 801             records->u.databaseOrSurDiagnostics->num_records;
 802         int i;
 803         srw_res->records = (Z_SRW_record *)
 804             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
 805         for (i = 0; i < srw_res->num_records; i++)
 806         {
 807             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
 808             if (npr->which != Z_NamePlusRecord_databaseRecord)
 809             {
 810                 srw_res->records[i].recordSchema = "diagnostic";
 811                 srw_res->records[i].recordPacking = m_s2z_packing;
 812                 srw_res->records[i].recordData_buf = "67";
 813                 srw_res->records[i].recordData_len = 2;
 814                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 815                 continue;
 816             }
 817             Z_External *r = npr->u.databaseRecord;
 818             oident *ent = oid_getentbyoid(r->direct_reference);
 819             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
 820             {
 821                 srw_res->records[i].recordSchema = m_schema;
 822                 srw_res->records[i].recordPacking = m_s2z_packing;
 823                 srw_res->records[i].recordData_buf = (char*)
 824                     r->u.octet_aligned->buf;
 825                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
 826                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 827             }
 828             else
 829             {
 830                 srw_res->records[i].recordSchema = "diagnostic";
 831                 srw_res->records[i].recordPacking = m_s2z_packing;
 832                 srw_res->records[i].recordData_buf = "67";
 833                 srw_res->records[i].recordData_len = 2;
 834                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 835             }
 836         }
 837     }
 838     if (records && records->which == Z_Records_NSD)
 839     {
 840         int http_code;
 841         http_code = z_to_srw_diag(odr_encode(), srw_res,
 842                                    records->u.nonSurrogateDiagnostic);
 843         if (http_code)
 844             return send_http_response(http_code);
 845     }
 846     return send_srw_response(srw_pdu);
 847
 848 }
 849
 850 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
 851                                         int num_diagnostics)
 852 {
 853     Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
 854     Z_SRW_explainResponse *er = res->u.explain_response;
 855
 856     Yaz_ProxyConfig *cfg = check_reconfigure();
 857     if (cfg)
 858     {
 859         int len;
 860         char *b = cfg->get_explain(odr_encode(), 0 /* target */,
 861                                    0 /* db */, &len);
 862         if (b)
 863         {
 864             er->record.recordData_buf = b;
 865             er->record.recordData_len = len;
 866             er->record.recordPacking = m_s2z_packing;
 867         }
 868     }
 869     er->diagnostics = diagnostics;
 870     er->num_diagnostics = num_diagnostics;
 871     return send_srw_response(res);
 872 }
 873
 874 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu, int *len)
 875 {
 876     if (m_http_version)
 877     {
 878         if (apdu->which == Z_APDU_initResponse)
 879         {
 880             Z_InitResponse *res = apdu->u.initResponse;
 881             if (*res->result == 0)
 882             {
 883                 send_to_srw_client_error(3, 0);
 884             }
 885             else if (!m_s2z_search_apdu)
 886             {
 887                 send_srw_explain_response(0, 0);
 888             }
 889             else
 890             {
 891                 handle_incoming_Z_PDU(m_s2z_search_apdu);
 892             }
 893         }
 894         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
 895         {
 896             m_s2z_search_apdu = 0;
 897             Z_SearchResponse *res = apdu->u.searchResponse;
 898             m_s2z_hit_count = *res->resultCount;
 899             if (res->records && res->records->which == Z_Records_NSD)
 900             {
 901                 send_to_srw_client_ok(0, res->records, 1);
 902             }
 903             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
 904             {
 905                 // adjust
 906                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
 907
 908                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
 909                 {
 910                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
 911                         > m_s2z_hit_count)
 912                         *pr->numberOfRecordsRequested =
 913                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
 914                 }
 915                 handle_incoming_Z_PDU(m_s2z_present_apdu);
 916             }
 917             else
 918             {
 919                 m_s2z_present_apdu = 0;
 920                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
 921             }
 922         }
 923         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
 924         {
 925             int start =
 926                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
 927
 928             m_s2z_present_apdu = 0;
 929             Z_PresentResponse *res = apdu->u.presentResponse;
 930             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
 931         }
 932     }
 933     else
 934     {
 935         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 936             yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 937                      apdu_name(apdu));
 938         int r = send_Z_PDU(apdu, len);
 939         logtime();
 940         return r;
 941     }
 942     return 0;
 943 }
 944
 945 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 946 {
 947     int len = 0;
 948     int kill_session = 0;
 949     if (apdu->which == Z_APDU_searchResponse)
 950     {
 951         Z_SearchResponse *sr = apdu->u.searchResponse;
 952         Z_Records *p = sr->records;
 953         if (p && p->which == Z_Records_NSD)
 954         {
 955             Z_DiagRec dr, *dr_p = &dr;
 956             dr.which = Z_DiagRec_defaultFormat;
 957             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 958
 959             *sr->searchStatus = 0;
 960             display_diagrecs(&dr_p, 1);
 961         }
 962         else
 963         {
 964             if (p && p->which == Z_Records_DBOSD)
 965             {
 966                 if (m_marcxml_flag)
 967                     convert_to_marcxml(p->u.databaseOrSurDiagnostics);
 968                 convert_xsl(p->u.databaseOrSurDiagnostics);
 969             }
 970             if (sr->resultCount)
 971             {
 972                 yaz_log(LOG_LOG, "%s%d hits", m_session_str,
 973                         *sr->resultCount);
 974                 if (*sr->resultCount < 0)
 975                 {
 976                     m_invalid_session = 1;
 977                     kill_session = 1;
 978
 979                     *sr->searchStatus = 0;
 980                     sr->records =
 981                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
 982                     *sr->resultCount = 0;
 983                 }
 984             }
 985         }
 986     }
 987     else if (apdu->which == Z_APDU_presentResponse)
 988     {
 989         Z_PresentResponse *sr = apdu->u.presentResponse;
 990         Z_Records *p = sr->records;
 991         if (p && p->which == Z_Records_NSD)
 992         {
 993             Z_DiagRec dr, *dr_p = &dr;
 994             dr.which = Z_DiagRec_defaultFormat;
 995             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 996             if (*sr->presentStatus == Z_PresentStatus_success)
 997                 *sr->presentStatus = Z_PresentStatus_failure;
 998             display_diagrecs(&dr_p, 1);
 999         }
1000         if (p && p->which == Z_Records_DBOSD)
1001         {
1002             if (m_marcxml_flag)
1003                 convert_to_marcxml(p->u.databaseOrSurDiagnostics);
1004             convert_xsl(p->u.databaseOrSurDiagnostics);
1005         }
1006     }
1007     else if (apdu->which == Z_APDU_initResponse)
1008     {
1009         if (m_initRequest_options)
1010         {
1011             Z_Options *nopt =
1012                 (Odr_bitmask *)odr_malloc(odr_encode(),
1013                                           sizeof(Odr_bitmask));
1014             ODR_MASK_ZERO(nopt);
1015
1016             int i;
1017             for (i = 0; i<24; i++)
1018                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1019                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1020                     ODR_MASK_SET(nopt, i);
1021             apdu->u.initResponse->options = nopt;
1022         }
1023         if (m_initRequest_version)
1024         {
1025             Z_ProtocolVersion *nopt =
1026                 (Odr_bitmask *)odr_malloc(odr_encode(),
1027                                           sizeof(Odr_bitmask));
1028             ODR_MASK_ZERO(nopt);
1029
1030             int i;
1031             for (i = 0; i<8; i++)
1032                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1033                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1034                     ODR_MASK_SET(nopt, i);
1035             apdu->u.initResponse->protocolVersion = nopt;
1036         }
1037     }
1038     int r = send_PDU_convert(apdu, &len);
1039     if (r)
1040         return r;
1041     m_bytes_sent += len;
1042     m_bw_stat.add_bytes(len);
1043     if (kill_session)
1044     {
1045         delete m_client;
1046         m_client = 0;
1047         m_parent->pre_init();
1048     }
1049     if (m_http_version)
1050     {
1051         if (!m_http_keepalive)
1052         {
1053 #if 1
1054             timeout(1);
1055 #else
1056             shutdown();
1057             return -1;
1058 #endif
1059         }
1060     }
1061     return r;
1062 }
1063
1064 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1065 {
1066     int len = 0;
1067     const char *apdu_name_tmp = apdu_name(apdu);
1068     int r = send_Z_PDU(apdu, &len);
1069     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1070         yaz_log (LOG_LOG, "%sSending %s to %s %d bytes",
1071                  get_session_str(),
1072                  apdu_name_tmp, get_hostname(), len);
1073     m_bytes_sent += len;
1074     return r;
1075 }
1076
1077 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1078 {
1079     if (apdu->which == Z_APDU_presentRequest)
1080     {
1081         Z_PresentRequest *pr = apdu->u.presentRequest;
1082         int toget = *pr->numberOfRecordsRequested;
1083         int start = *pr->resultSetStartPoint;
1084
1085         yaz_log(LOG_LOG, "%sPresent %s %d+%d", m_session_str,
1086                 pr->resultSetId, start, toget);
1087
1088         if (*m_parent->m_optimize == '0')
1089             return apdu;
1090
1091         if (!m_client->m_last_resultSetId)
1092         {
1093             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1094             new_apdu->u.presentResponse->records =
1095                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1096                                                pr->resultSetId);
1097             send_to_client(new_apdu);
1098             return 0;
1099         }
1100         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1101         {
1102             if (start+toget-1 > m_client->m_last_resultCount)
1103             {
1104                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1105                 new_apdu->u.presentResponse->records =
1106                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1107                 send_to_client(new_apdu);
1108                 return 0;
1109             }
1110             Z_NamePlusRecordList *npr;
1111             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1112                                           pr->preferredRecordSyntax,
1113                                           pr->recordComposition))
1114             {
1115                 yaz_log (LOG_LOG, "%sReturned cached records for present request",
1116                          m_session_str);
1117                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1118                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1119
1120                 new_apdu->u.presentResponse->numberOfRecordsReturned
1121                     = odr_intdup(odr_encode(), toget);
1122
1123                 new_apdu->u.presentResponse->records = (Z_Records*)
1124                     odr_malloc(odr_encode(), sizeof(Z_Records));
1125                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1126                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1127                 new_apdu->u.presentResponse->nextResultSetPosition =
1128                     odr_intdup(odr_encode(), start+toget);
1129
1130                 send_to_client(new_apdu);
1131                 return 0;
1132             }
1133         }
1134     }
1135
1136     if (apdu->which != Z_APDU_searchRequest)
1137         return apdu;
1138     Z_SearchRequest *sr = apdu->u.searchRequest;
1139     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1140     Yaz_Z_Databases this_databases;
1141
1142     this_databases.set(sr->num_databaseNames, (const char **)
1143                        sr->databaseNames);
1144
1145     this_query->set_Z_Query(sr->query);
1146
1147     char query_str[120];
1148     this_query->print(query_str, sizeof(query_str)-1);
1149     yaz_log(LOG_LOG, "%sSearch %s", m_session_str, query_str);
1150
1151     if (*m_parent->m_optimize != '0' &&
1152         m_client->m_last_ok && m_client->m_last_query &&
1153         m_client->m_last_query->match(this_query) &&
1154         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1155         m_client->m_last_databases.match(this_databases))
1156     {
1157         delete this_query;
1158         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1159             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1160         {
1161             Z_NamePlusRecordList *npr;
1162             int toget = *sr->mediumSetPresentNumber;
1163             Z_RecordComposition *comp = 0;
1164
1165             if (toget > m_client->m_last_resultCount)
1166                 toget = m_client->m_last_resultCount;
1167
1168             if (sr->mediumSetElementSetNames)
1169             {
1170                 comp = (Z_RecordComposition *)
1171                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1172                 comp->which = Z_RecordComp_simple;
1173                 comp->u.simple = sr->mediumSetElementSetNames;
1174             }
1175
1176             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1177                                           sr->preferredRecordSyntax, comp))
1178             {
1179                 yaz_log (LOG_LOG, "%sReturned cached records for medium set",
1180                          m_session_str);
1181                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1182                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1183                 new_apdu->u.searchResponse->resultCount =
1184                     &m_client->m_last_resultCount;
1185
1186                 new_apdu->u.searchResponse->numberOfRecordsReturned
1187                     = odr_intdup(odr_encode(), toget);
1188
1189                 new_apdu->u.searchResponse->presentStatus =
1190                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1191                 new_apdu->u.searchResponse->records = (Z_Records*)
1192                     odr_malloc(odr_encode(), sizeof(Z_Records));
1193                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1194                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1195                 new_apdu->u.searchResponse->nextResultSetPosition =
1196                     odr_intdup(odr_encode(), toget+1);
1197                 send_to_client(new_apdu);
1198                 return 0;
1199             }
1200             else
1201             {
1202                 // medium Set
1203                 // send present request (medium size)
1204                 yaz_log (LOG_LOG, "%sOptimizing search for medium set",
1205                          m_session_str);
1206
1207                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1208                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1209                 pr->referenceId = sr->referenceId;
1210                 pr->resultSetId = sr->resultSetName;
1211                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1212                 *pr->numberOfRecordsRequested = toget;
1213                 pr->recordComposition = comp;
1214                 m_client->m_sr_transform = 1;
1215                 return new_apdu;
1216             }
1217         }
1218         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1219             m_client->m_last_resultCount <= 0)
1220         {
1221             // large set. Return pseudo-search response immediately
1222             yaz_log (LOG_LOG, "%sOptimizing search for large set",
1223                      m_session_str);
1224             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1225             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1226             new_apdu->u.searchResponse->resultCount =
1227                 &m_client->m_last_resultCount;
1228             send_to_client(new_apdu);
1229             return 0;
1230         }
1231         else
1232         {
1233             Z_NamePlusRecordList *npr;
1234             int toget = m_client->m_last_resultCount;
1235             Z_RecordComposition *comp = 0;
1236             // small set
1237             // send a present request (small set)
1238
1239             if (sr->smallSetElementSetNames)
1240             {
1241                 comp = (Z_RecordComposition *)
1242                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1243                 comp->which = Z_RecordComp_simple;
1244                 comp->u.simple = sr->smallSetElementSetNames;
1245             }
1246
1247             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1248                                           sr->preferredRecordSyntax, comp))
1249             {
1250                 yaz_log (LOG_LOG, "%sReturned cached records for small set",
1251                          m_session_str);
1252                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1253                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1254                 new_apdu->u.searchResponse->resultCount =
1255                     &m_client->m_last_resultCount;
1256
1257                 new_apdu->u.searchResponse->numberOfRecordsReturned
1258                     = odr_intdup(odr_encode(), toget);
1259
1260                 new_apdu->u.searchResponse->presentStatus =
1261                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1262                 new_apdu->u.searchResponse->records = (Z_Records*)
1263                     odr_malloc(odr_encode(), sizeof(Z_Records));
1264                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1265                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1266                 new_apdu->u.searchResponse->nextResultSetPosition =
1267                     odr_intdup(odr_encode(), toget+1);
1268                 send_to_client(new_apdu);
1269                 return 0;
1270             }
1271             else
1272             {
1273                 yaz_log (LOG_LOG, "%sOptimizing search for small set",
1274                          m_session_str);
1275                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1276                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1277                 pr->referenceId = sr->referenceId;
1278                 pr->resultSetId = sr->resultSetName;
1279                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1280                 *pr->numberOfRecordsRequested = toget;
1281                 pr->recordComposition = comp;
1282                 m_client->m_sr_transform = 1;
1283                 return new_apdu;
1284             }
1285         }
1286     }
1287     else  // query doesn't match
1288     {
1289         delete m_client->m_last_query;
1290         m_client->m_last_query = this_query;
1291         m_client->m_last_ok = 0;
1292         m_client->m_cache.clear();
1293         m_client->m_resultSetStartPoint = 0;
1294
1295         xfree (m_client->m_last_resultSetId);
1296         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1297
1298         m_client->m_last_databases.set(sr->num_databaseNames,
1299                                        (const char **) sr->databaseNames);
1300     }
1301     return apdu;
1302 }
1303
1304
1305 void Yaz_Proxy::inc_request_no()
1306 {
1307     char *cp = strchr(m_session_str, ' ');
1308     m_request_no++;
1309     if (cp)
1310         sprintf(cp+1, "%d ", m_request_no);
1311 }
1312
1313 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1314 {
1315     inc_request_no();
1316
1317     m_bytes_recv += len;
1318
1319     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
1320         yaz_log (LOG_DEBUG, "%sReceiving %s from client %d bytes",
1321                  m_session_str, gdu_name(apdu), len);
1322
1323     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1324         shutdown();
1325
1326     m_bw_stat.add_bytes(len);
1327     m_pdu_stat.add_bytes(1);
1328
1329     gettimeofday(&m_time_tv, 0);
1330
1331     int bw_total = m_bw_stat.get_total();
1332     int pdu_total = m_pdu_stat.get_total();
1333
1334     int reduce = 0;
1335     if (m_bw_max)
1336     {
1337         if (bw_total > m_bw_max)
1338         {
1339             reduce = (bw_total/m_bw_max);
1340         }
1341     }
1342     if (m_pdu_max)
1343     {
1344         if (pdu_total > m_pdu_max)
1345         {
1346             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1347             reduce = (reduce > nreduce) ? reduce : nreduce;
1348         }
1349     }
1350     if (reduce)
1351     {
1352         yaz_log(LOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1353                 m_session_str, reduce, bw_total, pdu_total,
1354                 m_bw_max, m_pdu_max);
1355
1356         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1357         timeout(reduce);       // call us reduce seconds later
1358     }
1359     else if (apdu->which == Z_GDU_Z3950)
1360         handle_incoming_Z_PDU(apdu->u.z3950);
1361     else if (apdu->which == Z_GDU_HTTP_Request)
1362         handle_incoming_HTTP(apdu->u.HTTP_Request);
1363 }
1364
1365 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1366 {
1367     if (m_max_record_retrieve)
1368     {
1369         if (apdu->which == Z_APDU_presentRequest)
1370         {
1371             Z_PresentRequest *pr = apdu->u.presentRequest;
1372             if (pr->numberOfRecordsRequested &&
1373                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1374                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1375         }
1376     }
1377 }
1378
1379 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1380                                                      int error,
1381                                                      const char *addinfo)
1382 {
1383     Z_Records *rec = (Z_Records *)
1384         odr_malloc (odr, sizeof(*rec));
1385     int *err = (int *)
1386         odr_malloc (odr, sizeof(*err));
1387     Z_DiagRec *drec = (Z_DiagRec *)
1388         odr_malloc (odr, sizeof(*drec));
1389     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1390         odr_malloc (odr, sizeof(*dr));
1391     *err = error;
1392     rec->which = Z_Records_NSD;
1393     rec->u.nonSurrogateDiagnostic = dr;
1394     dr->diagnosticSetId =
1395         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1396     dr->condition = err;
1397     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1398     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1399     return rec;
1400 }
1401
1402 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1403 {
1404     if (apdu->which == Z_APDU_searchRequest &&
1405         apdu->u.searchRequest->query &&
1406         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1407         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1408     {
1409         Z_RPNQuery *rpnquery = 0;
1410         Z_SearchRequest *sr = apdu->u.searchRequest;
1411         char *addinfo = 0;
1412
1413         yaz_log(LOG_LOG, "%sCQL: %s", m_session_str,
1414                 sr->query->u.type_104->u.cql);
1415
1416         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1417                                           &rpnquery, odr_encode(),
1418                                           &addinfo);
1419         if (r == -3)
1420             yaz_log(LOG_LOG, "%sNo CQL to RPN table", m_session_str);
1421         else if (r)
1422         {
1423             yaz_log(LOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1424             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1425
1426             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1427             new_apdu->u.searchResponse->records =
1428                 create_nonSurrogateDiagnostics(odr_encode(),
1429                                                yaz_diag_srw_to_bib1(r),
1430                                                addinfo);
1431             *new_apdu->u.searchResponse->searchStatus = 0;
1432
1433             send_to_client(new_apdu);
1434
1435             return 0;
1436         }
1437         else
1438         {
1439             sr->query->which = Z_Query_type_1;
1440             sr->query->u.type_1 = rpnquery;
1441         }
1442         return apdu;
1443     }
1444     return apdu;
1445 }
1446
1447 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1448 {
1449     if (apdu->which == Z_APDU_searchRequest)
1450     {
1451         Z_SearchRequest *sr = apdu->u.searchRequest;
1452         int err = 0;
1453         char *addinfo = 0;
1454
1455         Yaz_ProxyConfig *cfg = check_reconfigure();
1456         if (cfg)
1457             err = cfg->check_query(odr_encode(), m_default_target,
1458                                    sr->query, &addinfo);
1459         if (err)
1460         {
1461             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1462
1463             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1464             new_apdu->u.searchResponse->records =
1465                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1466             *new_apdu->u.searchResponse->searchStatus = 0;
1467
1468             send_to_client(new_apdu);
1469
1470             return 0;
1471         }
1472     }
1473     return apdu;
1474 }
1475
1476 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1477 {
1478     m_marcxml_flag = 0;
1479     if (apdu->which == Z_APDU_searchRequest)
1480     {
1481         Z_SearchRequest *sr = apdu->u.searchRequest;
1482         int err = 0;
1483         char *addinfo = 0;
1484         Yaz_ProxyConfig *cfg = check_reconfigure();
1485
1486         Z_RecordComposition rc_temp, *rc = 0;
1487         if (sr->smallSetElementSetNames)
1488         {
1489             rc_temp.which = Z_RecordComp_simple;
1490             rc_temp.u.simple = sr->smallSetElementSetNames;
1491             rc = &rc_temp;
1492         }
1493
1494         if (cfg)
1495             err = cfg->check_syntax(odr_encode(),
1496                                     m_default_target,
1497                                     sr->preferredRecordSyntax, rc,
1498                                     &addinfo, &m_stylesheet_schema, &m_schema);
1499         if (err == -1)
1500         {
1501             sr->preferredRecordSyntax =
1502                 yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN, VAL_USMARC);
1503             m_marcxml_flag = 1;
1504         }
1505         else if (err)
1506         {
1507             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1508
1509             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1510             new_apdu->u.searchResponse->records =
1511                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1512             *new_apdu->u.searchResponse->searchStatus = 0;
1513
1514             send_to_client(new_apdu);
1515
1516             return 0;
1517         }
1518     }
1519     else if (apdu->which == Z_APDU_presentRequest)
1520     {
1521         Z_PresentRequest *pr = apdu->u.presentRequest;
1522         int err = 0;
1523         char *addinfo = 0;
1524         Yaz_ProxyConfig *cfg = check_reconfigure();
1525
1526         if (cfg)
1527             err = cfg->check_syntax(odr_encode(), m_default_target,
1528                                     pr->preferredRecordSyntax,
1529                                     pr->recordComposition,
1530                                     &addinfo, &m_stylesheet_schema, &m_schema);
1531         if (err == -1)
1532         {
1533             pr->preferredRecordSyntax =
1534                 yaz_oidval_to_z3950oid(odr_decode(), CLASS_RECSYN, VAL_USMARC);
1535             m_marcxml_flag = 1;
1536         }
1537         else if (err)
1538         {
1539             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1540
1541             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1542             new_apdu->u.presentResponse->records =
1543                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1544             *new_apdu->u.presentResponse->presentStatus =
1545                 Z_PresentStatus_failure;
1546
1547             send_to_client(new_apdu);
1548
1549             return 0;
1550         }
1551     }
1552     return apdu;
1553 }
1554
1555 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
1556 {
1557     if (!schema)
1558         return 0;
1559     Z_ElementSetNames *esn = (Z_ElementSetNames *)
1560         odr_malloc(o, sizeof(Z_ElementSetNames));
1561     esn->which = Z_ElementSetNames_generic;
1562     esn->u.generic = odr_strdup(o, schema);
1563     return esn;
1564 }
1565
1566 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1567 {
1568
1569     if (m_s2z_odr_init)
1570     {
1571         odr_destroy(m_s2z_odr_init);
1572         m_s2z_odr_init = 0;
1573     }
1574     if (m_s2z_odr_search)
1575     {
1576         odr_destroy(m_s2z_odr_search);
1577         m_s2z_odr_search = 0;
1578     }
1579
1580     m_http_keepalive = 0;
1581     m_http_version = 0;
1582     if (!strcmp(hreq->version, "1.0"))
1583     {
1584         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1585         if (v && !strcmp(v, "Keep-Alive"))
1586             m_http_keepalive = 1;
1587         else
1588             m_http_keepalive = 0;
1589         m_http_version = "1.0";
1590     }
1591     else
1592     {
1593         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1594         if (v && !strcmp(v, "close"))
1595             m_http_keepalive = 0;
1596         else
1597             m_http_keepalive = 1;
1598         m_http_version = "1.1";
1599     }
1600
1601     Z_SRW_PDU *srw_pdu = 0;
1602     Z_SOAP *soap_package = 0;
1603     char *charset = 0;
1604     Z_SRW_diagnostic *diagnostic = 0;
1605     int num_diagnostic = 0;
1606     if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1607                        &charset) == 0
1608         || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1609                           &charset, &diagnostic, &num_diagnostic) == 0)
1610     {
1611         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
1612         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
1613         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
1614         m_s2z_init_apdu = 0;
1615         m_s2z_search_apdu = 0;
1616         m_s2z_present_apdu = 0;
1617
1618         m_s2z_stylesheet = 0;
1619
1620         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
1621         {
1622             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
1623
1624             // recordXPath unsupported.
1625             if (srw_req->recordXPath)
1626             {
1627                 yaz_add_srw_diagnostic(odr_decode(),
1628                                        &diagnostic, &num_diagnostic,
1629                                        72, 0);
1630             }
1631             // must have a query
1632             if (!srw_req->query.cql)
1633             {
1634                 yaz_add_srw_diagnostic(odr_decode(),
1635                                        &diagnostic, &num_diagnostic,
1636                                        7, "query");
1637             }
1638             // sort unsupported
1639             if (srw_req->sort_type != Z_SRW_sort_type_none)
1640             {
1641                 yaz_add_srw_diagnostic(odr_decode(),
1642                                        &diagnostic, &num_diagnostic,
1643                                        80, 0);
1644             }
1645             // save stylesheet
1646             if (srw_req->stylesheet)
1647                 m_s2z_stylesheet =
1648                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1649
1650             // set packing for response records ..
1651             if (srw_req->recordPacking &&
1652                 !strcmp(srw_req->recordPacking, "xml"))
1653                 m_s2z_packing = Z_SRW_recordPacking_XML;
1654             else
1655                 m_s2z_packing = Z_SRW_recordPacking_string;
1656
1657             if (num_diagnostic)
1658             {
1659                 Z_SRW_PDU *srw_pdu =
1660                     yaz_srw_get(odr_encode(),
1661                                 Z_SRW_searchRetrieve_response);
1662                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1663
1664                 srw_res->diagnostics = diagnostic;
1665                 srw_res->num_diagnostics = num_diagnostic;
1666                 send_srw_response(srw_pdu);
1667                 return;
1668             }
1669
1670             // prepare search PDU
1671             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
1672                                           Z_APDU_searchRequest);
1673             Z_SearchRequest *z_searchRequest =
1674                 m_s2z_search_apdu->u.searchRequest;
1675
1676             z_searchRequest->num_databaseNames = 1;
1677             z_searchRequest->databaseNames = (char**)
1678                 odr_malloc(m_s2z_odr_search, sizeof(char *));
1679             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
1680                                                            srw_req->database);
1681
1682             // query transformation
1683             Z_Query *query = (Z_Query *)
1684                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
1685             z_searchRequest->query = query;
1686
1687             if (srw_req->query_type == Z_SRW_query_type_cql)
1688             {
1689                 Z_External *ext = (Z_External *)
1690                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
1691                 ext->direct_reference =
1692                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
1693                 ext->indirect_reference = 0;
1694                 ext->descriptor = 0;
1695                 ext->which = Z_External_CQL;
1696                 ext->u.cql = srw_req->query.cql;
1697
1698                 query->which = Z_Query_type_104;
1699                 query->u.type_104 =  ext;
1700             }
1701             else if (srw_req->query_type == Z_SRW_query_type_pqf)
1702             {
1703                 Z_RPNQuery *RPNquery;
1704                 YAZ_PQF_Parser pqf_parser;
1705
1706                 pqf_parser = yaz_pqf_create ();
1707
1708                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
1709                                           srw_req->query.pqf);
1710                 if (!RPNquery)
1711                 {
1712                     const char *pqf_msg;
1713                     size_t off;
1714                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
1715                     yaz_log(LOG_LOG, "%*s^\n", off+4, "");
1716                     yaz_log(LOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
1717
1718                     send_to_srw_client_error(10, 0);
1719                     return;
1720                 }
1721                 query->which = Z_Query_type_1;
1722                 query->u.type_1 =  RPNquery;
1723
1724                 yaz_pqf_destroy (pqf_parser);
1725             }
1726             else
1727             {
1728                 send_to_srw_client_error(7, "query");
1729                 return;
1730             }
1731
1732             // present
1733             m_s2z_present_apdu = 0;
1734             int max = 0;
1735             if (srw_req->maximumRecords)
1736                 max = *srw_req->maximumRecords;
1737             int start = 1;
1738             if (srw_req->startRecord)
1739                 start = *srw_req->startRecord;
1740             if (max > 0)
1741             {
1742                 // Some backend, such as Voyager doesn't honor piggyback
1743                 // So we use present always (0 &&).
1744                 if (0 && start <= 1)  // Z39.50 piggyback
1745                 {
1746                     *z_searchRequest->smallSetUpperBound = max;
1747                     *z_searchRequest->mediumSetPresentNumber = max;
1748                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
1749
1750                     z_searchRequest->preferredRecordSyntax =
1751                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1752                                                VAL_TEXT_XML);
1753                     if (srw_req->recordSchema)
1754                     {
1755                         z_searchRequest->smallSetElementSetNames =
1756                             z_searchRequest->mediumSetElementSetNames =
1757                             mk_esn_from_schema(m_s2z_odr_search,
1758                                                srw_req->recordSchema);
1759                     }
1760                 }
1761                 else   // Z39.50 present
1762                 {
1763                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
1764                                                    Z_APDU_presentRequest);
1765                     Z_PresentRequest *z_presentRequest =
1766                         m_s2z_present_apdu->u.presentRequest;
1767                     *z_presentRequest->resultSetStartPoint = start;
1768                     *z_presentRequest->numberOfRecordsRequested = max;
1769                     z_presentRequest->preferredRecordSyntax =
1770                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1771                                                VAL_TEXT_XML);
1772                     z_presentRequest->recordComposition =
1773                         (Z_RecordComposition *)
1774                         odr_malloc(m_s2z_odr_search,
1775                                    sizeof(Z_RecordComposition));
1776                     if (srw_req->recordSchema)
1777                     {
1778                         z_presentRequest->recordComposition->which =
1779                             Z_RecordComp_simple;
1780                         z_presentRequest->recordComposition->u.simple =
1781                             mk_esn_from_schema(m_s2z_odr_search,
1782                                                srw_req->recordSchema);
1783                     }
1784                 }
1785             }
1786             if (!m_client)
1787             {
1788                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1789                                             Z_APDU_initRequest);
1790
1791                 // prevent m_initRequest_apdu memory from being grabbed
1792                 // in Yaz_Proxy::handle_incoming_Z_PDU
1793                 m_initRequest_apdu = m_s2z_init_apdu;
1794                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1795                 return;
1796             }
1797             else
1798             {
1799                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1800                 return;
1801             }
1802         }
1803         else if (srw_pdu->which == Z_SRW_explain_request)
1804         {
1805             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
1806
1807             // save stylesheet
1808             if (srw_req->stylesheet)
1809                 m_s2z_stylesheet =
1810                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1811
1812             if (srw_req->recordPacking &&
1813                 !strcmp(srw_req->recordPacking, "xml"))
1814                 m_s2z_packing = Z_SRW_recordPacking_XML;
1815             else
1816                 m_s2z_packing = Z_SRW_recordPacking_string;
1817
1818             if (num_diagnostic)
1819             {
1820                 send_srw_explain_response(diagnostic, num_diagnostic);
1821                 return;
1822             }
1823
1824             if (!m_client)
1825             {
1826                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1827                                             Z_APDU_initRequest);
1828
1829                 // prevent m_initRequest_apdu memory from being grabbed
1830                 // in Yaz_Proxy::handle_incoming_Z_PDU
1831                 m_initRequest_apdu = m_s2z_init_apdu;
1832                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1833             }
1834             else
1835                 send_srw_explain_response(0, 0);
1836             return;
1837         }
1838         else if (srw_pdu->which == Z_SRW_scan_request)
1839         {
1840             yaz_add_srw_diagnostic(odr_decode(),
1841                                    &diagnostic, &num_diagnostic,
1842                                    4, "scan");
1843             Z_SRW_PDU *srw_pdu =
1844                 yaz_srw_get(odr_encode(),
1845                             Z_SRW_scan_response);
1846             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1847
1848             srw_res->diagnostics = diagnostic;
1849             srw_res->num_diagnostics = num_diagnostic;
1850             send_srw_response(srw_pdu);
1851             return;
1852         }
1853         else
1854         {
1855             send_to_srw_client_error(4, 0);
1856         }
1857     }
1858     int len = 0;
1859     Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
1860     send_GDU(p, &len);
1861     timeout(1);
1862 }
1863
1864 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
1865 {
1866     if (!m_client && m_invalid_session)
1867     {
1868         m_apdu_invalid_session = apdu;
1869         m_mem_invalid_session = odr_extract_mem(odr_decode());
1870         apdu = m_initRequest_apdu;
1871     }
1872
1873     // Determine our client.
1874     Z_OtherInformation **oi;
1875     get_otherInfoAPDU(apdu, &oi);
1876     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
1877     if (!m_client)
1878     {
1879         delete this;
1880         return;
1881     }
1882     m_client->m_server = this;
1883
1884     if (apdu->which == Z_APDU_initRequest)
1885     {
1886         if (apdu->u.initRequest->implementationId)
1887             yaz_log(LOG_LOG, "%simplementationId: %s",
1888                     m_session_str, apdu->u.initRequest->implementationId);
1889         if (apdu->u.initRequest->implementationName)
1890             yaz_log(LOG_LOG, "%simplementationName: %s",
1891                     m_session_str, apdu->u.initRequest->implementationName);
1892         if (apdu->u.initRequest->implementationVersion)
1893             yaz_log(LOG_LOG, "%simplementationVersion: %s",
1894                     m_session_str, apdu->u.initRequest->implementationVersion);
1895         if (m_initRequest_apdu == 0)
1896         {
1897             if (m_initRequest_mem)
1898                 nmem_destroy(m_initRequest_mem);
1899             m_initRequest_apdu = apdu;
1900             m_initRequest_mem = odr_extract_mem(odr_decode());
1901
1902             // save init options for the response..
1903             m_initRequest_options = apdu->u.initRequest->options;
1904
1905             apdu->u.initRequest->options =
1906                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
1907                                            sizeof(Odr_bitmask));
1908             ODR_MASK_ZERO(apdu->u.initRequest->options);
1909             int i;
1910             for (i = 0; i<= 24; i++)
1911                 ODR_MASK_SET(apdu->u.initRequest->options, i);
1912             ODR_MASK_CLEAR(apdu->u.initRequest->options,
1913                            Z_Options_negotiationModel);
1914
1915             // make new version
1916             m_initRequest_version = apdu->u.initRequest->protocolVersion;
1917             apdu->u.initRequest->protocolVersion =
1918                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
1919                                            sizeof(Odr_bitmask));
1920             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
1921
1922             for (i = 0; i<= 8; i++)
1923                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
1924         }
1925         if (m_client->m_init_flag)
1926         {
1927             if (handle_init_response_for_invalid_session(apdu))
1928                 return;
1929             Z_APDU *apdu2 = m_client->m_initResponse;
1930             apdu2->u.initResponse->otherInfo = 0;
1931             if (m_client->m_cookie && *m_client->m_cookie)
1932                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
1933                                            m_client->m_cookie);
1934             apdu2->u.initResponse->referenceId =
1935                 apdu->u.initRequest->referenceId;
1936             apdu2->u.initResponse->options = m_client->m_initResponse_options;
1937             apdu2->u.initResponse->protocolVersion =
1938                 m_client->m_initResponse_version;
1939
1940             send_to_client(apdu2);
1941             return;
1942         }
1943         m_client->m_init_flag = 1;
1944     }
1945     handle_max_record_retrieve(apdu);
1946
1947     if (apdu)
1948         apdu = handle_syntax_validation(apdu);
1949
1950     if (apdu)
1951         apdu = handle_query_transformation(apdu);
1952
1953     if (apdu)
1954         apdu = handle_query_validation(apdu);
1955
1956     if (apdu)
1957         apdu = result_set_optimize(apdu);
1958     if (!apdu)
1959     {
1960         m_client->timeout(m_target_idletime);  // mark it active even
1961         // though we didn't use it
1962         return;
1963     }
1964
1965     // delete other info part from PDU before sending to target
1966     get_otherInfoAPDU(apdu, &oi);
1967     if (oi)
1968         *oi = 0;
1969
1970     if (apdu->which == Z_APDU_presentRequest &&
1971         m_client->m_resultSetStartPoint == 0)
1972     {
1973         Z_PresentRequest *pr = apdu->u.presentRequest;
1974         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
1975         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
1976     } else {
1977         m_client->m_resultSetStartPoint = 0;
1978     }
1979     if (m_client->send_to_target(apdu) < 0)
1980     {
1981         delete m_client;
1982         m_client = 0;
1983         delete this;
1984     }
1985     else
1986         m_client->m_waiting = 1;
1987 }
1988
1989 void Yaz_Proxy::connectNotify()
1990 {
1991 }
1992
1993 void Yaz_Proxy::shutdown()
1994 {
1995     m_invalid_session = 0;
1996     // only keep if keep_alive flag is set...
1997     if (m_client &&
1998         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
1999         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2000         m_client->m_waiting == 0)
2001     {
2002         yaz_log(LOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2003                  m_session_str,
2004                  m_client->get_hostname());
2005         yaz_log(LOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2006                 m_session_str, m_client->m_pdu_recv,
2007                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2008                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2009         assert (m_client->m_waiting != 2);
2010         // Tell client (if any) that no server connection is there..
2011         m_client->m_server = 0;
2012         m_invalid_session = 0;
2013     }
2014     else if (m_client)
2015     {
2016         yaz_log (LOG_LOG, "%sShutdown (client to proxy) close %s",
2017                  m_session_str,
2018                  m_client->get_hostname());
2019         assert (m_client->m_waiting != 2);
2020         delete m_client;
2021     }
2022     else if (!m_parent)
2023     {
2024         yaz_log (LOG_LOG, "%sshutdown (client to proxy) bad state",
2025                  m_session_str);
2026         assert (m_parent);
2027     }
2028     else
2029     {
2030         yaz_log (LOG_LOG, "%sShutdown (client to proxy)",
2031                  m_session_str);
2032     }
2033     if (m_parent)
2034         m_parent->pre_init();
2035     delete this;
2036 }
2037
2038 const char *Yaz_ProxyClient::get_session_str()
2039 {
2040     if (!m_server)
2041         return "0 ";
2042     return m_server->get_session_str();
2043 }
2044
2045 void Yaz_ProxyClient::shutdown()
2046 {
2047     yaz_log (LOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2048              get_hostname());
2049     delete m_server;
2050     delete this;
2051 }
2052
2053 void Yaz_Proxy::failNotify()
2054 {
2055     inc_request_no();
2056     yaz_log (LOG_LOG, "%sConnection closed by client",
2057              get_session_str());
2058     shutdown();
2059 }
2060
2061 void Yaz_ProxyClient::failNotify()
2062 {
2063     if (m_server)
2064         m_server->inc_request_no();
2065     yaz_log (LOG_LOG, "%sConnection closed by target %s",
2066              get_session_str(), get_hostname());
2067     shutdown();
2068 }
2069
2070 void Yaz_ProxyClient::connectNotify()
2071 {
2072     const char *s = get_session_str();
2073     const char *h = get_hostname();
2074     yaz_log (LOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2075              m_target_idletime);
2076     timeout(m_target_idletime);
2077     if (!m_server)
2078         pre_init_client();
2079 }
2080
2081 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2082                                                   *the_PDU_Observable, int fd)
2083 {
2084     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2085 }
2086
2087 Yaz_ProxyClient::~Yaz_ProxyClient()
2088 {
2089     if (m_prev)
2090         *m_prev = m_next;
2091     if (m_next)
2092         m_next->m_prev = m_prev;
2093     m_waiting = 2;     // for debugging purposes only.
2094     odr_destroy(m_init_odr);
2095     delete m_last_query;
2096     xfree (m_last_resultSetId);
2097     xfree (m_cookie);
2098 }
2099
2100 void Yaz_ProxyClient::pre_init_client()
2101 {
2102     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2103     Z_InitRequest *req = apdu->u.initRequest;
2104
2105     int i;
2106     for (i = 0; i<= 24; i++)
2107         ODR_MASK_SET(req->options, i);
2108     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2109                    Z_Options_negotiationModel);
2110     for (i = 0; i<= 10; i++)
2111         ODR_MASK_SET(req->protocolVersion, i);
2112
2113     if (send_to_target(apdu) < 0)
2114     {
2115         delete this;
2116     }
2117     else
2118     {
2119         m_waiting = 1;
2120         m_init_flag = 1;
2121     }
2122 }
2123
2124 void Yaz_Proxy::pre_init()
2125 {
2126     int i;
2127     const char *name = 0;
2128     const char *zurl_in_use[MAX_ZURL_PLEX];
2129     int limit_bw, limit_pdu, limit_req;
2130     int target_idletime, client_idletime;
2131     int max_clients;
2132     int keepalive_limit_bw, keepalive_limit_pdu;
2133     int pre_init;
2134     const char *cql2rpn = 0;
2135
2136     Yaz_ProxyConfig *cfg = check_reconfigure();
2137
2138     zurl_in_use[0] = 0;
2139
2140     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2141         set_APDU_yazlog(1);
2142     else
2143         set_APDU_yazlog(0);
2144
2145     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2146                                           &limit_bw, &limit_pdu, &limit_req,
2147                                           &target_idletime, &client_idletime,
2148                                           &max_clients,
2149                                           &keepalive_limit_bw,
2150                                           &keepalive_limit_pdu,
2151                                           &pre_init,
2152                                           &cql2rpn) ; i++)
2153     {
2154         if (pre_init)
2155         {
2156             int j;
2157             for (j = 0; zurl_in_use[j]; j++)
2158             {
2159                 Yaz_ProxyClient *c;
2160                 int spare = 0;
2161                 int in_use = 0;
2162                 int other = 0;
2163                 for (c = m_clientPool; c; c = c->m_next)
2164                 {
2165                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
2166                     {
2167                         if (c->m_cookie == 0)
2168                         {
2169                             if (c->m_server == 0)
2170                                 spare++;
2171                             else
2172                                 in_use++;
2173                         }
2174                         else
2175                             other++;
2176                     }
2177                 }
2178                 yaz_log(LOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2179                         "preinit=%d",m_session_str,
2180                         name, zurl_in_use[j], in_use, other, spare, pre_init);
2181                 if (spare < pre_init)
2182                 {
2183                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2184                     c->m_next = m_clientPool;
2185                     if (c->m_next)
2186                         c->m_next->m_prev = &c->m_next;
2187                     m_clientPool = c;
2188                     c->m_prev = &m_clientPool;
2189
2190                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
2191                         c->set_APDU_yazlog(1);
2192                     else
2193                         c->set_APDU_yazlog(0);
2194
2195                     if (c->client(zurl_in_use[j]))
2196                     {
2197                         timeout(60);
2198                         delete c;
2199                         return;
2200                     }
2201                     c->timeout(30);
2202                     c->m_waiting = 1;
2203                     c->m_target_idletime = target_idletime;
2204                     c->m_seqno = m_seqno++;
2205                 }
2206             }
2207         }
2208     }
2209 }
2210
2211 void Yaz_Proxy::timeoutNotify()
2212 {
2213     if (m_parent)
2214     {
2215         if (m_bw_hold_PDU)
2216         {
2217             timeout(m_client_idletime);
2218             Z_GDU *apdu = m_bw_hold_PDU;
2219             m_bw_hold_PDU = 0;
2220
2221             if (apdu->which == Z_GDU_Z3950)
2222                 handle_incoming_Z_PDU(apdu->u.z3950);
2223             else if (apdu->which == Z_GDU_HTTP_Request)
2224                 handle_incoming_HTTP(apdu->u.HTTP_Request);
2225         }
2226         else
2227         {
2228             inc_request_no();
2229
2230             yaz_log (LOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2231             shutdown();
2232         }
2233     }
2234     else
2235     {
2236         timeout(600);
2237         pre_init();
2238     }
2239 }
2240
2241 void Yaz_Proxy::markInvalid()
2242 {
2243     m_client = 0;
2244     m_invalid_session = 1;
2245 }
2246
2247 void Yaz_ProxyClient::timeoutNotify()
2248 {
2249     if (m_server)
2250         m_server->inc_request_no();
2251
2252     yaz_log (LOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2253              get_hostname());
2254     m_waiting = 1;
2255     m_root->pre_init();
2256     if (m_server && m_init_flag)
2257     {
2258         // target timed out in a session that was properly initialized
2259         // server object stay alive but we mark it as invalid so it
2260         // gets initialized again
2261         m_server->markInvalid();
2262         m_server = 0;
2263     }
2264     shutdown();
2265 }
2266
2267 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2268                                  Yaz_Proxy *parent) :
2269     Yaz_Z_Assoc (the_PDU_Observable)
2270 {
2271     m_cookie = 0;
2272     m_next = 0;
2273     m_prev = 0;
2274     m_init_flag = 0;
2275     m_last_query = 0;
2276     m_last_resultSetId = 0;
2277     m_last_resultCount = 0;
2278     m_last_ok = 0;
2279     m_sr_transform = 0;
2280     m_waiting = 0;
2281     m_init_odr = odr_createmem (ODR_DECODE);
2282     m_initResponse = 0;
2283     m_initResponse_options = 0;
2284     m_initResponse_version = 0;
2285     m_resultSetStartPoint = 0;
2286     m_bytes_sent = m_bytes_recv = 0;
2287     m_pdu_recv = 0;
2288     m_server = 0;
2289     m_seqno = 0;
2290     m_target_idletime = 600;
2291     m_root = parent;
2292 }
2293
2294 const char *Yaz_Proxy::option(const char *name, const char *value)
2295 {
2296     if (!strcmp (name, "optimize")) {
2297         if (value) {
2298             xfree (m_optimize);
2299             m_optimize = xstrdup (value);
2300         }
2301         return m_optimize;
2302     }
2303     return 0;
2304 }
2305
2306 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2307 {
2308
2309 }
2310
2311 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2312 {
2313     if (apdu->which == Z_GDU_Z3950)
2314         recv_Z_PDU(apdu->u.z3950, len);
2315     else if (apdu->which == Z_GDU_HTTP_Response)
2316         recv_HTTP_response(apdu->u.HTTP_Response, len);
2317     else
2318         shutdown();
2319 }
2320
2321 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2322 {
2323     if (!m_invalid_session)
2324         return 0;
2325     m_invalid_session = 0;
2326     handle_incoming_Z_PDU(m_apdu_invalid_session);
2327     assert (m_mem_invalid_session);
2328     nmem_destroy(m_mem_invalid_session);
2329     m_mem_invalid_session = 0;
2330     return 1;
2331 }
2332
2333 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2334 {
2335     m_bytes_recv += len;
2336     m_pdu_recv++;
2337     m_waiting = 0;
2338     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2339         yaz_log (LOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2340                  apdu_name(apdu), get_hostname(), len);
2341     if (apdu->which == Z_APDU_initResponse)
2342     {
2343         if (!m_server)  // if this is a pre init session , check for more
2344             m_root->pre_init();
2345         NMEM nmem = odr_extract_mem (odr_decode());
2346         odr_reset (m_init_odr);
2347         nmem_transfer (m_init_odr->mem, nmem);
2348         m_initResponse = apdu;
2349         m_initResponse_options = apdu->u.initResponse->options;
2350         m_initResponse_version = apdu->u.initResponse->protocolVersion;
2351
2352         Z_InitResponse *ir = apdu->u.initResponse;
2353         char *im0 = ir->implementationName;
2354
2355         char *im1 = (char*)
2356             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2357         *im1 = '\0';
2358         if (im0)
2359         {
2360             strcat(im1, im0);
2361             strcat(im1, " ");
2362         }
2363         strcat(im1, "(YAZ Proxy)");
2364         ir->implementationName = im1;
2365
2366         nmem_destroy (nmem);
2367
2368         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2369             return;
2370     }
2371     if (apdu->which == Z_APDU_searchResponse)
2372     {
2373         Z_SearchResponse *sr = apdu->u.searchResponse;
2374         m_last_resultCount = *sr->resultCount;
2375         int status = *sr->searchStatus;
2376         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2377         {
2378             m_last_ok = 1;
2379
2380             if (sr->records && sr->records->which == Z_Records_DBOSD)
2381             {
2382                 m_cache.add(odr_decode(),
2383                             sr->records->u.databaseOrSurDiagnostics, 1,
2384                             *sr->resultCount);
2385             }
2386         }
2387     }
2388     if (apdu->which == Z_APDU_presentResponse)
2389     {
2390         Z_PresentResponse *pr = apdu->u.presentResponse;
2391         if (m_sr_transform)
2392         {
2393             m_sr_transform = 0;
2394             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2395             Z_SearchResponse *sr = new_apdu->u.searchResponse;
2396             sr->referenceId = pr->referenceId;
2397             *sr->resultCount = m_last_resultCount;
2398             sr->records = pr->records;
2399             sr->nextResultSetPosition = pr->nextResultSetPosition;
2400             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2401             apdu = new_apdu;
2402         }
2403         if (pr->records &&
2404             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2405         {
2406             m_cache.add(odr_decode(),
2407                         pr->records->u.databaseOrSurDiagnostics,
2408                         m_resultSetStartPoint, -1);
2409             m_resultSetStartPoint = 0;
2410         }
2411     }
2412     if (m_cookie)
2413         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2414     if (m_server)
2415     {
2416         m_server->send_to_client(apdu);
2417     }
2418     if (apdu->which == Z_APDU_close)
2419     {
2420         shutdown();
2421     }
2422 }
2423
2424 int Yaz_Proxy::server(const char *addr)
2425 {
2426     int r = Yaz_Z_Assoc::server(addr);
2427     if (!r)
2428     {
2429         yaz_log(LOG_LOG, "%sStarted proxy " VERSION " on %s", m_session_str, addr);
2430         timeout(1);
2431     }
2432     return r;
2433 }
2434