src/yaz-proxy.cpp

   1 /*
   2  * Copyright (c) 1998-2004, Index Data.
   3  * See the file LICENSE for details.
   4  *
   5  * $Id: yaz-proxy.cpp,v 1.95 2004-01-30 12:02:21 adam Exp $
   6  */
   7
   8 #include <assert.h>
   9 #include <time.h>
  10
  11 #include <yaz/srw.h>
  12 #include <yaz/marcdisp.h>
  13 #include <yaz/yaz-iconv.h>
  14 #include <yaz/log.h>
  15 #include <yaz/diagbib1.h>
  16 #include <yaz++/proxy.h>
  17 #include <yaz/pquery.h>
  18
  19 static const char *apdu_name(Z_APDU *apdu)
  20 {
  21     switch (apdu->which)
  22     {
  23     case Z_APDU_initRequest:
  24         return "initRequest";
  25     case Z_APDU_initResponse:
  26         return "initResponse";
  27     case Z_APDU_searchRequest:
  28         return "searchRequest";
  29     case Z_APDU_searchResponse:
  30         return "searchResponse";
  31     case Z_APDU_presentRequest:
  32         return "presentRequest";
  33     case Z_APDU_presentResponse:
  34         return "presentResponse";
  35     case Z_APDU_deleteResultSetRequest:
  36         return "deleteResultSetRequest";
  37     case Z_APDU_deleteResultSetResponse:
  38         return "deleteResultSetResponse";
  39     case Z_APDU_scanRequest:
  40         return "scanRequest";
  41     case Z_APDU_scanResponse:
  42         return "scanResponse";
  43     case Z_APDU_sortRequest:
  44         return "sortRequest";
  45     case Z_APDU_sortResponse:
  46         return "sortResponse";
  47     case Z_APDU_extendedServicesRequest:
  48         return "extendedServicesRequest";
  49     case Z_APDU_extendedServicesResponse:
  50         return "extendedServicesResponse";
  51     case Z_APDU_close:
  52         return "close";
  53     }
  54     return "other";
  55 }
  56
  57 static const char *gdu_name(Z_GDU *gdu)
  58 {
  59     switch(gdu->which)
  60     {
  61     case Z_GDU_Z3950:
  62         return apdu_name(gdu->u.z3950);
  63     case Z_GDU_HTTP_Request:
  64         return "HTTP Request";
  65     case Z_GDU_HTTP_Response:
  66         return "HTTP Response";
  67     }
  68     return "Unknown request/response";
  69 }
  70
  71 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
  72                      Yaz_Proxy *parent) :
  73     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
  74 {
  75     m_PDU_Observable = the_PDU_Observable;
  76     m_client = 0;
  77     m_parent = parent;
  78     m_clientPool = 0;
  79     m_seqno = 1;
  80     m_keepalive_limit_bw = 500000;
  81     m_keepalive_limit_pdu = 1000;
  82     m_proxyTarget = 0;
  83     m_default_target = 0;
  84     m_proxy_authentication = 0;
  85     m_max_clients = 150;
  86     m_log_mask = 0;
  87     m_seed = time(0);
  88     m_client_idletime = 600;
  89     m_target_idletime = 600;
  90     m_optimize = xstrdup ("1");
  91     strcpy(m_session_str, "0 ");
  92     m_session_no=0;
  93     m_bytes_sent = 0;
  94     m_bytes_recv = 0;
  95     m_bw_hold_PDU = 0;
  96     m_bw_max = 0;
  97     m_pdu_max = 0;
  98     m_max_record_retrieve = 0;
  99     m_reconfig_flag = 0;
 100     m_config_fname = 0;
 101     m_request_no = 0;
 102     m_invalid_session = 0;
 103     m_config = 0;
 104     m_marcxml_flag = 0;
 105     m_stylesheet_xsp = 0;
 106     m_s2z_stylesheet = 0;
 107     m_s2z_database = 0;
 108     m_schema = 0;
 109     m_initRequest_apdu = 0;
 110     m_initRequest_mem = 0;
 111     m_initRequest_options = 0;
 112     m_initRequest_version = 0;
 113     m_apdu_invalid_session = 0;
 114     m_mem_invalid_session = 0;
 115     m_s2z_odr_init = 0;
 116     m_s2z_odr_search = 0;
 117     m_s2z_init_apdu = 0;
 118     m_s2z_search_apdu = 0;
 119     m_s2z_present_apdu = 0;
 120     m_http_keepalive = 0;
 121     m_http_version = 0;
 122     m_soap_ns = 0;
 123     m_s2z_packing = Z_SRW_recordPacking_string;
 124     m_time_tv.tv_sec = 0;
 125     m_time_tv.tv_usec = 0;
 126 }
 127
 128 Yaz_Proxy::~Yaz_Proxy()
 129 {
 130     yaz_log(LOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
 131             m_bytes_sent, m_bytes_recv);
 132     nmem_destroy(m_initRequest_mem);
 133     nmem_destroy(m_mem_invalid_session);
 134     xfree (m_proxyTarget);
 135     xfree (m_default_target);
 136     xfree (m_proxy_authentication);
 137     xfree (m_optimize);
 138
 139     if (m_stylesheet_xsp)
 140         xsltFreeStylesheet(m_stylesheet_xsp);
 141
 142     xfree (m_schema);
 143     if (m_s2z_odr_init)
 144         odr_destroy(m_s2z_odr_init);
 145     if (m_s2z_odr_search)
 146         odr_destroy(m_s2z_odr_search);
 147     delete m_config;
 148 }
 149
 150 int Yaz_Proxy::set_config(const char *config)
 151 {
 152     delete m_config;
 153     m_config = new Yaz_ProxyConfig();
 154     xfree(m_config_fname);
 155     m_config_fname = xstrdup(config);
 156     int r = m_config->read_xml(config);
 157     if (!r)
 158         m_config->get_generic_info(&m_log_mask, &m_max_clients);
 159     return r;
 160 }
 161
 162 void Yaz_Proxy::set_default_target(const char *target)
 163 {
 164     xfree (m_default_target);
 165     m_default_target = 0;
 166     if (target)
 167         m_default_target = (char *) xstrdup (target);
 168 }
 169
 170 void Yaz_Proxy::set_proxy_authentication (const char *auth)
 171 {
 172     xfree (m_proxy_authentication);
 173     m_proxy_authentication = 0;
 174     if (auth)
 175         m_proxy_authentication = (char *) xstrdup (auth);
 176 }
 177
 178 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
 179 {
 180     if (m_parent)
 181         return m_parent->check_reconfigure();
 182
 183     Yaz_ProxyConfig *cfg = m_config;
 184     if (m_reconfig_flag)
 185     {
 186         yaz_log(LOG_LOG, "reconfigure");
 187         yaz_log_reopen();
 188         if (m_config_fname && cfg)
 189         {
 190             yaz_log(LOG_LOG, "reconfigure config %s", m_config_fname);
 191             int r = cfg->read_xml(m_config_fname);
 192             if (r)
 193                 yaz_log(LOG_WARN, "reconfigure failed");
 194             else
 195             {
 196                 m_log_mask = 0;
 197                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
 198             }
 199         }
 200         else
 201             yaz_log(LOG_LOG, "reconfigure");
 202         m_reconfig_flag = 0;
 203     }
 204     return cfg;
 205 }
 206
 207 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
 208                                             *the_PDU_Observable, int fd)
 209 {
 210     check_reconfigure();
 211     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
 212     new_proxy->m_config = 0;
 213     new_proxy->m_config_fname = 0;
 214     new_proxy->timeout(m_client_idletime);
 215     new_proxy->m_target_idletime = m_target_idletime;
 216     new_proxy->set_default_target(m_default_target);
 217     new_proxy->m_max_clients = m_max_clients;
 218     new_proxy->m_log_mask = m_log_mask;
 219     new_proxy->set_APDU_log(get_APDU_log());
 220     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
 221         new_proxy->set_APDU_yazlog(1);
 222     else
 223         new_proxy->set_APDU_yazlog(0);
 224     new_proxy->set_proxy_authentication(m_proxy_authentication);
 225     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
 226     m_session_no++;
 227     yaz_log (LOG_LOG, "%sNew session %s", new_proxy->m_session_str,
 228              the_PDU_Observable->getpeername());
 229     return new_proxy;
 230 }
 231
 232 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 233 {
 234     int oid[OID_SIZE];
 235     Z_OtherInformationUnit *oi;
 236     struct oident ent;
 237     ent.proto = PROTO_Z3950;
 238     ent.oclass = CLASS_USERINFO;
 239     ent.value = (oid_value) VAL_COOKIE;
 240     assert (oid_ent_to_oid (&ent, oid));
 241
 242     if (oid_ent_to_oid (&ent, oid) &&
 243         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 244         oi->which == Z_OtherInfo_characterInfo)
 245         return oi->information.characterInfo;
 246     return 0;
 247 }
 248
 249 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 250 {
 251     int oid[OID_SIZE];
 252     Z_OtherInformationUnit *oi;
 253     struct oident ent;
 254     ent.proto = PROTO_Z3950;
 255     ent.oclass = CLASS_USERINFO;
 256     ent.value = (oid_value) VAL_PROXY;
 257     if (oid_ent_to_oid (&ent, oid) &&
 258         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
 259         oi->which == Z_OtherInfo_characterInfo)
 260         return oi->information.characterInfo;
 261     return 0;
 262 }
 263
 264 const char *Yaz_Proxy::load_balance(const char **url)
 265 {
 266     int zurl_in_use[MAX_ZURL_PLEX];
 267     Yaz_ProxyClient *c;
 268     int i;
 269
 270     for (i = 0; i<MAX_ZURL_PLEX; i++)
 271         zurl_in_use[i] = 0;
 272     for (c = m_parent->m_clientPool; c; c = c->m_next)
 273     {
 274         for (i = 0; url[i]; i++)
 275             if (!strcmp(url[i], c->get_hostname()))
 276                 zurl_in_use[i]++;
 277     }
 278     int min = 100000;
 279     const char *ret = 0;
 280     for (i = 0; url[i]; i++)
 281     {
 282         yaz_log(LOG_DEBUG, "%szurl=%s use=%d",
 283                 m_session_str, url[i], zurl_in_use[i]);
 284         if (min > zurl_in_use[i])
 285         {
 286             ret = url[i];
 287             min = zurl_in_use[i];
 288         }
 289     }
 290     return ret;
 291 }
 292
 293 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
 294                                        const char *proxy_host)
 295 {
 296     assert (m_parent);
 297     Yaz_Proxy *parent = m_parent;
 298     Yaz_ProxyClient *c = m_client;
 299
 300     if (!m_proxyTarget)
 301     {
 302         const char *url[MAX_ZURL_PLEX];
 303         Yaz_ProxyConfig *cfg = check_reconfigure();
 304         if (proxy_host)
 305         {
 306 #if 1
 307 /* only to be enabled for debugging... */
 308             if (!strcmp(proxy_host, "stop"))
 309                 exit(0);
 310 #endif
 311             xfree(m_default_target);
 312             m_default_target = xstrdup(proxy_host);
 313             proxy_host = m_default_target;
 314         }
 315         int client_idletime = -1;
 316         const char *cql2rpn_fname = 0;
 317         url[0] = m_default_target;
 318         url[1] = 0;
 319         if (cfg)
 320         {
 321             int pre_init = 0;
 322             cfg->get_target_info(proxy_host, url, &m_bw_max,
 323                                  &m_pdu_max, &m_max_record_retrieve,
 324                                  &m_target_idletime, &client_idletime,
 325                                  &parent->m_max_clients,
 326                                  &m_keepalive_limit_bw,
 327                                  &m_keepalive_limit_pdu,
 328                                  &pre_init,
 329                                  &cql2rpn_fname);
 330         }
 331         if (client_idletime != -1)
 332         {
 333             m_client_idletime = client_idletime;
 334             timeout(m_client_idletime);
 335         }
 336         if (cql2rpn_fname)
 337             m_cql2rpn.set_pqf_file(cql2rpn_fname);
 338         if (!url[0])
 339         {
 340             yaz_log(LOG_LOG, "%sNo default target", m_session_str);
 341             return 0;
 342         }
 343         // we don't handle multiplexing for cookie session, so we just
 344         // pick the first one in this case (anonymous users will be able
 345         // to use any backend)
 346         if (cookie && *cookie)
 347             m_proxyTarget = (char*) xstrdup(url[0]);
 348         else
 349             m_proxyTarget = (char*) xstrdup(load_balance(url));
 350     }
 351     if (cookie && *cookie)
 352     {   // search in sessions with a cookie
 353         for (c = parent->m_clientPool; c; c = c->m_next)
 354         {
 355             assert (c->m_prev);
 356             assert (*c->m_prev == c);
 357             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
 358                 !strcmp(m_proxyTarget, c->get_hostname()))
 359             {
 360                 // Found it in cache
 361                 // The following handles "cancel"
 362                 // If connection is busy (waiting for PDU) and
 363                 // we have an initRequest we can safely do re-open
 364                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
 365                 {
 366                     yaz_log (LOG_LOG, "%s REOPEN target=%s", m_session_str,
 367                              c->get_hostname());
 368                     c->close();
 369                     c->m_init_flag = 0;
 370
 371                     c->m_last_ok = 0;
 372                     c->m_cache.clear();
 373                     c->m_last_resultCount = 0;
 374                     c->m_sr_transform = 0;
 375                     c->m_waiting = 0;
 376                     c->m_resultSetStartPoint = 0;
 377                     c->m_target_idletime = m_target_idletime;
 378                     if (c->client(m_proxyTarget))
 379                     {
 380                         delete c;
 381                         return 0;
 382                     }
 383                     c->timeout(30);
 384                 }
 385                 c->m_seqno = parent->m_seqno;
 386                 if (c->m_server && c->m_server != this)
 387                     c->m_server->m_client = 0;
 388                 c->m_server = this;
 389                 (parent->m_seqno)++;
 390                 yaz_log (LOG_DEBUG, "get_client 1 %p %p", this, c);
 391                 return c;
 392             }
 393         }
 394     }
 395     else if (!c)
 396     {
 397         // don't have a client session yet. Search in session w/o cookie
 398         for (c = parent->m_clientPool; c; c = c->m_next)
 399         {
 400             assert (c->m_prev);
 401             assert (*c->m_prev == c);
 402             if (c->m_server == 0 && c->m_cookie == 0 &&
 403                 c->m_waiting == 0 &&
 404                 !strcmp(m_proxyTarget, c->get_hostname()))
 405             {
 406                 // found it in cache
 407                 yaz_log (LOG_LOG, "%sREUSE %s",
 408                          m_session_str, c->get_hostname());
 409
 410                 c->m_seqno = parent->m_seqno;
 411                 assert(c->m_server == 0);
 412                 c->m_server = this;
 413
 414                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 415                     c->set_APDU_yazlog(1);
 416                 else
 417                     c->set_APDU_yazlog(0);
 418
 419                 (parent->m_seqno)++;
 420
 421                 parent->pre_init();
 422
 423                 return c;
 424             }
 425         }
 426     }
 427     if (!m_client)
 428     {
 429         if (apdu->which != Z_APDU_initRequest)
 430         {
 431             yaz_log (LOG_LOG, "%sno init request as first PDU", m_session_str);
 432             return 0;
 433         }
 434         Z_InitRequest *initRequest = apdu->u.initRequest;
 435
 436         if (!initRequest->idAuthentication)
 437         {
 438             if (m_proxy_authentication)
 439             {
 440                 initRequest->idAuthentication =
 441                     (Z_IdAuthentication *)
 442                     odr_malloc (odr_encode(),
 443                                 sizeof(*initRequest->idAuthentication));
 444                 initRequest->idAuthentication->which =
 445                     Z_IdAuthentication_open;
 446                 initRequest->idAuthentication->u.open =
 447                     odr_strdup (odr_encode(), m_proxy_authentication);
 448             }
 449         }
 450         // go through list of clients - and find the lowest/oldest one.
 451         Yaz_ProxyClient *c_min = 0;
 452         int min_seq = -1;
 453         int no_of_clients = 0;
 454         if (parent->m_clientPool)
 455             yaz_log (LOG_DEBUG, "Existing sessions");
 456         for (c = parent->m_clientPool; c; c = c->m_next)
 457         {
 458             yaz_log (LOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
 459                                c->m_waiting, c->get_hostname(),
 460                                c->m_cookie ? c->m_cookie : "");
 461             no_of_clients++;
 462             if (min_seq < 0 || c->m_seqno < min_seq)
 463             {
 464                 min_seq = c->m_seqno;
 465                 c_min = c;
 466             }
 467         }
 468         if (no_of_clients >= parent->m_max_clients)
 469         {
 470             c = c_min;
 471             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
 472             {
 473                 yaz_log (LOG_LOG, "%sMAXCLIENTS Destroy %d",
 474                          m_session_str, c->m_seqno);
 475                 if (c->m_server && c->m_server != this)
 476                     delete c->m_server;
 477                 c->m_server = 0;
 478             }
 479             else
 480             {
 481                 yaz_log (LOG_LOG, "%sMAXCLIENTS Reuse %d %d %s",
 482                          m_session_str,
 483                          c->m_seqno, parent->m_seqno, c->get_hostname());
 484                 xfree (c->m_cookie);
 485                 c->m_cookie = 0;
 486                 if (cookie)
 487                     c->m_cookie = xstrdup(cookie);
 488                 c->m_seqno = parent->m_seqno;
 489                 if (c->m_server && c->m_server != this)
 490                 {
 491                     c->m_server->m_client = 0;
 492                     delete c->m_server;
 493                 }
 494                 (parent->m_seqno)++;
 495                 c->m_target_idletime = m_target_idletime;
 496                 c->timeout(m_target_idletime);
 497
 498                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 499                     c->set_APDU_yazlog(1);
 500                 else
 501                     c->set_APDU_yazlog(0);
 502
 503                 return c;
 504             }
 505         }
 506         else
 507         {
 508             yaz_log (LOG_LOG, "%sNEW %d %s",
 509                      m_session_str, parent->m_seqno, m_proxyTarget);
 510             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
 511             c->m_next = parent->m_clientPool;
 512             if (c->m_next)
 513                 c->m_next->m_prev = &c->m_next;
 514             parent->m_clientPool = c;
 515             c->m_prev = &parent->m_clientPool;
 516         }
 517
 518         xfree (c->m_cookie);
 519         c->m_cookie = 0;
 520         if (cookie)
 521             c->m_cookie = xstrdup(cookie);
 522
 523         c->m_seqno = parent->m_seqno;
 524         c->m_init_flag = 0;
 525         c->m_last_resultCount = 0;
 526         c->m_last_ok = 0;
 527         c->m_cache.clear();
 528         c->m_sr_transform = 0;
 529         c->m_waiting = 0;
 530         c->m_resultSetStartPoint = 0;
 531         (parent->m_seqno)++;
 532         if (c->client(m_proxyTarget))
 533         {
 534             delete c;
 535             return 0;
 536         }
 537         c->m_target_idletime = m_target_idletime;
 538         c->timeout(30);
 539
 540         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
 541             c->set_APDU_yazlog(1);
 542         else
 543             c->set_APDU_yazlog(0);
 544     }
 545     yaz_log (LOG_DEBUG, "get_client 3 %p %p", this, c);
 546     return c;
 547 }
 548
 549 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 550 {
 551     int i;
 552     for (i = 0; i<num; i++)
 553     {
 554         oident *ent;
 555         Z_DefaultDiagFormat *r;
 556         Z_DiagRec *p = pp[i];
 557         if (p->which != Z_DiagRec_defaultFormat)
 558         {
 559             yaz_log(LOG_LOG, "%sError no diagnostics", m_session_str);
 560             return;
 561         }
 562         else
 563             r = p->u.defaultFormat;
 564         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
 565             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
 566             yaz_log(LOG_LOG, "%sError unknown diagnostic set", m_session_str);
 567         switch (r->which)
 568         {
 569         case Z_DefaultDiagFormat_v2Addinfo:
 570             yaz_log(LOG_LOG, "%sError %d %s:%s",
 571                     m_session_str,
 572                     *r->condition, diagbib1_str(*r->condition),
 573                     r->u.v2Addinfo);
 574             break;
 575         case Z_DefaultDiagFormat_v3Addinfo:
 576             yaz_log(LOG_LOG, "%sError %d %s:%s",
 577                     m_session_str,
 578                     *r->condition, diagbib1_str(*r->condition),
 579                     r->u.v3Addinfo);
 580             break;
 581         }
 582     }
 583 }
 584
 585 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 586 {
 587     if (!m_stylesheet_xsp || p->num_records <= 0)
 588         return 0;  /* no XSLT to be done ... */
 589
 590     m_stylesheet_offset = 0;
 591     m_stylesheet_nprl = p;
 592     m_stylesheet_apdu = apdu;
 593     timeout(0);
 594     return 1;
 595 }
 596
 597 void Yaz_Proxy::convert_xsl_delay()
 598 {
 599     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 600     if (npr->which == Z_NamePlusRecord_databaseRecord)
 601     {
 602         Z_External *r = npr->u.databaseRecord;
 603         if (r->which == Z_External_octet)
 604         {
 605             xmlDocPtr res, doc = xmlParseMemory(
 606                 (char*) r->u.octet_aligned->buf,
 607                 r->u.octet_aligned->len);
 608
 609             yaz_log(LOG_LOG, "%sXSLT convert %d",
 610                     m_session_str, m_stylesheet_offset);
 611             res = xsltApplyStylesheet(m_stylesheet_xsp, doc, 0);
 612             if (res)
 613             {
 614                 xmlChar *out_buf;
 615                 int out_len;
 616                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
 617
 618                 m_stylesheet_nprl->records[m_stylesheet_offset]->
 619                     u.databaseRecord =
 620                     z_ext_record(odr_encode(), VAL_TEXT_XML,
 621                                  (char*) out_buf, out_len);
 622                 xmlFree(out_buf);
 623                 xmlFreeDoc(res);
 624             }
 625             xmlFreeDoc(doc);
 626         }
 627     }
 628     m_stylesheet_offset++;
 629     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
 630     {
 631         m_stylesheet_nprl = 0;
 632         if (m_stylesheet_xsp)
 633             xsltFreeStylesheet(m_stylesheet_xsp);
 634         m_stylesheet_xsp = 0;
 635         timeout(m_client_idletime);
 636         int r = send_PDU_convert(m_stylesheet_apdu);
 637     }
 638     else
 639         timeout(0);
 640 }
 641
 642 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p)
 643 {
 644     int i;
 645
 646     yaz_marc_t mt = yaz_marc_create();
 647     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
 648     for (i = 0; i < p->num_records; i++)
 649     {
 650         Z_NamePlusRecord *npr = p->records[i];
 651         if (npr->which == Z_NamePlusRecord_databaseRecord)
 652         {
 653             Z_External *r = npr->u.databaseRecord;
 654             if (r->which == Z_External_octet)
 655             {
 656                 int rlen;
 657                 char *result;
 658                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
 659                                         r->u.octet_aligned->len,
 660                                         &result, &rlen))
 661                 {
 662                     yaz_iconv_t cd = yaz_iconv_open("UTF-8", "MARC-8");
 663                     WRBUF wrbuf = wrbuf_alloc();
 664
 665                     char outbuf[120];
 666                     size_t inbytesleft = rlen;
 667                     const char *inp = result;
 668                     while (cd && inbytesleft)
 669                     {
 670                         size_t outbytesleft = sizeof(outbuf);
 671                         char *outp = outbuf;
 672                         size_t r;
 673
 674                         r = yaz_iconv (cd, (char**) &inp,
 675                                        &inbytesleft,
 676                                        &outp, &outbytesleft);
 677                         if (r == (size_t) (-1))
 678                         {
 679                             int e = yaz_iconv_error(cd);
 680                             if (e != YAZ_ICONV_E2BIG)
 681                             {
 682                                 yaz_log(LOG_WARN, "conversion failure");
 683                                 break;
 684                             }
 685                         }
 686                         wrbuf_write(wrbuf, outbuf, outp - outbuf);
 687                     }
 688                     if (cd)
 689                         yaz_iconv_close(cd);
 690
 691                     npr->u.databaseRecord = z_ext_record(odr_encode(),
 692                                                          VAL_TEXT_XML,
 693                                                          wrbuf_buf(wrbuf),
 694                                                          wrbuf_len(wrbuf));
 695                     wrbuf_free(wrbuf, 1);
 696                 }
 697             }
 698         }
 699     }
 700     yaz_marc_destroy(mt);
 701 }
 702
 703 void Yaz_Proxy::logtime()
 704 {
 705     if (m_time_tv.tv_sec)
 706     {
 707         struct timeval tv;
 708         gettimeofday(&tv, 0);
 709         long diff = (tv.tv_sec - m_time_tv.tv_sec)*1000000 +
 710             (tv.tv_usec - m_time_tv.tv_usec);
 711         if (diff >= 0)
 712             yaz_log(LOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
 713                     diff/1000000, (diff/1000)%1000);
 714     }
 715     m_time_tv.tv_sec = 0;
 716     m_time_tv.tv_usec = 0;
 717 }
 718
 719 int Yaz_Proxy::send_http_response(int code)
 720 {
 721     ODR o = odr_encode();
 722     Z_GDU *gdu = z_get_HTTP_Response(o, code);
 723     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 724     if (m_http_version)
 725         hres->version = odr_strdup(o, m_http_version);
 726     if (m_http_keepalive)
 727         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 728     else
 729         timeout(0);
 730
 731     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 732     {
 733         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 734                  gdu_name(gdu));
 735     }
 736     int len;
 737     int r = send_GDU(gdu, &len);
 738     m_bytes_sent += len;
 739     m_bw_stat.add_bytes(len);
 740     logtime();
 741     return r;
 742 }
 743
 744 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
 745 {
 746     ODR o = odr_encode();
 747     const char *ctype = "text/xml";
 748     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
 749     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 750     if (m_http_version)
 751         hres->version = odr_strdup(o, m_http_version);
 752     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
 753     if (m_http_keepalive)
 754         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
 755     else
 756         timeout(0);
 757
 758     static Z_SOAP_Handler soap_handlers[2] = {
 759 #if HAVE_XSLT
 760         {"http://www.loc.gov/zing/srw/", 0,
 761          (Z_SOAP_fun) yaz_srw_codec},
 762 #endif
 763         {0, 0, 0}
 764     };
 765
 766     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
 767     soap_package->which = Z_SOAP_generic;
 768     soap_package->u.generic =
 769         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
 770     soap_package->u.generic->no = 0;
 771     soap_package->u.generic->ns = soap_handlers[0].ns;
 772     soap_package->u.generic->p = (void *) srw_pdu;
 773     soap_package->ns = m_soap_ns;
 774     z_soap_codec_enc_xsl(o, &soap_package,
 775                          &hres->content_buf, &hres->content_len,
 776                          soap_handlers, 0, m_s2z_stylesheet);
 777     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 778     {
 779         yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 780                  gdu_name(gdu));
 781     }
 782     int len;
 783     int r = send_GDU(gdu, &len);
 784     m_bytes_sent += len;
 785     m_bw_stat.add_bytes(len);
 786     logtime();
 787     return r;
 788 }
 789
 790 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
 791 {
 792     ODR o = odr_encode();
 793     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 794     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 795
 796     srw_res->num_diagnostics = 1;
 797     srw_res->diagnostics = (Z_SRW_diagnostic *)
 798         odr_malloc(o, sizeof(*srw_res->diagnostics));
 799     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
 800     return send_srw_response(srw_pdu);
 801 }
 802
 803 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
 804                              Z_DefaultDiagFormat *ddf)
 805 {
 806     int bib1_code = *ddf->condition;
 807     if (bib1_code == 109)
 808         return 404;
 809     srw_res->num_diagnostics = 1;
 810     srw_res->diagnostics = (Z_SRW_diagnostic *)
 811         odr_malloc(o, sizeof(*srw_res->diagnostics));
 812     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
 813                           yaz_diag_bib1_to_srw(*ddf->condition),
 814                           ddf->u.v2Addinfo);
 815     return 0;
 816 }
 817
 818 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
 819 {
 820     ODR o = odr_encode();
 821     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
 822     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
 823
 824     srw_res->numberOfRecords = odr_intdup (o, hits);
 825     if (records && records->which == Z_Records_DBOSD)
 826     {
 827         srw_res->num_records =
 828             records->u.databaseOrSurDiagnostics->num_records;
 829         int i;
 830         srw_res->records = (Z_SRW_record *)
 831             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
 832         for (i = 0; i < srw_res->num_records; i++)
 833         {
 834             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
 835             if (npr->which != Z_NamePlusRecord_databaseRecord)
 836             {
 837                 srw_res->records[i].recordSchema = "diagnostic";
 838                 srw_res->records[i].recordPacking = m_s2z_packing;
 839                 srw_res->records[i].recordData_buf = "67";
 840                 srw_res->records[i].recordData_len = 2;
 841                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 842                 continue;
 843             }
 844             Z_External *r = npr->u.databaseRecord;
 845             oident *ent = oid_getentbyoid(r->direct_reference);
 846             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
 847             {
 848                 srw_res->records[i].recordSchema = m_schema;
 849                 srw_res->records[i].recordPacking = m_s2z_packing;
 850                 srw_res->records[i].recordData_buf = (char*)
 851                     r->u.octet_aligned->buf;
 852                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
 853                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 854             }
 855             else
 856             {
 857                 srw_res->records[i].recordSchema = "diagnostic";
 858                 srw_res->records[i].recordPacking = m_s2z_packing;
 859                 srw_res->records[i].recordData_buf = "67";
 860                 srw_res->records[i].recordData_len = 2;
 861                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
 862             }
 863         }
 864     }
 865     if (records && records->which == Z_Records_NSD)
 866     {
 867         int http_code;
 868         http_code = z_to_srw_diag(odr_encode(), srw_res,
 869                                    records->u.nonSurrogateDiagnostic);
 870         if (http_code)
 871             return send_http_response(http_code);
 872     }
 873     return send_srw_response(srw_pdu);
 874
 875 }
 876
 877 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
 878                                         int num_diagnostics)
 879 {
 880     Yaz_ProxyConfig *cfg = check_reconfigure();
 881     if (cfg)
 882     {
 883         int len;
 884         char *b = cfg->get_explain(odr_encode(), 0 /* target */,
 885                                    m_s2z_database, &len);
 886         if (b)
 887         {
 888             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
 889             Z_SRW_explainResponse *er = res->u.explain_response;
 890
 891             er->record.recordData_buf = b;
 892             er->record.recordData_len = len;
 893             er->record.recordPacking = m_s2z_packing;
 894
 895             er->diagnostics = diagnostics;
 896             er->num_diagnostics = num_diagnostics;
 897             return send_srw_response(res);
 898         }
 899     }
 900     return send_http_response(404);
 901 }
 902
 903 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
 904 {
 905     if (m_http_version)
 906     {
 907         if (apdu->which == Z_APDU_initResponse)
 908         {
 909             Z_InitResponse *res = apdu->u.initResponse;
 910             if (*res->result == 0)
 911             {
 912                 send_to_srw_client_error(3, 0);
 913             }
 914             else if (!m_s2z_search_apdu)
 915             {
 916                 send_srw_explain_response(0, 0);
 917             }
 918             else
 919             {
 920                 handle_incoming_Z_PDU(m_s2z_search_apdu);
 921             }
 922         }
 923         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
 924         {
 925             m_s2z_search_apdu = 0;
 926             Z_SearchResponse *res = apdu->u.searchResponse;
 927             m_s2z_hit_count = *res->resultCount;
 928             if (res->records && res->records->which == Z_Records_NSD)
 929             {
 930                 send_to_srw_client_ok(0, res->records, 1);
 931             }
 932             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
 933             {
 934                 // adjust
 935                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
 936
 937                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
 938                 {
 939                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
 940                         > m_s2z_hit_count)
 941                         *pr->numberOfRecordsRequested =
 942                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
 943                 }
 944                 handle_incoming_Z_PDU(m_s2z_present_apdu);
 945             }
 946             else
 947             {
 948                 m_s2z_present_apdu = 0;
 949                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
 950             }
 951         }
 952         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
 953         {
 954             int start =
 955                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
 956
 957             m_s2z_present_apdu = 0;
 958             Z_PresentResponse *res = apdu->u.presentResponse;
 959             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
 960         }
 961     }
 962     else
 963     {
 964         int len = 0;
 965         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
 966             yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
 967                      apdu_name(apdu));
 968         int r = send_Z_PDU(apdu, &len);
 969         m_bytes_sent += len;
 970         m_bw_stat.add_bytes(len);
 971         logtime();
 972         return r;
 973     }
 974     return 0;
 975 }
 976
 977 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 978 {
 979     int kill_session = 0;
 980     if (apdu->which == Z_APDU_searchResponse)
 981     {
 982         Z_SearchResponse *sr = apdu->u.searchResponse;
 983         Z_Records *p = sr->records;
 984         if (p && p->which == Z_Records_NSD)
 985         {
 986             Z_DiagRec dr, *dr_p = &dr;
 987             dr.which = Z_DiagRec_defaultFormat;
 988             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
 989
 990             *sr->searchStatus = 0;
 991             display_diagrecs(&dr_p, 1);
 992         }
 993         else
 994         {
 995             if (p && p->which == Z_Records_DBOSD)
 996             {
 997                 if (m_marcxml_flag)
 998                     convert_to_marcxml(p->u.databaseOrSurDiagnostics);
 999                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1000                     return 0;
1001
1002             }
1003             if (sr->resultCount)
1004             {
1005                 yaz_log(LOG_LOG, "%s%d hits", m_session_str,
1006                         *sr->resultCount);
1007                 if (*sr->resultCount < 0)
1008                 {
1009                     m_invalid_session = 1;
1010                     kill_session = 1;
1011
1012                     *sr->searchStatus = 0;
1013                     sr->records =
1014                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1015                     *sr->resultCount = 0;
1016                 }
1017             }
1018         }
1019     }
1020     else if (apdu->which == Z_APDU_presentResponse)
1021     {
1022         Z_PresentResponse *sr = apdu->u.presentResponse;
1023         Z_Records *p = sr->records;
1024         if (p && p->which == Z_Records_NSD)
1025         {
1026             Z_DiagRec dr, *dr_p = &dr;
1027             dr.which = Z_DiagRec_defaultFormat;
1028             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1029             if (*sr->presentStatus == Z_PresentStatus_success)
1030                 *sr->presentStatus = Z_PresentStatus_failure;
1031             display_diagrecs(&dr_p, 1);
1032         }
1033         if (p && p->which == Z_Records_DBOSD)
1034         {
1035             if (m_marcxml_flag)
1036                 convert_to_marcxml(p->u.databaseOrSurDiagnostics);
1037             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1038                 return 0;
1039         }
1040     }
1041     else if (apdu->which == Z_APDU_initResponse)
1042     {
1043         if (m_initRequest_options)
1044         {
1045             Z_Options *nopt =
1046                 (Odr_bitmask *)odr_malloc(odr_encode(),
1047                                           sizeof(Odr_bitmask));
1048             ODR_MASK_ZERO(nopt);
1049
1050             int i;
1051             for (i = 0; i<24; i++)
1052                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1053                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1054                     ODR_MASK_SET(nopt, i);
1055             apdu->u.initResponse->options = nopt;
1056         }
1057         if (m_initRequest_version)
1058         {
1059             Z_ProtocolVersion *nopt =
1060                 (Odr_bitmask *)odr_malloc(odr_encode(),
1061                                           sizeof(Odr_bitmask));
1062             ODR_MASK_ZERO(nopt);
1063
1064             int i;
1065             for (i = 0; i<8; i++)
1066                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1067                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1068                     ODR_MASK_SET(nopt, i);
1069             apdu->u.initResponse->protocolVersion = nopt;
1070         }
1071     }
1072     int r = send_PDU_convert(apdu);
1073     if (r)
1074         return r;
1075     if (kill_session)
1076     {
1077         delete m_client;
1078         m_client = 0;
1079         m_parent->pre_init();
1080     }
1081     return r;
1082 }
1083
1084 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1085 {
1086     int len = 0;
1087     const char *apdu_name_tmp = apdu_name(apdu);
1088     int r = send_Z_PDU(apdu, &len);
1089     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1090         yaz_log (LOG_LOG, "%sSending %s to %s %d bytes",
1091                  get_session_str(),
1092                  apdu_name_tmp, get_hostname(), len);
1093     m_bytes_sent += len;
1094     return r;
1095 }
1096
1097 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1098 {
1099     if (apdu->which == Z_APDU_presentRequest)
1100     {
1101         Z_PresentRequest *pr = apdu->u.presentRequest;
1102         int toget = *pr->numberOfRecordsRequested;
1103         int start = *pr->resultSetStartPoint;
1104
1105         yaz_log(LOG_LOG, "%sPresent %s %d+%d", m_session_str,
1106                 pr->resultSetId, start, toget);
1107
1108         if (*m_parent->m_optimize == '0')
1109             return apdu;
1110
1111         if (!m_client->m_last_resultSetId)
1112         {
1113             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1114             new_apdu->u.presentResponse->records =
1115                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1116                                                pr->resultSetId);
1117             send_to_client(new_apdu);
1118             return 0;
1119         }
1120         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1121         {
1122             if (start+toget-1 > m_client->m_last_resultCount)
1123             {
1124                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1125                 new_apdu->u.presentResponse->records =
1126                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1127                 send_to_client(new_apdu);
1128                 return 0;
1129             }
1130             Z_NamePlusRecordList *npr;
1131             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1132                                           pr->preferredRecordSyntax,
1133                                           pr->recordComposition))
1134             {
1135                 yaz_log (LOG_LOG, "%sReturned cached records for present request",
1136                          m_session_str);
1137                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1138                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1139
1140                 new_apdu->u.presentResponse->numberOfRecordsReturned
1141                     = odr_intdup(odr_encode(), toget);
1142
1143                 new_apdu->u.presentResponse->records = (Z_Records*)
1144                     odr_malloc(odr_encode(), sizeof(Z_Records));
1145                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1146                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1147                 new_apdu->u.presentResponse->nextResultSetPosition =
1148                     odr_intdup(odr_encode(), start+toget);
1149
1150                 send_to_client(new_apdu);
1151                 return 0;
1152             }
1153         }
1154     }
1155
1156     if (apdu->which != Z_APDU_searchRequest)
1157         return apdu;
1158     Z_SearchRequest *sr = apdu->u.searchRequest;
1159     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1160     Yaz_Z_Databases this_databases;
1161
1162     this_databases.set(sr->num_databaseNames, (const char **)
1163                        sr->databaseNames);
1164
1165     this_query->set_Z_Query(sr->query);
1166
1167     char query_str[120];
1168     this_query->print(query_str, sizeof(query_str)-1);
1169     yaz_log(LOG_LOG, "%sSearch %s", m_session_str, query_str);
1170
1171     if (*m_parent->m_optimize != '0' &&
1172         m_client->m_last_ok && m_client->m_last_query &&
1173         m_client->m_last_query->match(this_query) &&
1174         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1175         m_client->m_last_databases.match(this_databases))
1176     {
1177         delete this_query;
1178         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1179             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1180         {
1181             Z_NamePlusRecordList *npr;
1182             int toget = *sr->mediumSetPresentNumber;
1183             Z_RecordComposition *comp = 0;
1184
1185             if (toget > m_client->m_last_resultCount)
1186                 toget = m_client->m_last_resultCount;
1187
1188             if (sr->mediumSetElementSetNames)
1189             {
1190                 comp = (Z_RecordComposition *)
1191                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1192                 comp->which = Z_RecordComp_simple;
1193                 comp->u.simple = sr->mediumSetElementSetNames;
1194             }
1195
1196             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1197                                           sr->preferredRecordSyntax, comp))
1198             {
1199                 yaz_log (LOG_LOG, "%sReturned cached records for medium set",
1200                          m_session_str);
1201                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1202                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1203                 new_apdu->u.searchResponse->resultCount =
1204                     &m_client->m_last_resultCount;
1205
1206                 new_apdu->u.searchResponse->numberOfRecordsReturned
1207                     = odr_intdup(odr_encode(), toget);
1208
1209                 new_apdu->u.searchResponse->presentStatus =
1210                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1211                 new_apdu->u.searchResponse->records = (Z_Records*)
1212                     odr_malloc(odr_encode(), sizeof(Z_Records));
1213                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1214                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1215                 new_apdu->u.searchResponse->nextResultSetPosition =
1216                     odr_intdup(odr_encode(), toget+1);
1217                 send_to_client(new_apdu);
1218                 return 0;
1219             }
1220             else
1221             {
1222                 // medium Set
1223                 // send present request (medium size)
1224                 yaz_log (LOG_LOG, "%sOptimizing search for medium set",
1225                          m_session_str);
1226
1227                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1228                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1229                 pr->referenceId = sr->referenceId;
1230                 pr->resultSetId = sr->resultSetName;
1231                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1232                 *pr->numberOfRecordsRequested = toget;
1233                 pr->recordComposition = comp;
1234                 m_client->m_sr_transform = 1;
1235                 return new_apdu;
1236             }
1237         }
1238         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1239             m_client->m_last_resultCount <= 0)
1240         {
1241             // large set. Return pseudo-search response immediately
1242             yaz_log (LOG_LOG, "%sOptimizing search for large set",
1243                      m_session_str);
1244             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1245             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1246             new_apdu->u.searchResponse->resultCount =
1247                 &m_client->m_last_resultCount;
1248             send_to_client(new_apdu);
1249             return 0;
1250         }
1251         else
1252         {
1253             Z_NamePlusRecordList *npr;
1254             int toget = m_client->m_last_resultCount;
1255             Z_RecordComposition *comp = 0;
1256             // small set
1257             // send a present request (small set)
1258
1259             if (sr->smallSetElementSetNames)
1260             {
1261                 comp = (Z_RecordComposition *)
1262                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1263                 comp->which = Z_RecordComp_simple;
1264                 comp->u.simple = sr->smallSetElementSetNames;
1265             }
1266
1267             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1268                                           sr->preferredRecordSyntax, comp))
1269             {
1270                 yaz_log (LOG_LOG, "%sReturned cached records for small set",
1271                          m_session_str);
1272                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1273                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1274                 new_apdu->u.searchResponse->resultCount =
1275                     &m_client->m_last_resultCount;
1276
1277                 new_apdu->u.searchResponse->numberOfRecordsReturned
1278                     = odr_intdup(odr_encode(), toget);
1279
1280                 new_apdu->u.searchResponse->presentStatus =
1281                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1282                 new_apdu->u.searchResponse->records = (Z_Records*)
1283                     odr_malloc(odr_encode(), sizeof(Z_Records));
1284                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1285                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1286                 new_apdu->u.searchResponse->nextResultSetPosition =
1287                     odr_intdup(odr_encode(), toget+1);
1288                 send_to_client(new_apdu);
1289                 return 0;
1290             }
1291             else
1292             {
1293                 yaz_log (LOG_LOG, "%sOptimizing search for small set",
1294                          m_session_str);
1295                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1296                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1297                 pr->referenceId = sr->referenceId;
1298                 pr->resultSetId = sr->resultSetName;
1299                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1300                 *pr->numberOfRecordsRequested = toget;
1301                 pr->recordComposition = comp;
1302                 m_client->m_sr_transform = 1;
1303                 return new_apdu;
1304             }
1305         }
1306     }
1307     else  // query doesn't match
1308     {
1309         delete m_client->m_last_query;
1310         m_client->m_last_query = this_query;
1311         m_client->m_last_ok = 0;
1312         m_client->m_cache.clear();
1313         m_client->m_resultSetStartPoint = 0;
1314
1315         xfree (m_client->m_last_resultSetId);
1316         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1317
1318         m_client->m_last_databases.set(sr->num_databaseNames,
1319                                        (const char **) sr->databaseNames);
1320     }
1321     return apdu;
1322 }
1323
1324
1325 void Yaz_Proxy::inc_request_no()
1326 {
1327     char *cp = strchr(m_session_str, ' ');
1328     m_request_no++;
1329     if (cp)
1330         sprintf(cp+1, "%d ", m_request_no);
1331 }
1332
1333 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1334 {
1335     inc_request_no();
1336
1337     m_bytes_recv += len;
1338
1339     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
1340         yaz_log (LOG_DEBUG, "%sReceiving %s from client %d bytes",
1341                  m_session_str, gdu_name(apdu), len);
1342
1343     if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
1344         shutdown();
1345
1346     m_bw_stat.add_bytes(len);
1347     m_pdu_stat.add_bytes(1);
1348
1349     gettimeofday(&m_time_tv, 0);
1350
1351     int bw_total = m_bw_stat.get_total();
1352     int pdu_total = m_pdu_stat.get_total();
1353
1354     int reduce = 0;
1355     if (m_bw_max)
1356     {
1357         if (bw_total > m_bw_max)
1358         {
1359             reduce = (bw_total/m_bw_max);
1360         }
1361     }
1362     if (m_pdu_max)
1363     {
1364         if (pdu_total > m_pdu_max)
1365         {
1366             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1367             reduce = (reduce > nreduce) ? reduce : nreduce;
1368         }
1369     }
1370     if (reduce)
1371     {
1372         yaz_log(LOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1373                 m_session_str, reduce, bw_total, pdu_total,
1374                 m_bw_max, m_pdu_max);
1375
1376         m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
1377         timeout(reduce);       // call us reduce seconds later
1378     }
1379     else if (apdu->which == Z_GDU_Z3950)
1380         handle_incoming_Z_PDU(apdu->u.z3950);
1381     else if (apdu->which == Z_GDU_HTTP_Request)
1382         handle_incoming_HTTP(apdu->u.HTTP_Request);
1383 }
1384
1385 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1386 {
1387     if (m_max_record_retrieve)
1388     {
1389         if (apdu->which == Z_APDU_presentRequest)
1390         {
1391             Z_PresentRequest *pr = apdu->u.presentRequest;
1392             if (pr->numberOfRecordsRequested &&
1393                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1394                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1395         }
1396     }
1397 }
1398
1399 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1400                                                      int error,
1401                                                      const char *addinfo)
1402 {
1403     Z_Records *rec = (Z_Records *)
1404         odr_malloc (odr, sizeof(*rec));
1405     int *err = (int *)
1406         odr_malloc (odr, sizeof(*err));
1407     Z_DiagRec *drec = (Z_DiagRec *)
1408         odr_malloc (odr, sizeof(*drec));
1409     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1410         odr_malloc (odr, sizeof(*dr));
1411     *err = error;
1412     rec->which = Z_Records_NSD;
1413     rec->u.nonSurrogateDiagnostic = dr;
1414     dr->diagnosticSetId =
1415         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1416     dr->condition = err;
1417     dr->which = Z_DefaultDiagFormat_v2Addinfo;
1418     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1419     return rec;
1420 }
1421
1422 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1423 {
1424     if (apdu->which == Z_APDU_searchRequest &&
1425         apdu->u.searchRequest->query &&
1426         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1427         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1428     {
1429         Z_RPNQuery *rpnquery = 0;
1430         Z_SearchRequest *sr = apdu->u.searchRequest;
1431         char *addinfo = 0;
1432
1433         yaz_log(LOG_LOG, "%sCQL: %s", m_session_str,
1434                 sr->query->u.type_104->u.cql);
1435
1436         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1437                                           &rpnquery, odr_encode(),
1438                                           &addinfo);
1439         if (r == -3)
1440             yaz_log(LOG_LOG, "%sNo CQL to RPN table", m_session_str);
1441         else if (r)
1442         {
1443             yaz_log(LOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1444             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1445
1446             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1447             new_apdu->u.searchResponse->records =
1448                 create_nonSurrogateDiagnostics(odr_encode(),
1449                                                yaz_diag_srw_to_bib1(r),
1450                                                addinfo);
1451             *new_apdu->u.searchResponse->searchStatus = 0;
1452
1453             send_to_client(new_apdu);
1454
1455             return 0;
1456         }
1457         else
1458         {
1459             sr->query->which = Z_Query_type_1;
1460             sr->query->u.type_1 = rpnquery;
1461         }
1462         return apdu;
1463     }
1464     return apdu;
1465 }
1466
1467 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1468 {
1469     if (apdu->which == Z_APDU_searchRequest)
1470     {
1471         Z_SearchRequest *sr = apdu->u.searchRequest;
1472         int err = 0;
1473         char *addinfo = 0;
1474
1475         Yaz_ProxyConfig *cfg = check_reconfigure();
1476         if (cfg)
1477             err = cfg->check_query(odr_encode(), m_default_target,
1478                                    sr->query, &addinfo);
1479         if (err)
1480         {
1481             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1482
1483             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1484             new_apdu->u.searchResponse->records =
1485                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1486             *new_apdu->u.searchResponse->searchStatus = 0;
1487
1488             send_to_client(new_apdu);
1489
1490             return 0;
1491         }
1492     }
1493     return apdu;
1494 }
1495
1496 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1497 {
1498     m_marcxml_flag = 0;
1499     if (apdu->which == Z_APDU_searchRequest)
1500     {
1501         Z_SearchRequest *sr = apdu->u.searchRequest;
1502         int err = 0;
1503         char *addinfo = 0;
1504         Yaz_ProxyConfig *cfg = check_reconfigure();
1505
1506         Z_RecordComposition rc_temp, *rc = 0;
1507         if (sr->smallSetElementSetNames)
1508         {
1509             rc_temp.which = Z_RecordComp_simple;
1510             rc_temp.u.simple = sr->smallSetElementSetNames;
1511             rc = &rc_temp;
1512         }
1513
1514         char *stylesheet_name = 0;
1515         if (cfg)
1516             err = cfg->check_syntax(odr_encode(),
1517                                     m_default_target,
1518                                     sr->preferredRecordSyntax, rc,
1519                                     &addinfo, &stylesheet_name, &m_schema);
1520         if (stylesheet_name)
1521         {
1522             if (m_stylesheet_xsp)
1523                 xsltFreeStylesheet(m_stylesheet_xsp);
1524             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1525                                                        stylesheet_name);
1526             m_stylesheet_offset = 0;
1527             xfree(stylesheet_name);
1528         }
1529         if (err == -1)
1530         {
1531             sr->preferredRecordSyntax =
1532                 yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN, VAL_USMARC);
1533             m_marcxml_flag = 1;
1534         }
1535         else if (err)
1536         {
1537             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1538
1539             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1540             new_apdu->u.searchResponse->records =
1541                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1542             *new_apdu->u.searchResponse->searchStatus = 0;
1543
1544             send_to_client(new_apdu);
1545
1546             return 0;
1547         }
1548     }
1549     else if (apdu->which == Z_APDU_presentRequest)
1550     {
1551         Z_PresentRequest *pr = apdu->u.presentRequest;
1552         int err = 0;
1553         char *addinfo = 0;
1554         Yaz_ProxyConfig *cfg = check_reconfigure();
1555
1556         char *stylesheet_name = 0;
1557         if (cfg)
1558             err = cfg->check_syntax(odr_encode(), m_default_target,
1559                                     pr->preferredRecordSyntax,
1560                                     pr->recordComposition,
1561                                     &addinfo, &stylesheet_name, &m_schema);
1562         if (stylesheet_name)
1563         {
1564             if (m_stylesheet_xsp)
1565                 xsltFreeStylesheet(m_stylesheet_xsp);
1566             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
1567                                                        stylesheet_name);
1568             m_stylesheet_offset = 0;
1569             xfree(stylesheet_name);
1570         }
1571         if (err == -1)
1572         {
1573             pr->preferredRecordSyntax =
1574                 yaz_oidval_to_z3950oid(odr_decode(), CLASS_RECSYN, VAL_USMARC);
1575             m_marcxml_flag = 1;
1576         }
1577         else if (err)
1578         {
1579             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1580
1581             new_apdu->u.presentResponse->referenceId = pr->referenceId;
1582             new_apdu->u.presentResponse->records =
1583                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1584             *new_apdu->u.presentResponse->presentStatus =
1585                 Z_PresentStatus_failure;
1586
1587             send_to_client(new_apdu);
1588
1589             return 0;
1590         }
1591     }
1592     return apdu;
1593 }
1594
1595 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
1596 {
1597     if (!schema)
1598         return 0;
1599     Z_ElementSetNames *esn = (Z_ElementSetNames *)
1600         odr_malloc(o, sizeof(Z_ElementSetNames));
1601     esn->which = Z_ElementSetNames_generic;
1602     esn->u.generic = odr_strdup(o, schema);
1603     return esn;
1604 }
1605
1606 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1607 {
1608     if (m_s2z_odr_init)
1609     {
1610         odr_destroy(m_s2z_odr_init);
1611         m_s2z_odr_init = 0;
1612     }
1613     if (m_s2z_odr_search)
1614     {
1615         odr_destroy(m_s2z_odr_search);
1616         m_s2z_odr_search = 0;
1617     }
1618
1619     m_http_keepalive = 0;
1620     m_http_version = 0;
1621     if (!strcmp(hreq->version, "1.0"))
1622     {
1623         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1624         if (v && !strcmp(v, "Keep-Alive"))
1625             m_http_keepalive = 1;
1626         else
1627             m_http_keepalive = 0;
1628         m_http_version = "1.0";
1629     }
1630     else
1631     {
1632         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1633         if (v && !strcmp(v, "close"))
1634             m_http_keepalive = 0;
1635         else
1636             m_http_keepalive = 1;
1637         m_http_version = "1.1";
1638     }
1639
1640     Z_SRW_PDU *srw_pdu = 0;
1641     Z_SOAP *soap_package = 0;
1642     char *charset = 0;
1643     Z_SRW_diagnostic *diagnostic = 0;
1644     int num_diagnostic = 0;
1645     if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1646                        &charset) == 0
1647         || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1648                           &charset, &diagnostic, &num_diagnostic) == 0)
1649     {
1650         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
1651         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
1652         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
1653         m_s2z_init_apdu = 0;
1654         m_s2z_search_apdu = 0;
1655         m_s2z_present_apdu = 0;
1656
1657         m_s2z_stylesheet = 0;
1658
1659         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
1660         {
1661             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
1662
1663             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1664             // recordXPath unsupported.
1665             if (srw_req->recordXPath)
1666             {
1667                 yaz_add_srw_diagnostic(odr_decode(),
1668                                        &diagnostic, &num_diagnostic,
1669                                        72, 0);
1670             }
1671             // must have a query
1672             if (!srw_req->query.cql)
1673             {
1674                 yaz_add_srw_diagnostic(odr_decode(),
1675                                        &diagnostic, &num_diagnostic,
1676                                        7, "query");
1677             }
1678             // sort unsupported
1679             if (srw_req->sort_type != Z_SRW_sort_type_none)
1680             {
1681                 yaz_add_srw_diagnostic(odr_decode(),
1682                                        &diagnostic, &num_diagnostic,
1683                                        80, 0);
1684             }
1685             // save stylesheet
1686             if (srw_req->stylesheet)
1687                 m_s2z_stylesheet =
1688                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1689
1690             // set packing for response records ..
1691             if (srw_req->recordPacking &&
1692                 !strcmp(srw_req->recordPacking, "xml"))
1693                 m_s2z_packing = Z_SRW_recordPacking_XML;
1694             else
1695                 m_s2z_packing = Z_SRW_recordPacking_string;
1696
1697             if (num_diagnostic)
1698             {
1699                 Z_SRW_PDU *srw_pdu =
1700                     yaz_srw_get(odr_encode(),
1701                                 Z_SRW_searchRetrieve_response);
1702                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1703
1704                 srw_res->diagnostics = diagnostic;
1705                 srw_res->num_diagnostics = num_diagnostic;
1706                 send_srw_response(srw_pdu);
1707                 return;
1708             }
1709
1710             // prepare search PDU
1711             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
1712                                           Z_APDU_searchRequest);
1713             Z_SearchRequest *z_searchRequest =
1714                 m_s2z_search_apdu->u.searchRequest;
1715
1716             z_searchRequest->num_databaseNames = 1;
1717             z_searchRequest->databaseNames = (char**)
1718                 odr_malloc(m_s2z_odr_search, sizeof(char *));
1719             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
1720                                                            srw_req->database);
1721
1722             // query transformation
1723             Z_Query *query = (Z_Query *)
1724                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
1725             z_searchRequest->query = query;
1726
1727             if (srw_req->query_type == Z_SRW_query_type_cql)
1728             {
1729                 Z_External *ext = (Z_External *)
1730                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
1731                 ext->direct_reference =
1732                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
1733                 ext->indirect_reference = 0;
1734                 ext->descriptor = 0;
1735                 ext->which = Z_External_CQL;
1736                 ext->u.cql = srw_req->query.cql;
1737
1738                 query->which = Z_Query_type_104;
1739                 query->u.type_104 =  ext;
1740             }
1741             else if (srw_req->query_type == Z_SRW_query_type_pqf)
1742             {
1743                 Z_RPNQuery *RPNquery;
1744                 YAZ_PQF_Parser pqf_parser;
1745
1746                 pqf_parser = yaz_pqf_create ();
1747
1748                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
1749                                           srw_req->query.pqf);
1750                 if (!RPNquery)
1751                 {
1752                     const char *pqf_msg;
1753                     size_t off;
1754                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
1755                     yaz_log(LOG_LOG, "%*s^\n", off+4, "");
1756                     yaz_log(LOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
1757
1758                     send_to_srw_client_error(10, 0);
1759                     return;
1760                 }
1761                 query->which = Z_Query_type_1;
1762                 query->u.type_1 =  RPNquery;
1763
1764                 yaz_pqf_destroy (pqf_parser);
1765             }
1766             else
1767             {
1768                 send_to_srw_client_error(7, "query");
1769                 return;
1770             }
1771
1772             // present
1773             m_s2z_present_apdu = 0;
1774             int max = 0;
1775             if (srw_req->maximumRecords)
1776                 max = *srw_req->maximumRecords;
1777             int start = 1;
1778             if (srw_req->startRecord)
1779                 start = *srw_req->startRecord;
1780             if (max > 0)
1781             {
1782                 // Some backend, such as Voyager doesn't honor piggyback
1783                 // So we use present always (0 &&).
1784                 if (0 && start <= 1)  // Z39.50 piggyback
1785                 {
1786                     *z_searchRequest->smallSetUpperBound = max;
1787                     *z_searchRequest->mediumSetPresentNumber = max;
1788                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
1789
1790                     z_searchRequest->preferredRecordSyntax =
1791                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1792                                                VAL_TEXT_XML);
1793                     if (srw_req->recordSchema)
1794                     {
1795                         z_searchRequest->smallSetElementSetNames =
1796                             z_searchRequest->mediumSetElementSetNames =
1797                             mk_esn_from_schema(m_s2z_odr_search,
1798                                                srw_req->recordSchema);
1799                     }
1800                 }
1801                 else   // Z39.50 present
1802                 {
1803                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
1804                                                    Z_APDU_presentRequest);
1805                     Z_PresentRequest *z_presentRequest =
1806                         m_s2z_present_apdu->u.presentRequest;
1807                     *z_presentRequest->resultSetStartPoint = start;
1808                     *z_presentRequest->numberOfRecordsRequested = max;
1809                     z_presentRequest->preferredRecordSyntax =
1810                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1811                                                VAL_TEXT_XML);
1812                     if (srw_req->recordSchema)
1813                     {
1814                         z_presentRequest->recordComposition =
1815                             (Z_RecordComposition *)
1816                             odr_malloc(m_s2z_odr_search,
1817                                        sizeof(Z_RecordComposition));
1818                         z_presentRequest->recordComposition->which =
1819                             Z_RecordComp_simple;
1820                         z_presentRequest->recordComposition->u.simple =
1821                             mk_esn_from_schema(m_s2z_odr_search,
1822                                                srw_req->recordSchema);
1823                     }
1824                 }
1825             }
1826             if (!m_client)
1827             {
1828                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1829                                             Z_APDU_initRequest);
1830
1831                 // prevent m_initRequest_apdu memory from being grabbed
1832                 // in Yaz_Proxy::handle_incoming_Z_PDU
1833                 m_initRequest_apdu = m_s2z_init_apdu;
1834                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1835                 return;
1836             }
1837             else
1838             {
1839                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1840                 return;
1841             }
1842         }
1843         else if (srw_pdu->which == Z_SRW_explain_request)
1844         {
1845             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
1846
1847             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1848
1849             // save stylesheet
1850             if (srw_req->stylesheet)
1851                 m_s2z_stylesheet =
1852                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1853
1854             if (srw_req->recordPacking &&
1855                 !strcmp(srw_req->recordPacking, "xml"))
1856                 m_s2z_packing = Z_SRW_recordPacking_XML;
1857             else
1858                 m_s2z_packing = Z_SRW_recordPacking_string;
1859
1860             if (num_diagnostic)
1861             {
1862                 send_srw_explain_response(diagnostic, num_diagnostic);
1863                 return;
1864             }
1865
1866             if (!m_client)
1867             {
1868                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1869                                             Z_APDU_initRequest);
1870
1871                 // prevent m_initRequest_apdu memory from being grabbed
1872                 // in Yaz_Proxy::handle_incoming_Z_PDU
1873                 m_initRequest_apdu = m_s2z_init_apdu;
1874                 handle_incoming_Z_PDU(m_s2z_init_apdu);
1875             }
1876             else
1877                 send_srw_explain_response(0, 0);
1878             return;
1879         }
1880         else if (srw_pdu->which == Z_SRW_scan_request)
1881         {
1882             m_s2z_database = odr_strdup(m_s2z_odr_init,
1883                                         srw_pdu->u.scan_request->database);
1884
1885             yaz_add_srw_diagnostic(odr_decode(),
1886                                    &diagnostic, &num_diagnostic,
1887                                    4, "scan");
1888             Z_SRW_PDU *srw_pdu =
1889                 yaz_srw_get(odr_encode(),
1890                             Z_SRW_scan_response);
1891             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1892
1893             srw_res->diagnostics = diagnostic;
1894             srw_res->num_diagnostics = num_diagnostic;
1895             send_srw_response(srw_pdu);
1896             return;
1897         }
1898         else
1899         {
1900             m_s2z_database = 0;
1901
1902             send_to_srw_client_error(4, 0);
1903         }
1904     }
1905     int len = 0;
1906     Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
1907     timeout(0);
1908     send_GDU(p, &len);
1909 }
1910
1911 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
1912 {
1913     if (!m_client && m_invalid_session)
1914     {
1915         m_apdu_invalid_session = apdu;
1916         m_mem_invalid_session = odr_extract_mem(odr_decode());
1917         apdu = m_initRequest_apdu;
1918     }
1919
1920     // Determine our client.
1921     Z_OtherInformation **oi;
1922     get_otherInfoAPDU(apdu, &oi);
1923     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
1924     if (!m_client)
1925     {
1926         delete this;
1927         return;
1928     }
1929     m_client->m_server = this;
1930
1931     if (apdu->which == Z_APDU_initRequest)
1932     {
1933         if (apdu->u.initRequest->implementationId)
1934             yaz_log(LOG_LOG, "%simplementationId: %s",
1935                     m_session_str, apdu->u.initRequest->implementationId);
1936         if (apdu->u.initRequest->implementationName)
1937             yaz_log(LOG_LOG, "%simplementationName: %s",
1938                     m_session_str, apdu->u.initRequest->implementationName);
1939         if (apdu->u.initRequest->implementationVersion)
1940             yaz_log(LOG_LOG, "%simplementationVersion: %s",
1941                     m_session_str, apdu->u.initRequest->implementationVersion);
1942         if (m_initRequest_apdu == 0)
1943         {
1944             if (m_initRequest_mem)
1945                 nmem_destroy(m_initRequest_mem);
1946             m_initRequest_apdu = apdu;
1947             m_initRequest_mem = odr_extract_mem(odr_decode());
1948
1949             // save init options for the response..
1950             m_initRequest_options = apdu->u.initRequest->options;
1951
1952             apdu->u.initRequest->options =
1953                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
1954                                            sizeof(Odr_bitmask));
1955             ODR_MASK_ZERO(apdu->u.initRequest->options);
1956             int i;
1957             for (i = 0; i<= 24; i++)
1958                 ODR_MASK_SET(apdu->u.initRequest->options, i);
1959             ODR_MASK_CLEAR(apdu->u.initRequest->options,
1960                            Z_Options_negotiationModel);
1961
1962             // make new version
1963             m_initRequest_version = apdu->u.initRequest->protocolVersion;
1964             apdu->u.initRequest->protocolVersion =
1965                 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
1966                                            sizeof(Odr_bitmask));
1967             ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
1968
1969             for (i = 0; i<= 8; i++)
1970                 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
1971         }
1972         if (m_client->m_init_flag)
1973         {
1974             if (handle_init_response_for_invalid_session(apdu))
1975                 return;
1976             Z_APDU *apdu2 = m_client->m_initResponse;
1977             apdu2->u.initResponse->otherInfo = 0;
1978             if (m_client->m_cookie && *m_client->m_cookie)
1979                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
1980                                            m_client->m_cookie);
1981             apdu2->u.initResponse->referenceId =
1982                 apdu->u.initRequest->referenceId;
1983             apdu2->u.initResponse->options = m_client->m_initResponse_options;
1984             apdu2->u.initResponse->protocolVersion =
1985                 m_client->m_initResponse_version;
1986
1987             send_to_client(apdu2);
1988             return;
1989         }
1990         m_client->m_init_flag = 1;
1991     }
1992     handle_max_record_retrieve(apdu);
1993
1994     if (apdu)
1995         apdu = handle_syntax_validation(apdu);
1996
1997     if (apdu)
1998         apdu = handle_query_transformation(apdu);
1999
2000     if (apdu)
2001         apdu = handle_query_validation(apdu);
2002
2003     if (apdu)
2004         apdu = result_set_optimize(apdu);
2005     if (!apdu)
2006     {
2007         m_client->timeout(m_target_idletime);  // mark it active even
2008         // though we didn't use it
2009         return;
2010     }
2011
2012     // delete other info part from PDU before sending to target
2013     get_otherInfoAPDU(apdu, &oi);
2014     if (oi)
2015         *oi = 0;
2016
2017     if (apdu->which == Z_APDU_presentRequest &&
2018         m_client->m_resultSetStartPoint == 0)
2019     {
2020         Z_PresentRequest *pr = apdu->u.presentRequest;
2021         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2022         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
2023     } else {
2024         m_client->m_resultSetStartPoint = 0;
2025     }
2026     if (m_client->send_to_target(apdu) < 0)
2027     {
2028         delete m_client;
2029         m_client = 0;
2030         delete this;
2031     }
2032     else
2033         m_client->m_waiting = 1;
2034 }
2035
2036 void Yaz_Proxy::connectNotify()
2037 {
2038 }
2039
2040 void Yaz_Proxy::shutdown()
2041 {
2042     m_invalid_session = 0;
2043     // only keep if keep_alive flag is set...
2044     if (m_client &&
2045         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2046         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2047         m_client->m_waiting == 0)
2048     {
2049         yaz_log(LOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2050                  m_session_str,
2051                  m_client->get_hostname());
2052         yaz_log(LOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2053                 m_session_str, m_client->m_pdu_recv,
2054                 m_client->m_bytes_sent + m_client->m_bytes_recv,
2055                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2056         assert (m_client->m_waiting != 2);
2057         // Tell client (if any) that no server connection is there..
2058         m_client->m_server = 0;
2059         m_invalid_session = 0;
2060     }
2061     else if (m_client)
2062     {
2063         yaz_log (LOG_LOG, "%sShutdown (client to proxy) close %s",
2064                  m_session_str,
2065                  m_client->get_hostname());
2066         assert (m_client->m_waiting != 2);
2067         delete m_client;
2068     }
2069     else if (!m_parent)
2070     {
2071         yaz_log (LOG_LOG, "%sshutdown (client to proxy) bad state",
2072                  m_session_str);
2073         assert (m_parent);
2074     }
2075     else
2076     {
2077         yaz_log (LOG_LOG, "%sShutdown (client to proxy)",
2078                  m_session_str);
2079     }
2080     if (m_parent)
2081         m_parent->pre_init();
2082     delete this;
2083 }
2084
2085 const char *Yaz_ProxyClient::get_session_str()
2086 {
2087     if (!m_server)
2088         return "0 ";
2089     return m_server->get_session_str();
2090 }
2091
2092 void Yaz_ProxyClient::shutdown()
2093 {
2094     yaz_log (LOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2095              get_hostname());
2096     delete m_server;
2097     delete this;
2098 }
2099
2100 void Yaz_Proxy::failNotify()
2101 {
2102     inc_request_no();
2103     yaz_log (LOG_LOG, "%sConnection closed by client",
2104              get_session_str());
2105     shutdown();
2106 }
2107
2108 void Yaz_ProxyClient::failNotify()
2109 {
2110     if (m_server)
2111         m_server->inc_request_no();
2112     yaz_log (LOG_LOG, "%sConnection closed by target %s",
2113              get_session_str(), get_hostname());
2114     shutdown();
2115 }
2116
2117 void Yaz_ProxyClient::connectNotify()
2118 {
2119     const char *s = get_session_str();
2120     const char *h = get_hostname();
2121     yaz_log (LOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2122              m_target_idletime);
2123     timeout(m_target_idletime);
2124     if (!m_server)
2125         pre_init_client();
2126 }
2127
2128 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2129                                                   *the_PDU_Observable, int fd)
2130 {
2131     return new Yaz_ProxyClient(the_PDU_Observable, 0);
2132 }
2133
2134 Yaz_ProxyClient::~Yaz_ProxyClient()
2135 {
2136     if (m_prev)
2137         *m_prev = m_next;
2138     if (m_next)
2139         m_next->m_prev = m_prev;
2140     m_waiting = 2;     // for debugging purposes only.
2141     odr_destroy(m_init_odr);
2142     delete m_last_query;
2143     xfree (m_last_resultSetId);
2144     xfree (m_cookie);
2145 }
2146
2147 void Yaz_ProxyClient::pre_init_client()
2148 {
2149     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2150     Z_InitRequest *req = apdu->u.initRequest;
2151
2152     int i;
2153     for (i = 0; i<= 24; i++)
2154         ODR_MASK_SET(req->options, i);
2155     ODR_MASK_CLEAR(apdu->u.initRequest->options,
2156                    Z_Options_negotiationModel);
2157     for (i = 0; i<= 10; i++)
2158         ODR_MASK_SET(req->protocolVersion, i);
2159
2160     if (send_to_target(apdu) < 0)
2161     {
2162         delete this;
2163     }
2164     else
2165     {
2166         m_waiting = 1;
2167         m_init_flag = 1;
2168     }
2169 }
2170
2171 void Yaz_Proxy::pre_init()
2172 {
2173     int i;
2174     const char *name = 0;
2175     const char *zurl_in_use[MAX_ZURL_PLEX];
2176     int limit_bw, limit_pdu, limit_req;
2177     int target_idletime, client_idletime;
2178     int max_clients;
2179     int keepalive_limit_bw, keepalive_limit_pdu;
2180     int pre_init;
2181     const char *cql2rpn = 0;
2182
2183     Yaz_ProxyConfig *cfg = check_reconfigure();
2184
2185     zurl_in_use[0] = 0;
2186
2187     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2188         set_APDU_yazlog(1);
2189     else
2190         set_APDU_yazlog(0);
2191
2192     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2193                                           &limit_bw, &limit_pdu, &limit_req,
2194                                           &target_idletime, &client_idletime,
2195                                           &max_clients,
2196                                           &keepalive_limit_bw,
2197                                           &keepalive_limit_pdu,
2198                                           &pre_init,
2199                                           &cql2rpn) ; i++)
2200     {
2201         if (pre_init)
2202         {
2203             int j;
2204             for (j = 0; zurl_in_use[j]; j++)
2205             {
2206                 Yaz_ProxyClient *c;
2207                 int spare = 0;
2208                 int in_use = 0;
2209                 int other = 0;
2210                 for (c = m_clientPool; c; c = c->m_next)
2211                 {
2212                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
2213                     {
2214                         if (c->m_cookie == 0)
2215                         {
2216                             if (c->m_server == 0)
2217                                 spare++;
2218                             else
2219                                 in_use++;
2220                         }
2221                         else
2222                             other++;
2223                     }
2224                 }
2225                 yaz_log(LOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2226                         "preinit=%d",m_session_str,
2227                         name, zurl_in_use[j], in_use, other, spare, pre_init);
2228                 if (spare < pre_init)
2229                 {
2230                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2231                     c->m_next = m_clientPool;
2232                     if (c->m_next)
2233                         c->m_next->m_prev = &c->m_next;
2234                     m_clientPool = c;
2235                     c->m_prev = &m_clientPool;
2236
2237                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
2238                         c->set_APDU_yazlog(1);
2239                     else
2240                         c->set_APDU_yazlog(0);
2241
2242                     if (c->client(zurl_in_use[j]))
2243                     {
2244                         timeout(60);
2245                         delete c;
2246                         return;
2247                     }
2248                     c->timeout(30);
2249                     c->m_waiting = 1;
2250                     c->m_target_idletime = target_idletime;
2251                     c->m_seqno = m_seqno++;
2252                 }
2253             }
2254         }
2255     }
2256 }
2257
2258 void Yaz_Proxy::timeoutNotify()
2259 {
2260     if (m_parent)
2261     {
2262         if (m_bw_hold_PDU)
2263         {
2264             timeout(m_client_idletime);
2265             Z_GDU *apdu = m_bw_hold_PDU;
2266             m_bw_hold_PDU = 0;
2267
2268             if (apdu->which == Z_GDU_Z3950)
2269                 handle_incoming_Z_PDU(apdu->u.z3950);
2270             else if (apdu->which == Z_GDU_HTTP_Request)
2271                 handle_incoming_HTTP(apdu->u.HTTP_Request);
2272         }
2273         else if (m_stylesheet_nprl)
2274             convert_xsl_delay();
2275         else
2276         {
2277             inc_request_no();
2278
2279             yaz_log (LOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2280             shutdown();
2281         }
2282     }
2283     else
2284     {
2285         timeout(600);
2286         pre_init();
2287     }
2288 }
2289
2290 void Yaz_Proxy::markInvalid()
2291 {
2292     m_client = 0;
2293     m_invalid_session = 1;
2294 }
2295
2296 void Yaz_ProxyClient::timeoutNotify()
2297 {
2298     if (m_server)
2299         m_server->inc_request_no();
2300
2301     yaz_log (LOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2302              get_hostname());
2303     m_waiting = 1;
2304     m_root->pre_init();
2305     if (m_server && m_init_flag)
2306     {
2307         // target timed out in a session that was properly initialized
2308         // server object stay alive but we mark it as invalid so it
2309         // gets initialized again
2310         m_server->markInvalid();
2311         m_server = 0;
2312     }
2313     shutdown();
2314 }
2315
2316 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2317                                  Yaz_Proxy *parent) :
2318     Yaz_Z_Assoc (the_PDU_Observable)
2319 {
2320     m_cookie = 0;
2321     m_next = 0;
2322     m_prev = 0;
2323     m_init_flag = 0;
2324     m_last_query = 0;
2325     m_last_resultSetId = 0;
2326     m_last_resultCount = 0;
2327     m_last_ok = 0;
2328     m_sr_transform = 0;
2329     m_waiting = 0;
2330     m_init_odr = odr_createmem (ODR_DECODE);
2331     m_initResponse = 0;
2332     m_initResponse_options = 0;
2333     m_initResponse_version = 0;
2334     m_resultSetStartPoint = 0;
2335     m_bytes_sent = m_bytes_recv = 0;
2336     m_pdu_recv = 0;
2337     m_server = 0;
2338     m_seqno = 0;
2339     m_target_idletime = 600;
2340     m_root = parent;
2341 }
2342
2343 const char *Yaz_Proxy::option(const char *name, const char *value)
2344 {
2345     if (!strcmp (name, "optimize")) {
2346         if (value) {
2347             xfree (m_optimize);
2348             m_optimize = xstrdup (value);
2349         }
2350         return m_optimize;
2351     }
2352     return 0;
2353 }
2354
2355 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2356 {
2357
2358 }
2359
2360 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2361 {
2362     if (apdu->which == Z_GDU_Z3950)
2363         recv_Z_PDU(apdu->u.z3950, len);
2364     else if (apdu->which == Z_GDU_HTTP_Response)
2365         recv_HTTP_response(apdu->u.HTTP_Response, len);
2366     else
2367         shutdown();
2368 }
2369
2370 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2371 {
2372     if (!m_invalid_session)
2373         return 0;
2374     m_invalid_session = 0;
2375     handle_incoming_Z_PDU(m_apdu_invalid_session);
2376     assert (m_mem_invalid_session);
2377     nmem_destroy(m_mem_invalid_session);
2378     m_mem_invalid_session = 0;
2379     return 1;
2380 }
2381
2382 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2383 {
2384     m_bytes_recv += len;
2385
2386     m_pdu_recv++;
2387     m_waiting = 0;
2388     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2389         yaz_log (LOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2390                  apdu_name(apdu), get_hostname(), len);
2391     if (apdu->which == Z_APDU_initResponse)
2392     {
2393         if (!m_server)  // if this is a pre init session , check for more
2394             m_root->pre_init();
2395         NMEM nmem = odr_extract_mem (odr_decode());
2396         odr_reset (m_init_odr);
2397         nmem_transfer (m_init_odr->mem, nmem);
2398         m_initResponse = apdu;
2399         m_initResponse_options = apdu->u.initResponse->options;
2400         m_initResponse_version = apdu->u.initResponse->protocolVersion;
2401
2402         Z_InitResponse *ir = apdu->u.initResponse;
2403         char *im0 = ir->implementationName;
2404
2405         char *im1 = (char*)
2406             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2407         *im1 = '\0';
2408         if (im0)
2409         {
2410             strcat(im1, im0);
2411             strcat(im1, " ");
2412         }
2413         strcat(im1, "(YAZ Proxy)");
2414         ir->implementationName = im1;
2415
2416         nmem_destroy (nmem);
2417
2418         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2419             return;
2420     }
2421     if (apdu->which == Z_APDU_searchResponse)
2422     {
2423         Z_SearchResponse *sr = apdu->u.searchResponse;
2424         m_last_resultCount = *sr->resultCount;
2425         int status = *sr->searchStatus;
2426         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2427         {
2428             m_last_ok = 1;
2429
2430             if (sr->records && sr->records->which == Z_Records_DBOSD)
2431             {
2432                 m_cache.add(odr_decode(),
2433                             sr->records->u.databaseOrSurDiagnostics, 1,
2434                             *sr->resultCount);
2435             }
2436         }
2437     }
2438     if (apdu->which == Z_APDU_presentResponse)
2439     {
2440         Z_PresentResponse *pr = apdu->u.presentResponse;
2441         if (m_sr_transform)
2442         {
2443             m_sr_transform = 0;
2444             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2445             Z_SearchResponse *sr = new_apdu->u.searchResponse;
2446             sr->referenceId = pr->referenceId;
2447             *sr->resultCount = m_last_resultCount;
2448             sr->records = pr->records;
2449             sr->nextResultSetPosition = pr->nextResultSetPosition;
2450             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2451             apdu = new_apdu;
2452         }
2453         if (pr->records &&
2454             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2455         {
2456             m_cache.add(odr_decode(),
2457                         pr->records->u.databaseOrSurDiagnostics,
2458                         m_resultSetStartPoint, -1);
2459             m_resultSetStartPoint = 0;
2460         }
2461     }
2462     if (m_cookie)
2463         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2464     if (m_server)
2465     {
2466         m_server->send_to_client(apdu);
2467     }
2468     if (apdu->which == Z_APDU_close)
2469     {
2470         shutdown();
2471     }
2472 }
2473
2474 int Yaz_Proxy::server(const char *addr)
2475 {
2476     int r = Yaz_Z_Assoc::server(addr);
2477     if (!r)
2478     {
2479         yaz_log(LOG_LOG, "%sStarted proxy " VERSION " on %s", m_session_str, addr);
2480         timeout(1);
2481     }
2482     return r;
2483 }
2484